Repository: proxmox/qemu-server Branch: master Commit: 1412b2b4c697 Files: 427 Total size: 1.9 MB Directory structure: gitextract_1y8lnyny/ ├── .gitignore ├── Makefile ├── debian/ │ ├── changelog │ ├── control │ ├── copyright │ ├── docs │ ├── rules │ ├── source/ │ │ └── format │ └── triggers └── src/ ├── Makefile ├── PVE/ │ ├── API2/ │ │ ├── Makefile │ │ ├── NodeCapabilities/ │ │ │ ├── Makefile │ │ │ └── Qemu/ │ │ │ └── Migration.pm │ │ ├── Qemu/ │ │ │ ├── Agent.pm │ │ │ ├── CPU.pm │ │ │ ├── CPUFlags.pm │ │ │ ├── HMPPerms.pm │ │ │ ├── Machine.pm │ │ │ └── Makefile │ │ └── Qemu.pm │ ├── CLI/ │ │ ├── Makefile │ │ ├── qm.pm │ │ └── qmrestore.pm │ ├── Makefile │ ├── QMPClient.pm │ ├── QemuConfig/ │ │ ├── Makefile │ │ └── NoWrite.pm │ ├── QemuConfig.pm │ ├── QemuMigrate/ │ │ ├── Helpers.pm │ │ └── Makefile │ ├── QemuMigrate.pm │ ├── QemuServer/ │ │ ├── Agent.pm │ │ ├── BlockJob.pm │ │ ├── Blockdev.pm │ │ ├── CGroup.pm │ │ ├── CPUConfig.pm │ │ ├── Cfg2Cmd/ │ │ │ ├── Makefile │ │ │ └── Timer.pm │ │ ├── Cfg2Cmd.pm │ │ ├── Cloudinit.pm │ │ ├── DBusVMState.pm │ │ ├── Drive.pm │ │ ├── DriveDevice.pm │ │ ├── Helpers.pm │ │ ├── ImportDisk.pm │ │ ├── Machine.pm │ │ ├── Makefile │ │ ├── Memory.pm │ │ ├── MetaInfo.pm │ │ ├── Monitor.pm │ │ ├── Network.pm │ │ ├── OVMF.pm │ │ ├── PCI.pm │ │ ├── QMPHelpers.pm │ │ ├── QSD.pm │ │ ├── QemuImage.pm │ │ ├── RNG.pm │ │ ├── RunState.pm │ │ ├── StateFile.pm │ │ ├── USB.pm │ │ ├── Virtiofs.pm │ │ └── VolumeChain.pm │ ├── QemuServer.pm │ └── VZDump/ │ ├── Makefile │ └── QemuServer.pm ├── bin/ │ ├── Makefile │ ├── qm │ ├── qmextract │ └── qmrestore ├── qmeventd/ │ ├── .clang-format │ ├── Makefile │ ├── qmeventd.c │ ├── qmeventd.h │ └── qmeventd.service ├── query-machine-capabilities/ │ ├── Makefile │ ├── pve-query-machine-capabilities.service │ └── query-machine-capabilities.c ├── test/ │ ├── Makefile │ ├── MigrationTest/ │ │ ├── QemuMigrateMock.pm │ │ ├── QmMock.pm │ │ └── Shared.pm │ ├── cfg2cmd/ │ │ ├── README.adoc │ │ ├── aarch64/ │ │ │ ├── simple-arm-host.conf │ │ │ ├── simple-arm-host.conf.cmd │ │ │ ├── simple-arm.conf │ │ │ ├── simple-arm.conf.cmd │ │ │ ├── simple-x86-on-arm-host.conf │ │ │ └── simple-x86-on-arm-host.conf.cmd │ │ ├── aio.conf │ │ ├── aio.conf.cmd │ │ ├── bootorder-empty.conf │ │ ├── bootorder-empty.conf.cmd │ │ ├── bootorder-legacy.conf │ │ ├── bootorder-legacy.conf.cmd │ │ ├── bootorder.conf │ │ ├── bootorder.conf.cmd │ │ ├── cputype-icelake-client-deprecation.conf │ │ ├── cputype-icelake-client-deprecation.conf.cmd │ │ ├── custom-cpu-model-defaults.conf │ │ ├── custom-cpu-model-defaults.conf.cmd │ │ ├── custom-cpu-model-host-phys-bits.conf │ │ ├── custom-cpu-model-host-phys-bits.conf.cmd │ │ ├── custom-cpu-model.conf │ │ ├── custom-cpu-model.conf.cmd │ │ ├── efi-ovmf-without-efidisk.conf │ │ ├── efi-ovmf-without-efidisk.conf.cmd │ │ ├── efi-raw-old.conf │ │ ├── efi-raw-old.conf.cmd │ │ ├── efi-raw-template.conf │ │ ├── efi-raw-template.conf.cmd │ │ ├── efi-raw.conf │ │ ├── efi-raw.conf.cmd │ │ ├── efi-secboot-and-tpm-q35.conf │ │ ├── efi-secboot-and-tpm-q35.conf.cmd │ │ ├── efi-secboot-and-tpm.conf │ │ ├── efi-secboot-and-tpm.conf.cmd │ │ ├── efidisk-on-rbd.conf │ │ ├── efidisk-on-rbd.conf.cmd │ │ ├── i440fx-viommu-intel.conf │ │ ├── i440fx-viommu-virtio.conf │ │ ├── i440fx-viommu-virtio.conf.cmd │ │ ├── i440fx-win10-hostpci.conf │ │ ├── i440fx-win10-hostpci.conf.cmd │ │ ├── ide-no-media-error.conf │ │ ├── ide.conf │ │ ├── ide.conf.cmd │ │ ├── memory-hotplug-hugepages.conf │ │ ├── memory-hotplug-hugepages.conf.cmd │ │ ├── memory-hotplug.conf │ │ ├── memory-hotplug.conf.cmd │ │ ├── memory-hugepages-1g.conf │ │ ├── memory-hugepages-1g.conf.cmd │ │ ├── memory-hugepages-2m.conf │ │ ├── memory-hugepages-2m.conf.cmd │ │ ├── minimal-defaults-to-new-machine.conf │ │ ├── minimal-defaults-unsupported-pve-version.conf │ │ ├── minimal-defaults.conf │ │ ├── minimal-defaults.conf.cmd │ │ ├── netdev-7.0-multiqueues.conf │ │ ├── netdev-7.0-multiqueues.conf.cmd │ │ ├── netdev-7.1-multiqueues.conf │ │ ├── netdev-7.1-multiqueues.conf.cmd │ │ ├── netdev-7.1.conf │ │ ├── netdev-7.1.conf.cmd │ │ ├── netdev.conf │ │ ├── netdev.conf.cmd │ │ ├── netdev_vxlan.conf │ │ ├── netdev_vxlan.conf.cmd │ │ ├── old-qemu.conf │ │ ├── os-l24.conf │ │ ├── os-l24.conf.cmd │ │ ├── os-other.conf │ │ ├── os-other.conf.cmd │ │ ├── os-solaris.conf │ │ ├── os-solaris.conf.cmd │ │ ├── ostype-usb13-error.conf │ │ ├── pinned-version-pxe-pve.conf │ │ ├── pinned-version-pxe-pve.conf.cmd │ │ ├── pinned-version-pxe.conf │ │ ├── pinned-version-pxe.conf.cmd │ │ ├── pinned-version.conf │ │ ├── pinned-version.conf.cmd │ │ ├── q35-ide.conf │ │ ├── q35-ide.conf.cmd │ │ ├── q35-linux-hostpci-driver-keep.conf │ │ ├── q35-linux-hostpci-driver-keep.conf.cmd │ │ ├── q35-linux-hostpci-mapping.conf │ │ ├── q35-linux-hostpci-mapping.conf.cmd │ │ ├── q35-linux-hostpci-multifunction.conf │ │ ├── q35-linux-hostpci-multifunction.conf.cmd │ │ ├── q35-linux-hostpci-template.conf │ │ ├── q35-linux-hostpci-template.conf.cmd │ │ ├── q35-linux-hostpci-x-pci-overrides.conf │ │ ├── q35-linux-hostpci-x-pci-overrides.conf.cmd │ │ ├── q35-linux-hostpci.conf │ │ ├── q35-linux-hostpci.conf.cmd │ │ ├── q35-no-viommu-with-aw-bits.conf │ │ ├── q35-simple-6.0.conf │ │ ├── q35-simple-6.0.conf.cmd │ │ ├── q35-simple-7.0.conf │ │ ├── q35-simple-7.0.conf.cmd │ │ ├── q35-simple-pinned-6.1.conf │ │ ├── q35-simple-pinned-6.1.conf.cmd │ │ ├── q35-simple.conf │ │ ├── q35-simple.conf.cmd │ │ ├── q35-usb13-error.conf │ │ ├── q35-usb2.conf │ │ ├── q35-usb2.conf.cmd │ │ ├── q35-usb3.conf │ │ ├── q35-usb3.conf.cmd │ │ ├── q35-viommu-intel-aw-bits.conf │ │ ├── q35-viommu-intel-aw-bits.conf.cmd │ │ ├── q35-viommu-intel-guest-phys-bits.conf │ │ ├── q35-viommu-intel-guest-phys-bits.conf.cmd │ │ ├── q35-viommu-intel.conf │ │ ├── q35-viommu-intel.conf.cmd │ │ ├── q35-viommu-virtio-aw-bits.conf │ │ ├── q35-viommu-virtio-aw-bits.conf.cmd │ │ ├── q35-viommu-virtio.conf │ │ ├── q35-viommu-virtio.conf.cmd │ │ ├── q35-win10-hostpci.conf │ │ ├── q35-win10-hostpci.conf.cmd │ │ ├── q35-windows-pinning.conf │ │ ├── q35-windows-pinning.conf.cmd │ │ ├── qemu-xhci-7.1.conf │ │ ├── qemu-xhci-7.1.conf.cmd │ │ ├── qemu-xhci-q35-7.1.conf │ │ ├── qemu-xhci-q35-7.1.conf.cmd │ │ ├── qga-fs-freeze-backup-legacy.conf │ │ ├── qga-fs-freeze-backup-legacy.conf.cmd │ │ ├── qga-fs-freeze.conf │ │ ├── qga-fs-freeze.conf.cmd │ │ ├── qga-minimal.conf │ │ ├── qga-minimal.conf.cmd │ │ ├── scsiblk.conf │ │ ├── scsiblk.conf.cmd │ │ ├── scsihw-lsi.conf │ │ ├── scsihw-lsi.conf.cmd │ │ ├── scsihw-lsi53c810.conf │ │ ├── scsihw-lsi53c810.conf.cmd │ │ ├── scsihw-megasas.conf │ │ ├── scsihw-megasas.conf.cmd │ │ ├── scsihw-pvscsi.conf │ │ ├── scsihw-pvscsi.conf.cmd │ │ ├── scsihw-virtio-scsi-single.conf │ │ ├── scsihw-virtio-scsi-single.conf.cmd │ │ ├── seabios_serial.conf │ │ ├── seabios_serial.conf.cmd │ │ ├── sev-es.conf │ │ ├── sev-es.conf.cmd │ │ ├── sev-snp.conf │ │ ├── sev-snp.conf.cmd │ │ ├── sev-std.conf │ │ ├── sev-std.conf.cmd │ │ ├── simple-backingchain.conf │ │ ├── simple-backingchain.conf.cmd │ │ ├── simple-balloon-free-page-reporting.conf │ │ ├── simple-balloon-free-page-reporting.conf.cmd │ │ ├── simple-btrfs.conf │ │ ├── simple-btrfs.conf.cmd │ │ ├── simple-cifs.conf │ │ ├── simple-cifs.conf.cmd │ │ ├── simple-disk-passthrough.conf │ │ ├── simple-disk-passthrough.conf.cmd │ │ ├── simple-lvm.conf │ │ ├── simple-lvm.conf.cmd │ │ ├── simple-lvmthin.conf │ │ ├── simple-lvmthin.conf.cmd │ │ ├── simple-rbd.conf │ │ ├── simple-rbd.conf.cmd │ │ ├── simple-virtio-blk.conf │ │ ├── simple-virtio-blk.conf.cmd │ │ ├── simple-zfs-over-iscsi.conf │ │ ├── simple-zfs-over-iscsi.conf.cmd │ │ ├── simple1-template.conf │ │ ├── simple1-template.conf.cmd │ │ ├── simple1.conf │ │ ├── simple1.conf.cmd │ │ ├── spice-enhancments.conf │ │ ├── spice-enhancments.conf.cmd │ │ ├── spice-linux-4.1.conf │ │ ├── spice-linux-4.1.conf.cmd │ │ ├── spice-usb3.conf │ │ ├── spice-usb3.conf.cmd │ │ ├── spice-win.conf │ │ ├── spice-win.conf.cmd │ │ ├── startdate-l26.conf │ │ ├── startdate-l26.conf.cmd │ │ ├── startdate-win11.conf │ │ ├── startdate-win11.conf.cmd │ │ ├── unsupported-storage-content-type.conf │ │ ├── usb13-error.conf │ │ ├── vnc-clipboard-spice.conf │ │ ├── vnc-clipboard-spice.conf.cmd │ │ ├── vnc-clipboard-std.conf │ │ └── vnc-clipboard-std.conf.cmd │ ├── parse-config-expected/ │ │ ├── cloudinit-snapshot.conf │ │ ├── cloudinit-snapshot.conf.strict.error │ │ ├── duplicate-sections.conf │ │ ├── duplicate-sections.conf.strict.error │ │ ├── unknown-sections.conf │ │ ├── unknown-sections.conf.strict.error │ │ ├── verify-snapshot.conf │ │ └── verify-snapshot.conf.strict.error │ ├── parse-config-input/ │ │ ├── cloudinit-snapshot.conf │ │ ├── duplicate-sections.conf │ │ ├── fleecing-section.conf │ │ ├── locked.conf │ │ ├── plain.conf │ │ ├── regular-vm-efi.conf │ │ ├── sections.conf │ │ ├── snapshots.conf │ │ ├── unknown-sections.conf │ │ └── verify-snapshot.conf │ ├── restore-config-expected/ │ │ ├── 139.conf │ │ ├── 142.conf │ │ ├── 1422.conf │ │ └── 179.conf │ ├── restore-config-input/ │ │ ├── 139.conf │ │ ├── 142.conf │ │ ├── 1422.conf │ │ └── 179.conf │ ├── run_config2command_tests.pl │ ├── run_parse_config_tests.pl │ ├── run_pci_addr_checks.pl │ ├── run_pci_reservation_tests.pl │ ├── run_qemu_img_convert_tests.pl │ ├── run_qemu_migrate_tests.pl │ ├── run_qemu_restore_config_tests.pl │ ├── run_snapshot_tests.pl │ ├── snapshot-expected/ │ │ ├── commit/ │ │ │ └── qemu-server/ │ │ │ ├── 101.conf │ │ │ ├── 102.conf │ │ │ ├── 201.conf │ │ │ ├── 202.conf │ │ │ └── 203.conf │ │ ├── create/ │ │ │ └── qemu-server/ │ │ │ ├── 101.conf │ │ │ ├── 102.conf │ │ │ ├── 103.conf │ │ │ ├── 104.conf │ │ │ ├── 105.conf │ │ │ ├── 106.conf │ │ │ ├── 201.conf │ │ │ ├── 202.conf │ │ │ ├── 203.conf │ │ │ ├── 301.conf │ │ │ ├── 302.conf │ │ │ └── 303.conf │ │ ├── delete/ │ │ │ └── qemu-server/ │ │ │ ├── 101.conf │ │ │ ├── 102.conf │ │ │ ├── 103.conf │ │ │ ├── 104.conf │ │ │ ├── 105.conf │ │ │ ├── 106.conf │ │ │ ├── 201.conf │ │ │ ├── 202.conf │ │ │ ├── 203.conf │ │ │ └── 204.conf │ │ ├── prepare/ │ │ │ └── qemu-server/ │ │ │ ├── 101.conf │ │ │ ├── 102.conf │ │ │ ├── 103.conf │ │ │ ├── 104.conf │ │ │ ├── 200.conf │ │ │ ├── 201.conf │ │ │ ├── 202.conf │ │ │ └── 300.conf │ │ └── rollback/ │ │ └── qemu-server/ │ │ ├── 101.conf │ │ ├── 102.conf │ │ ├── 103.conf │ │ ├── 104.conf │ │ ├── 105.conf │ │ ├── 106.conf │ │ ├── 201.conf │ │ ├── 202.conf │ │ ├── 203.conf │ │ ├── 204.conf │ │ ├── 205.conf │ │ ├── 206.conf │ │ ├── 207.conf │ │ ├── 301.conf │ │ ├── 302.conf │ │ └── 303.conf │ ├── snapshot-input/ │ │ ├── commit/ │ │ │ └── qemu-server/ │ │ │ ├── 101.conf │ │ │ ├── 102.conf │ │ │ ├── 201.conf │ │ │ ├── 202.conf │ │ │ └── 203.conf │ │ ├── create/ │ │ │ └── qemu-server/ │ │ │ ├── 101.conf │ │ │ ├── 102.conf │ │ │ ├── 103.conf │ │ │ ├── 104.conf │ │ │ ├── 105.conf │ │ │ ├── 106.conf │ │ │ ├── 201.conf │ │ │ ├── 202.conf │ │ │ ├── 203.conf │ │ │ ├── 301.conf │ │ │ ├── 302.conf │ │ │ └── 303.conf │ │ ├── delete/ │ │ │ └── qemu-server/ │ │ │ ├── 101.conf │ │ │ ├── 102.conf │ │ │ ├── 103.conf │ │ │ ├── 104.conf │ │ │ ├── 105.conf │ │ │ ├── 106.conf │ │ │ ├── 201.conf │ │ │ ├── 202.conf │ │ │ ├── 203.conf │ │ │ └── 204.conf │ │ ├── prepare/ │ │ │ └── qemu-server/ │ │ │ ├── 101.conf │ │ │ ├── 102.conf │ │ │ ├── 103.conf │ │ │ ├── 104.conf │ │ │ ├── 200.conf │ │ │ ├── 201.conf │ │ │ ├── 202.conf │ │ │ └── 300.conf │ │ └── rollback/ │ │ └── qemu-server/ │ │ ├── 101.conf │ │ ├── 102.conf │ │ ├── 103.conf │ │ ├── 104.conf │ │ ├── 105.conf │ │ ├── 106.conf │ │ ├── 201.conf │ │ ├── 202.conf │ │ ├── 203.conf │ │ ├── 204.conf │ │ ├── 205.conf │ │ ├── 206.conf │ │ ├── 207.conf │ │ ├── 301.conf │ │ ├── 302.conf │ │ └── 303.conf │ ├── snapshot-test.pm │ ├── test.vmdk │ └── test_get_replicatable_volumes.pl └── usr/ ├── Makefile ├── bootsplash.xcf ├── dbus-vmstate ├── modules-load.conf ├── org.qemu.VMState1.conf ├── pve-bridge ├── pve-bridge-hotplug ├── pve-bridgedown ├── pve-dbus-vmstate@.service ├── pve-q35-4.0.cfg ├── pve-q35.cfg └── pve-usb.cfg ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ *.1.gz *.1.pod *.5.gz *.5.pod *.buildinfo *.changes *.deb /.vscode/ /qemu-server-[0-9]*/ build qm.bash-completion sparsecp vmtar ================================================ FILE: Makefile ================================================ include /usr/share/dpkg/default.mk export PACKAGE=qemu-server BUILDDIR ?= $(PACKAGE)-$(DEB_VERSION_UPSTREAM) export DESTDIR ?= GITVERSION:=$(shell git rev-parse HEAD) DEB=$(PACKAGE)_$(DEB_VERSION_UPSTREAM_REVISION)_$(DEB_HOST_ARCH).deb DBG_DEB=$(PACKAGE)-dbgsym_$(DEB_VERSION_UPSTREAM_REVISION)_$(DEB_HOST_ARCH).deb DSC=$(PACKAGE)_$(DEB_VERSION_UPSTREAM_REVISION).dsc DEBS=$(DEB) $(DBG_DEB) all: .PHONY: tidy tidy: git ls-files ':*.p[ml]'| xargs -n4 -P0 proxmox-perltidy cd src; proxmox-perltidy bin/qm bin/qmextract bin/qmrestore usr/pve-bridgedown usr/pve-bridge .PHONY: dinstall dinstall: deb dpkg -i $(DEB) $(BUILDDIR): rm -rf $@ $@.tmp cp -a src $@.tmp cp -a debian $@.tmp/ echo "git clone git://git.proxmox.com/git/qemu-server.git\\ngit checkout $(GITVERSION)" > $@.tmp/debian/SOURCE mv $@.tmp $@ .PHONY: deb deb: $(DEBS) $(DBG_DEB): $(DEB) $(DEB): $(BUILDDIR) cd $(BUILDDIR); dpkg-buildpackage -b -us -uc lintian $(DEBS) .PHONY: dsc dsc: $(DSC) $(DSC): $(BUILDDIR) cd $(BUILDDIR); dpkg-buildpackage -S -us -uc -d lintian $(DSC) sbuild: $(DSC) sbuild $(DSC) .PHONY: upload upload: UPLOAD_DIST ?= $(DEB_DISTRIBUTION) upload: $(DEB) tar cf - $(DEBS) | ssh -X repoman@repo.proxmox.com upload --product pve --dist $(UPLOAD_DIST) --arch $(DEB_HOST_ARCH) .PHONY: clean clean: rm -rf $(PACKAGE)-*/ *.deb *.build *.buildinfo *.changes *.dsc $(PACKAGE)_*.tar.?z .PHONY: distclean distclean: clean ================================================ FILE: debian/changelog ================================================ qemu-server (9.1.9) trixie; urgency=medium * qmp client: set default timeouts for more block-related commands ('block-commit', 'block-stream', 'blockdev-reopen', and others) to avoid the previous five-second fallback failing prematurely while the operation waits for in-flight I/O to drain. * fix #7066: api: allow live snapshot create and remove for a qcow2 TPM state drive on a storage with the 'snapshot-as-volume-chain' option enabled, by routing the block-level QMP commands for the TPM disk to the QEMU storage daemon that holds the FUSE-exported state. Removing the top-most snapshot live remains unsupported, as the FUSE export holds the 'resize' permission that 'block-commit' and 'block-stream' require. -- Proxmox Support Team Sat, 25 Apr 2026 19:46:01 +0200 qemu-server (9.1.8) trixie; urgency=medium * cloud-init: dns: avoid "uninitialized value in split" warning if there is no 'searchdomain' cloud-init setting and also no 'search' setting in the host's resolv.conf. * api: dump cloud-init: mask password for user without VM.Config.Cloudinit privilege. * api: confirm Sys.Console privilege when creating or restoring a VM and actively enabling HA for it. -- Proxmox Support Team Wed, 22 Apr 2026 16:27:25 +0200 qemu-server (9.1.7) trixie; urgency=medium * start vm: clarify the informational BitLocker message to state that protectors must be disabled before enrolling UEFI keys, matching the existing wording in the web UI and documentation. * hotplug pending: avoid the USB-leftover and USB-hotplug-support checks, along with the associated QMP calls, for unrelated config changes that do not touch USB devices. Explicitly clean up a newly-plugged 'xhci' controller in the error path of a failing USB hotplug, now that the implicit cleanup is gone. * migration: only warn about a missing 'host_mtu' setting on a network device when it is actually unexpected, to avoid confusing log noise on legacy machine versions with default MTU. * backup: log 'starting backup via QMP command' between the filesystem freeze and thaw messages, to make it obvious that the ordering of freeze, backup start, and thaw is correct. * backup: vma: prefix the bubbled-up QEMU error on backup start with 'executing QMP command to start backup failed -', to make the failing operation easier to identify. * cpu config: warn when a VM uses a qemu64- or kvm64-derived CPU model without the 'pdpe1gb' flag, as OVMF will then limit the guest to 40 phys-bits, which can prevent booting VMs with large amounts of memory or with GPU passthrough using lots of vRAM. * api: disk move: log the core operations (allocate, copy, rename, remove source) in the task log; clone disk: log the target volume ID after allocating; api: disk reassign: include the target disk key and use 'reassign' in the log message to align with the UI term. * api: vncproxy: align the VM-specific vncproxy endpoint for non-serial displays with the serial-display behavior: listen only on localhost when used through the 'websocket' parameter, and use TLS otherwise. This may affect external VNC clients that connect directly without going through 'vncwebsocket' or that do not support TLS. * api: vnc/termproxy: bind VNC and SPICE tickets to the listen port to harden authentication for interactive console sessions. Also extends to the SPICE proxy ticket used during live migration. * api: vncproxy: always use a freshly generated password when authenticating via the VNC protocol, improving entropy compared to the previous ticket-derived scheme. * api: termproxy: pin the accepted ticket in the spawned termproxy process as an additional defense-in-depth measure. -- Proxmox Support Team Tue, 21 Apr 2026 22:33:39 +0200 qemu-server (9.1.6) trixie; urgency=medium * guest agent: rename 'guest-fsfreeze' setting to 'freeze-fs' for consistency with the existing agent sub-property naming. The old name and 'freeze-fs-on-backup' are kept as aliases. * guest agent: fix clone, import and backup not properly checking whether the agent is actually enabled before attempting to freeze guest file systems. * guest agent: fs thaw: check command result instead of silently ignoring errors. * api: clone: log the new VMID at the start of the task, as it was not recorded in the task log, system log, or inside the task worker UPID before. * external migration: block if HA is configured regardless of HA management state, as migrating an HA-managed VM can confuse the HA stack even when in 'ignored' state. * query cpu flags: fix KVM support detection and TCG model selection for non-x86 hosts, and add support for querying CPU flags on aarch64. -- Proxmox Support Team Thu, 26 Mar 2026 23:43:39 +0100 qemu-server (9.1.5) trixie; urgency=medium * fix #1964: add 'guest-fsfreeze' QEMU guest agent setting to control whether filesystem freeze/thaw commands are issued for snapshots, replications and clones — not just backups. The previous 'freeze-fs-on-backup' setting is deprecated in favor of the new one. * agent: file-read: allow specifying a byte offset and maximum byte count, and allow keeping the base64-encoding of the response content. This enables reading large files block-by-block and avoids inflated payload sizes for binary data. * extend and improve experimental aarch64 VM support. Allow querying CPU models, CPU flags and machine types for a given architecture via the API. Add aarch64 CPU models. Guard x86-specific CPU options like hyperv enlightenments and the 'hidden' flag by architecture. Fix S3/S4 ACPI power state not being disabled for ARM-based virt machines, as ARM has no concept of those. Fix machine version pinning and default architecture detection on non-x86 hosts. * vm start: check efi: extend the UEFI 2023 certificate enrollment check to all VMs with pre-enrolled keys, not just Windows 10 and 11, as common Linux distributions also require the updated certificates for secure boot. * ovmf: also enroll the Microsoft Corporation KEK 2K CA 2023 key exchange key, as not having it can still cause secure boot update failures. * apply pending config changes: efi disk: fix ms-cert marker not being updated in config after certificate enrollment via API, which could leave a stale marker. * vm status: guard memory stat collection with eval to avoid breaking the whole VM status API call or pvestatd collection if it fails. * verify device removal: mention that the device might still be busy in the guest on timeout, which is a common cause especially on Windows with ongoing IO on VirtIO block devices. * qmp: switch from deprecated block-job-complete to job-complete command. * blockdev: introduce dedicated module for snapshot-as-volume-chain handling to resolve cyclic dependency between Blockdev and BlockJob modules. * config: document default balloon behavior and fix 'ballon' typo. -- Proxmox Support Team Fri, 06 Mar 2026 12:48:12 +0100 qemu-server (9.1.4.1) trixie; urgency=medium * vm status: guard memory stat collection with eval to avoid breaking the whole VM status API call or pvestatd collection if it fails. -- Proxmox Support Team Mon, 30 Mar 2026 09:21:37 +0200 qemu-server (9.1.4) trixie; urgency=medium * vga: allow live-migration with clipboard with QEMU 10.1, which gained support for doing so. * fix #6935: vmstatus: use CGroup for host memory usage to avoid performance impact of reading the PSS metrics when there is a significant amount of kernel same-page memory being merged. * migration: vm start: avoid config write when there are left-over VM-state related properties. * config: fix OVMF typo in error message. * cpu config: update CPU models for QEMU 10.1. * VM start: catch outdated Zen 5 CPU micro-code firmware that results in the RDSEED CPU feature not being available for a VM and output a more telling error message. * enroll-efi-keys: do not remove EFI disk when config was modified during operation. * config: apply pending: efi: enroll Microsoft UEFI CA 2023 when setting ms-cert=2023 option to allow updating to the newer CA via API too. * ovmf: also enroll the Windows UEFI CA 2023 key, which is separate from the Microsoft key. * migration: prohibit renaming special cloud-init drive on storage migration. -- Proxmox Support Team Tue, 20 Jan 2026 18:33:11 +0100 qemu-server (9.1.3) trixie; urgency=medium * fix #7092: fix regression with DBUS cleanup when migrating during node shutdown -- Proxmox Support Team Wed, 17 Dec 2025 15:11:20 +0100 qemu-server (9.1.2) trixie; urgency=medium * check efi vars: avoid warning about accessing an undefined value from Perl when no OS type is set. * query machine capabilities: remove left-over debug print. * dbus-vmstate: fix method call on dbus object resolving to wrong instance. * migrate: remove left-over dbus-vmstate instance when migrating without conntrack state. -- Proxmox Support Team Wed, 10 Dec 2025 15:33:03 +0100 qemu-server (9.1.1) trixie; urgency=medium * fix #7044: migration: cap the auto-increased allowed-downtime-limit to the maximum value that QEMU supports. * qmp client: bump timeout for synchronous block-dev snapshot commands to one hour, as they can take quite a bit of time, especially with qcow2 internal snapshots. * qm cleanup: avoid potentially problematic cleanup during rollback. * block-dev: add NBD: fix handling of IPv6 host to unbreak TCP disk live migration for such setups in PVE 9. * create: assume HA state 'started' when live-restoring a VM, to avoid that the HA manager is stopping the VM during the restore due to not knowing better. * create: set, not add, HA resource request-state when restoring a VM that is already a HA resource. -- Proxmox Support Team Sat, 29 Nov 2025 23:26:50 +0100 qemu-server (9.1.0) trixie; urgency=medium * vm start: check efi vars: add missing newline in print. * cpu config: nested-virt: update description of parameter to always recommend a CPU model similar to the host. * snapshot: prohibit live snapshot (remove) of qcow2 TPM drive on storage with snapshot-as-volume-chain for the time being. -- Proxmox Support Team Wed, 19 Nov 2025 13:03:38 +0100 qemu-server (9.0.30) trixie; urgency=medium * api: add 'allow-ksm' to memory options that are allowed to be configured by non-root users. * vm start: ovmf: do not auto-enroll Microsoft UEFI CA 2023 for now, as this triggers an active Bitlocker inside the VM and the admin doing the restart of the VM might not have the Bitlocker recovery code available. Instead print a informational message in the start task log for now. * qm cli: add enroll-efi-keys command to update the efidisk for Windows VMs that do not yet have the newer Microsoft UEFI CA 2023 enrolled. -- Proxmox Support Team Tue, 18 Nov 2025 15:10:24 +0100 qemu-server (9.0.29) trixie; urgency=medium * api: create vm: use ha-manager command to add VM as an HA resource, this fixes adding a VM to HA when using the qm CLI tool directly. * qmrestore CLI tool: allow starting a VM after it was restored successfully. * qmrestore CLI tool: allow adding a VM as an HA resource after it was restored. -- Proxmox Support Team Mon, 17 Nov 2025 22:31:09 +0100 qemu-server (9.0.28) trixie; urgency=medium * fix #6934: vm start: don't apply pending changes when resuming from hibernation. * Intel TDX: add support for configuring a quote-generation-socket to enable a remote attestation flow. * vm start: fix migration regression with Windows by only enrolling EFI certs on cold start. * qm import: use schema variant with 'import-from' disk allocation support. -- Proxmox Support Team Mon, 17 Nov 2025 17:14:55 +0100 qemu-server (9.0.27) trixie; urgency=medium * fix #6985: ovmf: auto-enroll newer Microsoft UEFI CA 2023 for existing Windows 10 and 11, and Windows Server 2016, 2019, 2022 and 2025. * cpu config: introduce vendor-agnostic 'nested-virt' CPU flag. The flag will automatically resolve to the flag required for the current CPU vendor of the host. This flag should be combined with a vendor-specific vCPU type, e.g. like EPYC-v4 or Cooperlake-v2, matching your host CPU vendor, otherwise nested virtualization capabilities can still be limited inside the guest OS, especially on Windows based VMs. * close #5291: support disabling KSM for specific VMs through a config memory option flag. -- Proxmox Support Team Fri, 14 Nov 2025 21:51:59 +0100 qemu-server (9.0.26) trixie; urgency=medium * fix #7014: fix regression starting aarch64 VMs with machine type >= 10.1. * prepare fix for #6613: pass purge param to HA stack when removing VM. * fix #5779: vm start: pass storage hints when activating guest volumes, which, e.g., sets the rxbounce buffer option for Windows VMs on RBD volumes. * migration: offline volumes: drop deprecated special casing for TPM state. * tpm: support non-raw volumes via FUSE exports for swtpm using the qemu-storage-daemon., * fix #4693: drive: allow non-raw image formats for TPM state drive. -- Proxmox Support Team Fri, 14 Nov 2025 01:05:04 +0100 qemu-server (9.0.25) trixie; urgency=medium * pci: add 'driver' option for passthough of devices which cannot use the default 'vfio-pci' driver. * api: fix permission check for guest net device without bridge which was accidentally restricted to root only. * clone disk: fix handling of snapshot-as-volume-chain for EFI disks. * status: rrddata: use fixed pve-vm-9.0 path. * migration: conntrack: avoid crash when dbus-vmstate object cannot be added or not added quickly enough. * adapt secure virtualization code used for AMD SEV to prepare for Intel TDX. * add initial Intel TDX support. * query machine capabilities: only query features of matching CPU vendor. -- Proxmox Support Team Thu, 13 Nov 2025 12:00:45 +0100 qemu-server (9.0.24) trixie; urgency=medium * api: create/store: allow adding newly created VM to be managed as HA resource. * pending config: avoid warning about accessing an uninitialized value when comparing new hotplug configuration with the existing one. * improve performance of helper to query some config properties that is used in a hot-path of the HA stack and can impact the performance of the cluster resource scheduler when there are many VMs managed by HA. * vm status: make sure disk{read, write} adhere to return schema to fix temporary regression for PDM. * config schema: document hugepages option. * fix #6989: cloudinit: enable joliet ISO extension for the generated config disks to improve compatibility with file names mandated by the nocloud ISO image disk format and Windows guests. -- Proxmox Support Team Mon, 10 Nov 2025 17:29:52 +0100 qemu-server (9.0.23) trixie; urgency=medium * fix #6713: snapshot volume chain: fix snapshot after disk move with zeroinit. * api: dbus-vmstate: fix return property name in schema. * vm status: also queue query-proxmox-support QMP commands to avoid stacking timeouts. * api: add missing snapshot info to get-config return schema. * agent: implement fsfreeze helper to better handle lost commands to avoid unnecessarily blocking the QMP socket for a prolonged time. * migration: conntrack: work around systemd issue where scope for VM might become blocked. * fix #6828: remote migration: bump timeout for writing configuration to accommodate volume activation. Allow for up to 2 minutes instead of just 10 seconds. * fix #6882: backup provider api: fix backup with TPM state by correctly generating node name. * backup: fleecing: avoid warning when querying block node size for TPM state. * cfg2cmd: turn off the high-precision event timer (HPET) for Linux VMs running at least kernel 2.6 and machine type >= 10.1, which will avoid slightly increased CPU usage with newer QEMU versions. Further, Linux kernel since v2.6.26+ (from July 2008!) can use the kvm-clock timer. -- Proxmox Support Team Fri, 03 Oct 2025 22:12:22 +0200 qemu-server (9.0.22) trixie; urgency=medium * partially fix #6805: - api: modify vm config: privilege checks for VM-state-related properties. - api: clone: properly remove all snapshot-related info. * api: create/update: disallow setting 'running-nets-host-mtu' via API. * resume from suspended: properly handle 'nets-host-mtu'. * vm command line: handle 'nets-host-mtu' property in snapshot. * vm start: remove left-over VM-state-related properties. * api schema: fix broken line continuation in the description for three properties. -- Proxmox Support Team Wed, 17 Sep 2025 18:37:06 +0200 qemu-server (9.0.21) trixie; urgency=medium * fix #6608: expose vIOMMU driver address space bit width 'aw-bits' option. * migration: tell users to upgrade the target node if nets-host-mtu is not supported by the target qemu-server package version. -- Proxmox Support Team Wed, 10 Sep 2025 15:17:11 +0200 qemu-server (9.0.20) trixie; urgency=medium * api: rrd: add missing ds parameter for generating PNG graph. * virtio-net: fix migration between default/non-default MTUs starting with machine version 10.0+pve1. * api: vm start: introduce nets-host-mtu parameter for migration compat. * snapshot: introduce running-nets-host-mtu property. -- Proxmox Support Team Thu, 04 Sep 2025 19:49:27 +0200 qemu-server (9.0.19) trixie; urgency=medium * fix generating block-dev option on VM live import. * fix #6680: do not use a top throttle node when using scsi-block, like for passthrough of SCSI devices and 10+ machine version, as passed through disks do not support bandwidth limitations. -- Proxmox Support Team Tue, 26 Aug 2025 09:35:28 +0200 qemu-server (9.0.18) trixie; urgency=medium * close #6378: expose guest-phys-bits CPU option to cope with using the host CPU type on CPU models where the CPU address width and the IOMMU address width are different. On such systems the guest-phys-bits option needs to be set to properly support huge pages and/or VFIO. * fix #6675: template backup: fix regression with IDE/SATA and blockdev. * fix #6680: avoid setting nonexistent 'write-cache' and 'drive_id' options for passed-through disks using SCSI with machine version >= 10.0. * dbus-vmstate: fix installation path for script. -- Proxmox Support Team Thu, 14 Aug 2025 12:33:11 +0200 qemu-server (9.0.17) trixie; urgency=medium * fix #6648: api: available machine versions: ensure array of versions is correctly ordered numerically. * prohibit using snapshot-as-volume-chain qcow2 images with VMs that use a pre-10.0 machine version. * blockdev: attach/detach: silence errors for QMP commands for which failure may be expected. * blockdev: delete/replace: re-use common detach helper to avoid false-positive errors in the syslog for devices that QEMU already auto-removes itself. -- Proxmox Support Team Tue, 12 Aug 2025 15:10:38 +0200 qemu-server (9.0.16) trixie; urgency=medium * vmstate: always quiesce warnings on VM stop cleanup as at this point, the dbus-vmstate helper is not expected to be running anymore, stopping it here is just done out of precaution to ensure it will be cleaned up in any case. -- Proxmox Support Team Tue, 05 Aug 2025 12:21:02 +0200 qemu-server (9.0.15) trixie; urgency=medium * vm state: improve cleaning up dbus-vmstate and avoid spurious warning if guest gets shutdown from the inside. -- Proxmox Support Team Mon, 04 Aug 2025 16:02:28 +0200 qemu-server (9.0.14) trixie; urgency=medium * api: migration preconditions: fix default value for "comigrated" HA resources, * api: migration checks: rename return prop to dependent-ha-resources and improve description. -- Proxmox Support Team Fri, 01 Aug 2025 18:37:51 +0200 qemu-server (9.0.13) trixie; urgency=medium * fix #6580: blockdev: commit: re-open target format node as writable if necessary. * blockdev: delete: delete format block node first. -- Proxmox Support Team Thu, 31 Jul 2025 14:26:03 +0200 qemu-server (9.0.12) trixie; urgency=medium * api: migration preconditions: add checks for ha resource affinity rules -- Proxmox Support Team Thu, 31 Jul 2025 11:33:45 +0200 qemu-server (9.0.11) trixie; urgency=medium * metrics: add pressure stall information to status. * vm status: add memhost as accumulated PSS for all processes in a VMs cgroup for complete host view of VM memory consumption. * vm status: switch 'mem' stat to PSS of VM cgroup if there is no balloon info. * RRD metrics: use new pve-storage-9.0 format RRD file location, if it exists. -- Proxmox Support Team Thu, 31 Jul 2025 04:50:09 +0200 qemu-server (9.0.10) trixie; urgency=medium * delete snapshot: exit delete early if we cannot find a snapshot * api: add module exposing node migration capabilities * fix #5180: dbus-vmstate: add daemon for QEMUs dbus-vmstate interface and integrate helper for live-migrating connection-tracking info to better ensure currently active connections can continue to work seamlessly if the network environment supports it. * migrate: flush old VM conntrack entries after successful migration. * image convert (clone): avoid combining target image options and zeroinit filter. * image convert (clone): make using zeroinit with target-image-opts work. -- Proxmox Support Team Wed, 30 Jul 2025 23:06:51 +0200 qemu-server (9.0.9) trixie; urgency=medium * blockdev: ovmf: use correct cache mode for EFI disk to avoid slowness when booting on certain storages like kRBD. * migration: fix handling of status reported by QEMU to cover all relevant cases with QEMU binary version 10.0 and avoid aborted migrations. * disk mirror: avoid left-over block node attached to QEMU in error scenario. Not detaching would also make subsequent live-migration fail after an earlier error. * snapshot-as-volume-chain: ensure backing file references are kept relative upon snapshot deletion. This ensures the backing chain stays intact should the volumes be moved to a different path. -- Proxmox Support Team Tue, 29 Jul 2025 16:53:45 +0200 qemu-server (9.0.8) trixie; urgency=medium * fix #6562: fix online-removal of an intermediate snapshot for directory-based storage types and snapshot-as-volume-chain. * fix discard for blockdev backend, used by VMs with machine version 10 or newer. * fix #6543: use 'discard-no-unref' qcow2 option when creating snapshots as volume backing-chain. * device unplug: bump timeout for removing virtio scsi controller from 5 seconds to 15 seconds. -- Proxmox Support Team Fri, 25 Jul 2025 16:14:22 +0200 qemu-server (9.0.7) trixie; urgency=medium * drive device: fix regression with missing SCSI device ID and thus missing '/dev/disk/by-id' paths inside the VM for Linux based guests. * fix #6556: vm status: fix querying block stats. -- Proxmox Support Team Wed, 23 Jul 2025 15:41:34 +0200 qemu-server (9.0.6) trixie; urgency=medium * use storage layer's helper to query the allowed formats and the best possible default format for a specific storage config entry. * blockdev: also set read-only flag on top throttle node, fixing backup of VM templates with the recently adopted blockdev interface. -- Proxmox Support Team Tue, 22 Jul 2025 18:13:51 +0200 qemu-server (9.0.5) trixie; urgency=medium * ovmf: use proper drive properties for temporary efivars drive to avoid an error from get_drive_id about no interface being found. -- Proxmox Support Team Mon, 21 Jul 2025 21:37:10 +0200 qemu-server (9.0.4) trixie; urgency=medium * refuse storage-migration for guests with volume-chain-snapshots for now. -- Proxmox Support Team Fri, 18 Jul 2025 14:09:34 +0200 qemu-server (9.0.3) trixie; urgency=medium * net: default to the bridge MTU for the MTU of virtio network devices * api: agent: use more specific guest agent privileges * api: monitor: improve permission handling * api: monitor: require Sys.Audit or Sys.Modify privilege -- Proxmox Support Team Thu, 17 Jul 2025 22:09:41 +0200 qemu-server (9.0.2) trixie; urgency=medium * fix #6400: pci: allow other pci domains than 0000 for NVIDIA vGPUs. * fix #6466: aarch64: pci: properly print higher-index PCI bridge addresses. * add initial support for storage managed external snapshots. -- Proxmox Support Team Thu, 17 Jul 2025 01:20:05 +0200 qemu-server (9.0.1) trixie; urgency=medium * cfg2cmd: require at least QEMU binary version 6.0 * drive: remove geometry options gone since QEMU 3.1 * vm start: assert that migration type is set for 'tcp' migration * various code refactoring and cleanup, introducing: BlockDev module BlockJob module Network module OVMF module QemuImage module QemuMigrate::Helpers module RunState module StateFile module * vm start/commandline: also clean up pci reservation when config_to_command() fails * vm start/commandline: activate volumes before config_to_command() * qm: showcmd: never reserve PCI devices * fix #5985: qmp client: increase timeout for {device, netdev, object}_{add, del} commands * qmp client: add default timeouts for more blockdev commands * switch to blockdev and block-mirror with machine version >= 10.0 * blockdev: add workaround for issue #3229 * assume that SDN is available * schema: remove unused pve-qm-ipconfig standard option * move find_vmstate_storage() helper to QemuConfig module * adopt perltidy also for executables without perl extension * backup: use blockdev for fleecing images * backup: use blockdev for TPM state file * clone disk: skip check for aio=default (io_uring) compatibility starting with machine version 10.0 * test: migration: update running machine to 10.0 * partially fix #3227: ensure that target image for mirror has the same size for EFI disks * blockdev: pass along machine version to storage layer * net: use pve-firewall helper for deciding whether to create fw bridges -- Proxmox Support Team Thu, 03 Jul 2025 13:40:14 +0200 qemu-server (9.0.0) trixie; urgency=medium * re-build for Debian 13 Trixie based Proxmox VE 9 releases. * move module-load config from /etc to /usr, as a file with the same name in the /etc/modules-load.d/ directory takes precedence over a file in the /usr/lib/modules-load.d/ directory, an admin can still overrides this in a easy manner. * drop duplicated network hook script from /var, they have been migrated to /usr/libexec since a while now, and as all users must upgrade to latest PVE 8.4 before being able to upgrade to PVE 9 they are simply not required anymore. -- Proxmox Support Team Tue, 17 Jun 2025 15:27:09 +0200 qemu-server (8.3.13) bookworm; urgency=medium * fix #6317: backup: properly cleanup fleecing images after the backup of a stopped VM. * disk rescan: avoid adding fleecing images as unused disks. * tests: add config-to-command tests for disk pass-through with RBD, kRBD and ZFS over iSCSI storages. * migrate: avoid warning about uninitialized value if transferred memory did not change between iterations. * tests: add qemu-img convert tests for disks on RBD and BTRFS storages with snapshots. * virtiofs: prevent issue with Windows OS and too many files. The number of open file handles by the virtiofsd process would increase even if the guest opened files sequentially. Use the 'inode-file-handles=prefer' commandline option to avoid this. * qmeventd: auto-format code base using clang-format. * tests: add config-to-command tests for aio and cache settings on various storages. -- Proxmox Support Team Mon, 16 Jun 2025 10:04:17 +0200 qemu-server (8.3.12) bookworm; urgency=medium * virtiofs: drop unsafe and problematic writeback option completely fow now. -- Proxmox Support Team Tue, 08 Apr 2025 17:30:49 +0200 qemu-server (8.3.11) bookworm; urgency=medium * handle various deprecations of options in upstream QEMU 9.2. * parse config: skip unknown sections and warn about their presence. * parse config: warn about duplicate sections. * vzdump: skip all special sections inside guest config. * fix #5440: vzdump: better cleanup fleecing images after hard errors using a new "fleecing" special config section where the allocated fleecing images get recorded. This record will be used to clean them up on any error. * migration: attempt to clean up potential left-over fleecing images. * vm-network-scripts: move scripts to '/usr/libexec', keep old locations for now to avoid a race condition on package upgrade. * qmeventd: go back to extracting VMID from command line instead of cgroup file to address sporadic errors when trying to query the VMID from an exited process. * fix #1027: add support for shared mounts using Virtiofs. * block live-migration, snapshot (with RAM) and hibernation if any Virtiofs device is configured. -- Proxmox Support Team Mon, 07 Apr 2025 23:35:50 +0200 qemu-server (8.3.10) bookworm; urgency=medium * drive: remove ancient and for a long time unsupported 'cow' from formats. * confidential virtualization: add AMD SEV-SNP support. * allow non-root users to add a virtion RNG (random number generator) device using /dev/u?random or /dev/hwrng as source. * config: add S3 and S4 power state properties to machine option. * disable S3 and S4 power states for new machine versions, these states are almost never useful for virtual machines and can even cause (hibernation or suspend related) problems. * api: qemu machine capabilities: return custom pveX versions and include core changes for them in a new property. * backup: allow adding fleecing images also for EFI and TPM * backup: keep track of block-node size for fleecing to avoid problems with, e.g., efidisks on some storage types. * backup: implement backup and restore operations for external provider storage plugins. -- Proxmox Support Team Sun, 06 Apr 2025 21:39:19 +0200 qemu-server (8.3.9) bookworm; urgency=medium * fix #5284: api: move-disk, clone: assert content type support for target storage to avoid situations where one can clone or move a disk to a storage where it then cannot be used anymore, blocking VM start. * disallow deleting the tpmstate and efidisk from configuration while the VM is running to avoid an inconsistent state. * api: termproxy/vncproxy: fix 'use of uninitialized value' warning when checking vga type * migration: extend clean-up of resources after successful migration, like freeing up PCI mdev usage again. * migration blockers: allow mapped devices for offline migration as long as the mapping is also available on the target node. * pci passthrough: enable live-migration for PCI passthrough device that are handled by a resource mapping that declared support for live-migration. * api: migration preconditions: return detailed info for mapped-resources and deprecated previously returned flat list, it will be removed in a future major release. * api: migrate preconditions: include not available mapped resources also for running VMs to improve UX. * migration: show amount of traffic for transferring vfio state in progress log. * migration: report short summary for transfer traffic stats once migration finished. -- Proxmox Support Team Thu, 03 Apr 2025 17:43:04 +0200 qemu-server (8.3.8) bookworm; urgency=medium * resume: error out if VM is a template * backup: also restore VM power state for early failures * config: add special class that prevents writing the configuration * fix #6007: template backup: use minimized configuration for handling the full vm start * vzdump: align behavior for vma backup with PBS backup for templates * config: make attempts at writing out NoWrite configs fatal -- Proxmox Support Team Tue, 18 Feb 2025 14:59:25 +0100 qemu-server (8.3.7) bookworm; urgency=medium * api: fix ova live import of disks by using correct format for source image * qmp helpers: use the HMP interface as a stopgap to deal with QEMU 9.2's stricter requirements for the serialized format sent over the QMP. -- Proxmox Support Team Tue, 04 Feb 2025 17:12:25 +0100 qemu-server (8.3.6) bookworm; urgency=medium * api: do not set an empty machine option when pinning -- Proxmox Support Team Mon, 20 Jan 2025 14:45:58 +0100 qemu-server (8.3.5) bookworm; urgency=medium * unify CDROM medium hotplug and startup code -- Proxmox Support Team Mon, 20 Jan 2025 11:21:50 +0100 qemu-server (8.3.4) bookworm; urgency=medium * vm shutdown: check if QEMU Guest Agent (QGA) is actually running before relying on it for handling the shutdown. This avoids waiting for the timeout just to then fail the task. * api: VM status: document 'cpu' and 'mem' return types * api: create disks: also convert the 'tpmstate' and 'efidisk0' volumes to base image for VM templates * machine: fallback to QEMU version from the VM creation time for windows VMs that get started with QEMU version 9.1 or later, if there is no explicit version set. * api: update vm config: pin machine version when switching to Windows os type, just like we do since a long time for newly created VMs with a Windows OS type. * machine: log informational line when pinning machine version for Windows guest -- Proxmox Support Team Fri, 17 Jan 2025 19:32:17 +0100 qemu-server (8.3.3) bookworm; urgency=medium * fix #5980: import disk: fix spurious warning if no target disk is defined * api: clone: always do a full clone of tpmstate volumes * swtpm: check that format of tpmstate volume is raw as swtpm currently doesn't support anything else anyway. * create disk: disallow adding existing non-raw volumes as tpmstate0, like as for new volumes. -- Proxmox Support Team Sun, 15 Dec 2024 14:26:32 +0100 qemu-server (8.3.2) bookworm; urgency=medium * use image format from storage layer for PVE-managed volumes -- Proxmox Support Team Mon, 09 Dec 2024 10:08:14 +0100 qemu-server (8.3.1) bookworm; urgency=medium * api: clone vm: make error for unsupported volumes appear in a deterministic order and include the ID of the problematic volume. * fix #3588: helper: consider NIC count for config-specific timeout -- Proxmox Support Team Thu, 05 Dec 2024 12:38:12 +0100 qemu-server (8.3.0) bookworm; urgency=medium * api: import working storage: improve error message * migration: drop outdated check that was introduced for PVE 7.2 to PVE 7.3 to guard against (relatively low) fallout of the newly introduced cloudinit section in the VM config. This check can sometimes trigger as false-positive. -- Proxmox Support Team Wed, 20 Nov 2024 12:12:00 +0100 qemu-server (8.2.7) bookworm; urgency=medium * backup: cleanup: check if VM is running before issuing QMP commands * restore: die early when there is no size for a device * add tiny C program to query some hardware capabilities from CPUID * backup: prepare: cancel previous job if still running, which can happen after a hard failure, e.g. if the vzdump task was canceled. * parse config: allow config keys with minus sign * import disk: add 'target-disk' option to add imported volume to disk * import disk: convert imported volume disks to base images for templates * fix #5301: convert added volume disks to base image for templates * use OVF module from the pve-storage package, it was moved over there * api: create: implement extracting disks when needed for import-from * api: create: add 'import-working-storage' parameter for overriding the staging directory used to extract to-be-imported disks. * api: check untrusted image files for the new import content type for proactive hardening. * config: add initial AMD SEV support * migration: add amd-sev to non-migratable resources blockers -- Proxmox Support Team Mon, 18 Nov 2024 21:48:24 +0100 qemu-server (8.2.6) bookworm; urgency=medium * vm start: add syslog info with which PID a VM process was started * pci: avoid the hard requirement for a passthrough device to be reset on start to fix a recent regression from improving error handling from the pve-common package. -- Proxmox Support Team Mon, 11 Nov 2024 20:38:25 +0100 qemu-server (8.2.5) bookworm; urgency=medium * remote migration: fix online migration via API clients * fix #5714: fix calloc parameter ordering * fix typos in user-visible strings * api: status: add some missing description for status return properties * pci: device selection: don't reserve PCI IDs when VM is already running to avoid false-positive if, e.g., one is checking the generated QEMU command using `qm showcmd`. * pci: mdev: adapt to NVIDIA's modern interface with kernel >= 6.8 -- Proxmox Support Team Thu, 24 Oct 2024 18:55:37 +0200 qemu-server (8.2.4) bookworm; urgency=medium * fix #5528: ensure cgroup limits like CPU weight are set correctly and in the same way for both cases, the cold-start and when changing while the VM is running * fix #5619: honor link-down setting when hot-plugging NIC * resume: bump timeout for query-status, as activating the block drives in QEMU on resume can take a bit of time during which the QEMU process might not respond to QMP commands yet. * fix #4493: cloud-init: fix generated Windows config * drive mirror: prevent wrongly logging success when completion fails for some other reasons than for which it's safe to retry on. * migration: avoid crash with heavy IO on migrating a disk backed by local storage, if the VM is running QEMU 8.2 or newer. -- Proxmox Support Team Tue, 30 Jul 2024 21:36:25 +0200 qemu-server (8.2.3) bookworm; urgency=medium * config: net: avoid duplicate ipam entries on nic update * fix #5574: api: fix permission check for 'spice' usb port -- Proxmox Support Team Mon, 22 Jul 2024 19:42:15 +0200 qemu-server (8.2.2) bookworm; urgency=medium * schema: mention that migrate_downtime parameter will be auto-increased if migration cannot converge otherwise. * remote migration: enable schema validation * suspend: also cleanup in error scenario when QMP command 'savevm-end' fails rather than leaving orphaned volume around and VM configuration locked. * human monitor commands: increase timeout to sensible values. In particular to fix #5440 where detaching a fleecing image would fail, because of the low timeout. * vma restore: increase timeout for reading header and improve error messages. * fix #5562: improve TPM version handling by disallowing change in the configuration after creation of the state file (the actual state file cannot change version either). Also avoid warning about undefined value when version is not explicitly set and fix schema to mention the actual default for the version, being 'v1.2'. * fix #3352: minimize config when starting templates for PBS backup. A template might reside on a host that does not have the necessary resources or devices to start the VM with all settings. * block jobs: improve error handling and messages for drive mirror and live-restore/live-import. * fix #5572: avoid warning about uninitialized value when cloning cloud-init disk. * CLI autocomplete: also list backup archives from PBS storages and without compressor extension. * remove outdated QEMU version checks working around missing features in QEMU binaries 4.2 and 4.0.1. * vga: mention that type 'cirrus' is recommended against. -- Proxmox Support Team Fri, 12 Jul 2024 16:09:25 +0200 qemu-server (8.2.1) bookworm; urgency=medium * cpu config: fix get_cpu_bitness always reverting to default cpu type -- Proxmox Support Team Wed, 24 Apr 2024 11:49:03 +0200 qemu-server (8.2.0) bookworm; urgency=medium * qmeventd: also treat 'prelaunch' and 'suspended' states as active to avoid issues when backing up VMs that currently are in those states. * OS type: add Windows Server 2025 as supported, map it to the same virtual hardware profile as the Windows 11 one. -- Proxmox Support Team Tue, 23 Apr 2024 17:09:20 +0200 qemu-server (8.1.4) bookworm; urgency=medium * api: create vm: add missing import for serializing machine type to fix a regression of version 8.1.2. -- Proxmox Support Team Sat, 20 Apr 2024 12:28:35 +0200 qemu-server (8.1.3) bookworm; urgency=medium * firewall: add handling for new nftables based firewall implementation, which currently is a opt-in drop-in replacement for the older iptables- based one. -- Proxmox Support Team Fri, 19 Apr 2024 20:23:39 +0200 qemu-server (8.1.2) bookworm; urgency=medium * fix #4136: backup: implement fleecing option for improved guest stability when the backup target is slow * fix #4474: stop VM: add 'overrule-shutdown' parameter to prevent currently running shutdown tasks from blocking the stop task * fix #1905: disk move: allow moving unused disks * fix #3784: add vIOMMU parameter to support passthrough of PCI devices to nested virtual machines * fix #5363: cloudinit: fix regression to make creation of scsi cloudinit disks possible again -- Proxmox Support Team Fri, 19 Apr 2024 16:09:18 +0200 qemu-server (8.1.1) bookworm; urgency=medium * config: pending network: avoid undef-warning on old/new comparison * support live-import for 'import-from' disk options on create * qm: add 'import' command for importing a VM through a volumeid from a storage that provides the new 'import' content-type. * disk import: warn when fallback is used instead of requested format * cpu config: die on hotplug of non x86_64 CPUs -- Proxmox Support Team Thu, 14 Mar 2024 14:04:34 +0100 qemu-server (8.1.0) bookworm; urgency=medium * migration: do not allow live-migration with VNC clipboard, it's not yet supported by QEMU's vdagent device, which gets used for this feature. * cpu config: add QEMU 8.1 cpu models * fix #4501: TCP migration: start vm: move port reservation and usage closer together * fix #2258: select correct device when removing drive snapshot via QEMU, as the device where the disk is currently attached needs to be targeted, not the one where the disk was attached at the time the snapshot was taken * fix #4957: allow one to set the vendor and product information for SCSI-Disks explicitly * migration: remember original volume names from the source so that they can get deactivated even if the volume name has to change due to being already in use on the target storage. * fix #4085: properly activate the storage(s) of custom cloud-init snippets * fix #1734: clone VM: if deactivation of source volume fails demote error to warning. Deactivation can fail if the source is still, or again, in use, e.g., due to parallel clones of the same template. * mediated device pass-through: avoid race condition for cleaning up on VM reboot * prevent starting a 32-bit VM using a 64-bit OVMF BIOS * QMP client: increase default timeout for drive-mirror to 10 minutes like for other block operations. -- Proxmox Support Team Fri, 08 Mar 2024 15:00:25 +0100 qemu-server (8.0.10) bookworm; urgency=medium * sdn: pass vmid and hostname to allow requesting a new mapping -- Proxmox Support Team Wed, 22 Nov 2023 14:12:46 +0100 qemu-server (8.0.9) bookworm; urgency=medium * add clipboard option to to vga config entry * api: add clipboard variable to return at status/current * recommend libpve-network-perl for SDN support * initial support for dhcp ip allocation in dhcp-enabled SDN zones -- Proxmox Support Team Tue, 21 Nov 2023 15:40:27 +0100 qemu-server (8.0.8) bookworm; urgency=medium * fix #2816: restore: remove timeout when allocating disks * start: increase maximal timeout if using PCI passthrough * fix #4522: api: vncproxy: also set environment variable for the ticket if the websocket option is not set * backup, migrate: fix races with suspended VMs that can wake up * cpu hotplug: cannot change feature online, so keep these as pending change * nbd-stop: increase timeout to 25s * start: add warning if a deprecated machine version is configured -- Proxmox Support Team Sun, 12 Nov 2023 18:54:37 +0100 qemu-server (8.0.7) bookworm; urgency=medium * fix #4822: vzdump: fix PBS encryption for guests without disks * qmeventd: fix parsing of VMID in presence of legacy cgroup entries * api: check access for already configured bridge when updating vNIC * fix #4620: make 'ide1' and 'ide3' drive keys work for machine type q35 * cloudinit: fix two checks that were mistakenly restricted to root only, one for setting the ciupgrade option and one for updating the cloudinit drive * migration: improve format hint when allocating live-migrated disk on the target to make e.g. remote-migration with qcow2 and LVM-thin target work * net: fix setting value for tx_queue_size * fix #3963: allow backup of template VM with immutable TPM drive -- Proxmox Support Team Mon, 21 Aug 2023 11:30:45 +0200 qemu-server (8.0.6) bookworm; urgency=medium * cloudinit: restore previous default for package upgrades * migration: only migrate disks used by the guest, not also those that are owned by them (VMID in name) but not referenced in the config * migration: fail when aliased volume are detected, as referencing the same volume multiple times can lead to unexpected behavior in a migration. * migration: fix issue with qcow2 cloudinit disk live migration -- Proxmox Support Team Wed, 21 Jun 2023 13:03:01 +0200 qemu-server (8.0.5) bookworm; urgency=medium * restore: extend permissions checks * vm start: always reset any failed-state of the VM systemd scope to avoid failing a re-start after, e.g., a OOM kill. -- Proxmox Support Team Wed, 21 Jun 2023 09:17:41 +0200 qemu-server (8.0.4) bookworm; urgency=medium * vCPU config: add new x86-64-v2, x86-64-v3 and x86-64-v4 models * fix #4784: helpers: cope with native versions in manager version check * enable cluster mapped USB devices for guests * enable cluster mapped PCI devices for guests -- Proxmox Support Team Mon, 19 Jun 2023 07:24:11 +0200 qemu-server (8.0.3) bookworm; urgency=medium * qemu: fix permission check call -- Proxmox Support Team Fri, 09 Jun 2023 12:20:40 +0200 qemu-server (8.0.2) bookworm; urgency=medium * cfg2cmd: use actual backend names instead of removed tty and paraport aliases * cfg2cmd: replace deprecated no-acpi option with acpi=off machine flag * cfg2cmd: replace deprecated no-hpet option with hpet=off machine flag * schema: avoid using deprecated -no-hpet in example for 'args' property, instead pass thate via new machine option * allow setting ipconfigX with VM.Config.Cloudinit * fix #3428: cloudinit: add parameter for upgrade on boot * cloudinit: fix 'pending' api endpoint * fast plug options: add migrate_downtime and migrate_speed for convenience * fix #517: api: allow resizing qcow2 disk with snapshots * fix #2315: api: have resize endpoint spawn a worker task * cloudinit: pass through hostname via fqdn field * qmeventd: extract vmid from cgroup file instead of cmdline * config: implement method to calculate derived properties from a config * api: check bridge access for create, update, clone & restore * qm: remote migration: improve error when storage cannot be found -- Proxmox Support Team Fri, 09 Jun 2023 10:26:19 +0200 qemu-server (8.0.1) bookworm; urgency=medium * fix #4737: qmeventd: gracefully handle interrupted epoll_wait call * handle and warn about VM network interfaces not attached to any bridges * block resize: avoid passing zero size to QMP command * qmrestore: improve description of bwlimit parameter * api: switch agent api call to 'array' type * tests: fix invoking migration tests with make -- Proxmox Support Team Wed, 07 Jun 2023 13:50:09 +0200 qemu-server (8.0.0) bookworm; urgency=medium * never enable 'smm' flag for the 'virt' machine type (doesn't exist) * test: mock calls that can fail in a chroot environment * rebuild for Debian Bookworm based releases -- Proxmox Support Team Fri, 19 May 2023 15:07:45 +0200 qemu-server (7.4-3) bullseye; urgency=medium * backup prepare: fix format detection for disks without storage ID * backup prepare: improve error messages -- Proxmox Support Team Mon, 27 Mar 2023 11:17:16 +0200 qemu-server (7.4-2) bullseye; urgency=medium * avoid list context for volume_size_info calls as otherwise we unnecessarily take a slower code path -- Proxmox Support Team Tue, 21 Mar 2023 16:51:01 +0100 qemu-server (7.4-1) bullseye; urgency=medium * fix #4553: nvidia vgpu: reuse smbios uuid for '-uuid' parameter * pci: workaround nvidia driver issue on mdev cleanup * memory: hotplug: sort by numerical ID rather than slot when unplugging * memory: use the DIMM list info from QEMU for unplug -- Proxmox Support Team Mon, 20 Mar 2023 17:24:45 +0100 qemu-server (7.3-4) bullseye; urgency=medium * fix #4378: standardized error for missing OVMF files * schema: memory: be precise that unit is binary prefix * close #2792: allow online migration with replicated snapshots * schema: OS type: note that the l26 type is also compatible with Linux 6.x * hotplug: disk: mark the 'aio' (async IO) as non-hotpluggable to avoid suggesting that it already changed * fix #4525: clone disk: disallow mirror if it might cause problems with io_uring using the same heuristics as for start-up * start: make not being able to set polling interval for ballooning non-critical * swtpm: enable logging to `/run/qemu-server/$vmid-swtpm.log` * fix #4140: vzdump: transform the previous hardcoded behavior of issuing a fs-freeze and fs-thaw if QGA is enabled into an overridable option named 'fs-freeze-on-backup' * update network dev: MTU is not hot-pluggable, avoid suggesting so * fix #4249: make image clone or conversion respect bandwidth limit -- Proxmox Support Team Thu, 23 Feb 2023 17:12:42 +0100 qemu-server (7.3-3) bullseye; urgency=medium * rollback: ignore auto-start request if VM is already running * memory hot-plug: check correct value for maximal memory check * fix #4435: device list: avoid error for undefined value * fix #4358: ignore any suspended lock when destroying a VM * migration: log error from query-migrate, if any, upon migration failure * cd rom handling: return a clearer error when there is no CD-ROM drive * migration: nbd export: switch away from deprecated QMP command -- Proxmox Support Team Mon, 16 Jan 2023 13:52:30 +0100 qemu-server (7.3-2) bullseye; urgency=medium * fix #4372: improve edge-case for config-loading on VM resume when migrating * ovmf cmd assembly: re-work and re-order arguments assembly -- Proxmox Support Team Fri, 16 Dec 2022 12:54:53 +0100 qemu-server (7.3-1) bullseye; urgency=medium * vm resume: improve loading just recently moved config on nocheck/migrate handling -- Proxmox Support Team Mon, 21 Nov 2022 13:43:59 +0100 qemu-server (7.2-12) bullseye; urgency=medium * config: only save unique tags when updating them via the API * api: create/update vm: fix clamping CPU units function calls -- Proxmox Support Team Mon, 21 Nov 2022 08:36:06 +0100 qemu-server (7.2-11) bullseye; urgency=medium * fdb: only manage FDB entries for Linux bridges, ignore OVS for now -- Proxmox Support Team Sun, 20 Nov 2022 16:30:28 +0100 qemu-server (7.2-10) bullseye; urgency=medium * fix #4321: properly check cloud-init drive permissions, require both VM.Config.CDROM and VM.Config.Cloudinit, and not VM.Config.Disk, for being able to add a cloud init drive in the first place. * api: config update: enforce new tag permission system when setting or removing tags from a guest * parse config: do not validate informative values in cloud init section * fix edge-cases on new cloudinit pending/active recording * mtunnel: add API endpoints * migrate: add foundation for remote (external cluster) migration, add respective endpoints and qm `remote-migrate` CLI command * memory hotplug: make max-memory dynamically calculated from the physicall address bits the VM will use, that is the actual one from the config, if set, the one from the host for CPU type host and 40 bits as fallback for everything else. Calculate the addressable memory (e.g., 40 bits = 1 TiB) and half that for the possible max-memory a VM can use, using the previous hard-coded 4 TiB as overall maximum for backward compat. Admins with inhomogeneous CPUs and thus possible different bit-widths need to take special care themselves to ensure that a VM with memory hot-plug configured can run on other nodes, for example for live-migration. -- Proxmox Support Team Thu, 17 Nov 2022 17:48:03 +0100 qemu-server (7.2-8) bullseye; urgency=medium * fix #4296: virtio-net: enable packed queues for qemu 7.1 * virtio-net: increase defaults rx- and tx-queue-size to 1024 * fix #4296: virtio-net: enable packed queues for QEMU machines using 7.1 or newer * net: increase max queues to 64 * fix #4284: add read-only to non-hotpluggable disk options * delay cloudinit generation in hotplug * record cloud-init changes in the cloudinit section * rework cloudint config pending handling -- Proxmox Support Team Wed, 16 Nov 2022 18:23:39 +0100 qemu-server (7.2-7) bullseye; urgency=medium * api: create/update vm: automatically clamp cpuunit value depending of cgroup version * improve cloud init support and add cloudinit hotplug * vzdump: skip `special:cloudinit` section * fix #3890 - GUI: warn for unlikely iothread config clashes * fix #4228: add `start` parameter to snapshot rollback API so that one can automatocally start the VM after rollback finished. * vm start/stop: cleanup passed-through pci devices in more situations * fix #3593: allow one to configure task set affinity for VMs * fix #4324: USB: use qemu-xhci for machine versions >= 7.1 * usb: increase max USB devices from 5 to 14 for modern 7.1 machine and OS versions (Linux 2.6+ and Windows 8+) * fix #4201: delete cloud-init disk on rollback * net devs: register vNIC MAC-Address manually to FDB on start/resume if bridge has learning disabled -- Proxmox Support Team Sun, 13 Nov 2022 15:46:18 +0100 qemu-server (7.2-6) bullseye; urgency=medium * schema: move 'pve-targetstorage' to pve-common -- Proxmox Support Team Mon, 07 Nov 2022 16:22:50 +0100 qemu-server (7.2-5) bullseye; urgency=medium * qmp client: increase guest fstrim timeout to 10 minutes * fix #3577: prevent suspension for VMs with pci passthrough * cpu config: map deprecated IceLake-Client CPU type to IceLake-Server * snapshot: save VM state: propagate error from QEMU * api: create disks: avoid adding secondary cloud-init drives * vzdump: TPM state: escape drive string * qmp client: increase default fallback timeout to 5s * fix regex matching network devices in qm cleanup so that vNICs with double digit IDs are covered too * qmeventd: rework 'forced_cleanup' handling and set timeout to 60s * qmeventd: send QMP 'quit' command instead of SIGTERM * vzdump: set max-workers QMP option when specified and supported * fix #4099: disable io_uring for virtual disks on CIFS storages for now * qm: move VM-disk related commands to own command group, keep old ones around for backward compatibility -- Proxmox Support Team Mon, 07 Nov 2022 16:15:16 +0100 qemu-server (7.2-4) bullseye; urgency=medium * fix #3754: encode JSON as utf8 for CLI * cpuconfig: add amd epyc milan model * fix #4115: enable option to name QEMU threads after their main purpose * fix #4119: give namespace parameter to live-restore * automatically add 'uuid' parameter when passing through NVIDIA vGPU * vzdump/pbs: die with missing, but configured encryption key * vzdump/pbs: die with missing, but configured master key -- Proxmox Support Team Tue, 16 Aug 2022 13:59:20 +0200 qemu-server (7.2-3) bullseye; urgency=medium * support pbs namespaces -- Proxmox Support Team Thu, 12 May 2022 15:14:39 +0200 qemu-server (7.2-2) bullseye; urgency=medium * api: reassign disk: drop moved disk from boot order * explicitly check some prerequisites for virtio-gl display -- Proxmox Support Team Mon, 02 May 2022 17:26:16 +0200 qemu-server (7.2-1) bullseye; urgency=medium * migrate: add log for guest fstrim and make a failure noticeable * migrate: resume initially running VM when failing after convergence * parse vm config: remove "\s*" from multi-line comment regex * memory: enable balloon free-page-reporting for auto-memory reclaim * enable spice also for virtio-gl and virtio-gpu displays and report so in status API * api: create: allow overriding non-disk options during restore * fix #3861: migrate: fix live migration when cloud-init changes storage -- Proxmox Support Team Thu, 28 Apr 2022 18:35:22 +0200 qemu-server (7.1-5) bullseye; urgency=medium * avoid writing the config if there are no pending changes to apply * fix #3792: cloudinit: use of uninitialized value * pci: allow override of PCI vendor/device ids * drive mirror monitor: warn when suspend/resume/freeze/thaw calls fail * update config: allow setting boot-order and dev in one go * migrate: move tunnel-helpers to pve-guest-common * fix #3683: agent file-write: enable user to encode the content themselves * cpu units: lower minimum for accessing full cgroupv2 range * fix #3845: also clamp cpu units to cgroup dependent valid range on hotplug * clone disk: force raw format for TPM state * fix #3886: QEMU restore: verify storage allows images before writing * fix #3733: bump the timeout used to wait that a for backup started VM is fully stopped (i.e., it's "$vmid.scope vanished) to 20 seconds after the backup has finished to * qmp client: increase timeout for thaw to better accommodate the QGA running in Windows VMs * api: vm start: 'force-cpu' is for internal migration use only, mark as such * device unplug: verify that unplugging SCSI disk completed before continuing with remaining unplug work. * clone disk: remove ancient check for min QEMU version 2.7 * clone disk: pass in efi vars size rather than config * clone disk: allow cloning from an unused or unreferenced disk * parse ovf: untaint path when getting the file's size info * image convert: allow block device as source * fix #3424: api: snapshot delete: wait for active replication * PCI: allow longer pci domains * fix #3957: spell 'occurred' correctly * clone disk: also clone EFI disk from snapshot * api: add endpoint for parsing .ovf files * api: support VM disk import * migrate: keep VM paused after migration if it was before * vga: add virtio-gl display type for VIRGL * restore: cleanup oldconf: also clean up snapshots from kept volumes * restore: also deactivate/destroy cloud-init disk upon error -- Proxmox Support Team Mon, 25 Apr 2022 20:15:59 +0200 qemu-server (7.1-4) bullseye; urgency=medium * migrate: send updated TPM state volume ID to target node on local-storage migration -- Proxmox Support Team Mon, 22 Nov 2021 17:07:13 +0100 qemu-server (7.1-3) bullseye; urgency=medium * replication: do not setup NBD server on VM migrate for the TPM state, QEMU cannot access it directly and we already migrate it via the non-QEMU storage migration anyway. -- Proxmox Support Team Tue, 16 Nov 2021 14:04:45 +0100 qemu-server (7.1-2) bullseye; urgency=medium * cfg2cmd: disable SMM when display=none and SeaBIOS is both used * pci: do not reserve pci-ids for mediated devices, already handled by sysfs anyway * exclude efidisk and tpmstate for boot disk selection heuristic -- Proxmox Support Team Mon, 15 Nov 2021 16:59:23 +0100 qemu-server (7.0-19) bullseye; urgency=medium * rollback: improve interaction with snapshot replication * cli: qm: rename 'move_disk' command to 'move-disk' with an alias for backward compatibility * pi: move-disk: add possibility to reassign a disk to another VM * turn SMM off when SeaBIOS and a serial-display are used in combination to avoid a possible boot loop -- Proxmox Support Team Thu, 11 Nov 2021 12:49:10 +0100 qemu-server (7.0-18) bullseye; urgency=medium * use non SMM ovmf code file for i440fx machines * fix hot-unplugging (removing) a cpulimit on a running VM * vm start: only print tpm-related message if there is an actual instance * vzdump: increase timeout for QMP 'cont' command after backup started * drives: expose readonly flag for SCSI/VirtIO drives as 'ro' property * qemu-agent: allow hotplug of the 'fstrim cloned disk' option * fix #2429: allow to specify cloud-init vendor snippet via 'cicustom' * config: add new meta property with the VM creation time * config: meta: also save the QEMU version installed during creation * cfg2cmd: switch off ACPI hotplug on bridges for q35 VMs with linux as ostype to avoid changes in network interface naming due to systemd's predictable naming scheme -- Proxmox Support Team Thu, 04 Nov 2021 15:29:55 +0100 qemu-server (7.0-17) bullseye; urgency=medium * fix #3258: block vm start when a PCI(e) device is already in use * snapshot: fix TPM state with RBD * swtpm: wait for PID file to appear before continuing with VM start * OS type: add entry for Windows 11/Server 2022 -- Proxmox Support Team Thu, 21 Oct 2021 11:57:09 +0200 qemu-server (7.0-16) bullseye; urgency=medium * ovmf: support secure boot enabled code images * ovmf: support provisioning an EFI vars template with secureboot by default on and distribution + Microsofts secure-boot key pre-enrolled -- Proxmox Support Team Tue, 05 Oct 2021 20:22:18 +0200 qemu-server (7.0-15) bullseye; urgency=medium * api: return task-worker UPID in create template endpoint * api: destroy VM: remove pending volumes as well * fix #3075: add TPM v1.2 and v2.0 support via swtpm~ -- Proxmox Support Team Tue, 05 Oct 2021 07:24:52 +0200 qemu-server (7.0-14) bullseye; urgency=medium * fix #3581: pass size via argument for memory-backend-ram QMP call * fix #3608: improve removal of the underlying SCSI controller when removing last drive on it * migrate: do not suggest that we map shared storages to avoid that subsequent checks could result in false negatives. -- Proxmox Support Team Wed, 22 Sep 2021 09:31:06 +0200 qemu-server (7.0-13) bullseye; urgency=medium * fix bootorder regression with implicit default order -- Proxmox Support Team Thu, 5 Aug 2021 14:03:14 +0200 qemu-server (7.0-12) bullseye; urgency=medium * fix #3371: import ovf: allow the use of dots in the VM name * bootorder: fix double entry on cdrom edit -- Proxmox Support Team Fri, 30 Jul 2021 16:53:44 +0200 qemu-server (7.0-11) bullseye; urgency=medium * nic: support the intel e1000e model * lvm: avoid the use of io_uring for now * live-restore: fail early if target storage doesn't exist * api: always add new CD drives to bootorder * fix #2563: allow live migration with local cloud-init disk -- Proxmox Support Team Fri, 23 Jul 2021 11:08:48 +0200 qemu-server (7.0-10) bullseye; urgency=medium * avoid using io_uring for drives backed by LVM and configured for write-back or write-through cache -- Proxmox Support Team Wed, 07 Jul 2021 14:56:34 +0200 qemu-server (7.0-9) bullseye; urgency=medium * cpu weight: always clamp value to lower maximum for cgroup v2 and fix defaults (v1 -> 1024, v2 -> 100) * api: improve error handling when applying pending config changes -- Proxmox Support Team Wed, 07 Jul 2021 12:02:13 +0200 qemu-server (7.0-7) bullseye; urgency=medium * improve #3329: ensure write-back is used over write-around for EFI disk, as OVMF profits a lot from cached writes due to its frequent read-modify-write operations -- Proxmox Support Team Mon, 05 Jul 2021 20:49:50 +0200 qemu-server (7.0-6) bullseye; urgency=medium * live-restore: preload efidisk before starting VM * For now do not use io_uring for drives backed by Ceph RBD, with KRBD and write-back or write-through cache enabled, as in that case some polling/IO may hang in QEMU 6.0. -- Proxmox Support Team Fri, 02 Jul 2021 09:45:06 +0200 qemu-server (7.0-5) bullseye; urgency=medium * don't default to O_DIRECT (cache=none) on btrfs without nocow * fix #2175: api: update VM: check old drive-config for permissions too to ensure a valid transition when limited to CDROM changes. -- Proxmox Support Team Thu, 24 Jun 2021 18:58:19 +0200 qemu-server (7.0-4) bullseye; urgency=medium * enable io-uring support by default when running QEMU 6.0 or newer * VM start: always check if storages of volumes support correct content-type * use KillMode 'process' for systemd scope to cope with depreacation of KillMode=none * cli, api: handle new warnings task status * improve backup of templates with EFI disks and with SATA and IDE disk controllers in use -- Proxmox Support Team Wed, 23 Jun 2021 12:57:27 +0200 qemu-server (7.0-3) bullseye; urgency=medium * vzdump: add master key support * vzdump: drop legacy fallback logging for dirty-bitmap * vm destroy: do not remove unreferenced disks by default * fix #3329: turn on cache=writeback for efidisks on rbd * avoid setting LUN number for drives when the `pvscsi` controller is used, as that cannot handle multiple LUNs, increase the `scsi-id` instead * config: limit description/comment length to 8 KiB * migrate: enforce that image content type is available and configured on target storage -- Proxmox Support Team Mon, 21 Jun 2021 11:17:52 +0200 qemu-server (7.0-2) bullseye; urgency=medium * api: clone: sort vm disks to keep numbers consistent * api: VM status: make template property optional in return object * add compatibility for QEMU 6.0 * destroy VM: always remove (referenced) VM state volumes * destroy VM: also check if unused volumes are base images * live-restore: log more similar to regular restore, outputting the user the PBS repo/snapshot and target for each drive. -- Proxmox Support Team Fri, 28 May 2021 12:46:36 +0200 qemu-server (7.0-1) pve; urgency=medium * re-build for Proxmox VE 7 / Debian Bullseye -- Proxmox Support Team Thu, 13 May 2021 19:11:18 +0200 qemu-server (6.4-2) pve; urgency=medium * fix #2862: allow sata/ide template backups * migration: improve speed-limits for >1G connections again * fix getting bootdisk size for new bootorder config scheme -- Proxmox Support Team Thu, 29 Apr 2021 16:16:04 +0200 qemu-server (6.4-1) pve; urgency=medium * fix the +pveN versioned machine types when PXE is used * migration: avoid re-scanning all volumes * migration: do not always set default speed limit if none is configured * migration: rework logging to more humand friendly format, avoiding to much output noise * qmrestore: add live-restore option for CLI tool * live-restore: hold 'create' lock during operation * live-restore: don't remove VM on error, to allow an VM user to save any new data before retrying the operation. * fix #3369: auto-start vm after failed stopmode backup -- Proxmox Support Team Fri, 23 Apr 2021 16:26:54 +0200 qemu-server (6.3-11) pve; urgency=medium * enable live-restore tech preview for Proxmox Backup Server hosted backup snapshots. * drive mirror: rework periodic status reporting to be human friendlier * drive mirror: stop logging progress for a disk once it got ready * image convert: use human-readable units in progress report -- Proxmox Support Team Thu, 15 Apr 2021 18:32:06 +0200 qemu-server (6.3-10) pve; urgency=medium * increase timeout for block (disk) resize QMP command * fix #3314: cloudinit: IPv6 requires type 'static6' * fix #2670: cloudinit: enable SLAAC again now that client support is there -- Proxmox Support Team Tue, 30 Mar 2021 18:40:58 +0200 qemu-server (6.3-9) pve; urgency=medium * restore vma: fix applying storage-specific bandwidth limit * snapshot: set migration caps before savevm-start * vzdump: improve error logging for query-proxmox-support to avoid false-positives -- Proxmox Support Team Fri, 26 Mar 2021 09:47:27 +0100 qemu-server (6.3-8) pve; urgency=medium * qm status: sort hash keys on verbose output * improve windows VM version pinning on VM creation -- Proxmox Support Team Fri, 12 Mar 2021 10:01:09 +0100 qemu-server (6.3-7) pve; urgency=medium * vzdump: increase Proxmox Backup Server backup QMP command timeout -- Proxmox Support Team Tue, 09 Mar 2021 08:21:43 +0100 qemu-server (6.3-6) pve; urgency=medium * fix #3324: clone disk: use larger blocksize for EFI disk * fix #3301: status: add currently running machine and QEMU version to full status * api: add endpoint to list all available QEMU machine type and version tuples * always pin virtual machines with Windows as ostype to a fixed QEMU machine version by default. For existing VMs with Windows based OS-type use the 5.1 machine version (or the next available one, for older QEMU versions) to improve stability of the hardware layout from Windows point of view. Linux and other OS types are not as sensitive to those changes, so keep the default to the currently latest available machine versions for non-Windows VMs. * update VM: check for CDROM not just drive permissions when removing a device -- Proxmox Support Team Fri, 05 Mar 2021 21:42:59 +0100 qemu-server (6.3-5) pve; urgency=medium * cloudinit: add opennebula config format * cloudinit: remove pending delete on online regenerate image * snapshot/save-vm: periodically print progress and show information about drives during snapshot * qmeventd: explicitly close() pidfds -- Proxmox Support Team Thu, 11 Feb 2021 18:05:18 +0100 qemu-server (6.3-4) pve; urgency=medium * audio: add the "none" dummy audio backend * fix drive-mirror completion with cloudinit * vm destroy: allow opt-out of purging unreferenced disks * fix #2788: do not resume vms after backup if they were paused before * anchor CPU flag regex to avoid arbitrary flag suffixes -- Proxmox Support Team Thu, 28 Jan 2021 17:21:07 +0100 qemu-server (6.3-3) pve; urgency=medium * api: adapt VM destroy and purge description * clone disk: fix regression in offline clone of efidisk * cloudinit: fix cloning/restoring of cloudinit disks in raw format -- Proxmox Support Team Tue, 15 Dec 2020 16:33:01 +0100 qemu-server (6.3-2) pve; urgency=medium * PBS: use improved method to assemble repository url, fixing issues when using IPv6 or non-default ports -- Proxmox Support Team Thu, 03 Dec 2020 18:06:25 +0100 qemu-server (6.3-1) pve; urgency=medium * deactivate volumes after storage migrate * print query-proxmox-support result in 'full' status * clone disk: avoid errors after disk was moved by QEMU * replace cgroups_write by cgroup change_cpu_shares && change_cpu_quota -- Proxmox Support Team Wed, 25 Nov 2020 14:30:50 +0100 qemu-server (6.2-20) pve; urgency=medium * don't migrate replicated VM whose replication job is marked for removal * ensure qmeventd service is stopped after pve-guests and pve-ha-lrm service on shutdown -- Proxmox Support Team Thu, 12 Nov 2020 17:08:45 +0100 qemu-server (6.2-19) pve; urgency=medium * fix #3113: unbreak drive hotplug * qmeventd: add handling for -no-shutdown QEMU instances, to avoid errors if the guest OS shuts down the VM during a backup job. -- Proxmox Support Team Thu, 05 Nov 2020 13:37:00 +0100 qemu-server (6.2-18) pve; urgency=medium * migrate: tell QEMU to enable dirty-bitmap migration, if supported * partially fix #3056: always try to cancel backups when failed to start job -- Proxmox Support Team Thu, 29 Oct 2020 18:23:13 +0100 qemu-server (6.2-17) pve; urgency=medium * bootorder: don't print empty 'order=' property -- Proxmox Support Team Thu, 22 Oct 2020 16:08:57 +0200 qemu-server (6.2-16) pve; urgency=medium * fix #3010: add 'bootorder' parameter for better control of boot devices * fix VM clone from snapshot with cloudinit disk * fix various possible issues by avoiding conditionally declared variables altogether * PCI passthrough: fix setting VGA to 'none' when marking passed-through device as 'Primary GPU' -- Proxmox Support Team Mon, 19 Oct 2020 15:51:48 +0200 qemu-server (6.2-15) pve; urgency=medium * fix #2570: add 'keephugepages' config property * vzdump: log 'finishing' state for Proxmox Backup Server jobs, to avoid suggesting that the backup is stuck at 100%. This can happen when the validation and mark of pre-existing chunks needs a bit longer. -- Proxmox Support Team Tue, 29 Sep 2020 17:44:28 +0200 qemu-server (6.2-14) pve; urgency=medium * vzdump: allow bandwidth limit also PBS backup * avoid a warning when checking the VMs bios * fix #2862: properly backup (all) VM templates -- Proxmox Support Team Mon, 24 Aug 2020 19:33:54 +0200 qemu-server (6.2-13) pve; urgency=medium * fix use of bandwidth limits with offline storage migrate * allow one to add CPU features with a dot, like "+sse4.2", correctly * vzdump: improve logging output and report dirty bitmap state for each disk * vzdump: display actually uploaded chunks as 'write' speed to conform more closely with the actual network transmission line-speed. * fix #2749: vga: disable the display EDID information for the combination of Windows SeaBIOS and VGA guests to avoid a reduced list of possible screen resolutions. (Windows may cache the list of possible resolutions, uninstalling 'Microsoft Basic Display Adapter' and rebooting may then help) -- Proxmox Support Team Thu, 20 Aug 2020 11:42:47 +0200 qemu-server (6.2-11) pve; urgency=medium * fix #2857: restore: pass keyfile to pbs-restore * fix #2728: die/warn if target is not a replication target when live-migrating -- Proxmox Support Team Fri, 24 Jul 2020 08:13:29 +0200 qemu-server (6.2-10) pve; urgency=medium * pass-through: fix mdev cmdline generation * docs: add man page cpu-models.conf(5) * start: set resume parameter for VM start anytime there is a 'vmstate' in the config, not just when it has the 'suspend' lock -- Proxmox Support Team Mon, 13 Jul 2020 13:37:37 +0200 qemu-server (6.2-9) pve; urgency=medium * support encrypted pbs backups -- Proxmox Support Team Fri, 10 Jul 2020 14:23:46 +0200 qemu-server (6.2-8) pve; urgency=medium * backup: detect PBS features and use only supported * fix #2671: include CPU format in man page again -- Proxmox Support Team Thu, 09 Jul 2020 15:14:36 +0200 qemu-server (6.2-6) pve; urgency=medium * vzdump: fix variable redeclaration warning * make backup log more friendlier to read for humans -- Proxmox Support Team Tue, 07 Jul 2020 19:00:09 +0200 qemu-server (6.2-4) pve; urgency=medium * fix #2787: properly parse vga for vncproxy * vncproxy: allow to request a generated VNC password * fix #2794: allow legacy IGD passthrough * avoid backup command timeout with PBS * fix #2741: add VM.Config.Cloudinit permission * enable dirty-bitmap incremental backups for PBS -- Proxmox Support Team Tue, 30 Jun 2020 11:33:35 +0200 qemu-server (6.2-3) pve; urgency=medium * fix #2748: make order of cloudinit interfaces consistent * fix #2774: add early check for non-managed volumes * allow to force MTU for a VM net-device -- Proxmox Support Team Thu, 04 Jun 2020 11:17:09 +0200 qemu-server (6.2-2) pve; urgency=medium * adapt net-device hotplug to more strict QMP schema of QEMU 5.0 -- Proxmox Support Team Fri, 08 May 2020 13:00:18 +0200 qemu-server (6.2-1) pve; urgency=medium * qmrestore: fix VMA restore from STDIN -- Proxmox Support Team Thu, 07 May 2020 21:51:01 +0200 qemu-server (6.1-20) pve; urgency=medium * cfg2cmd: fix uninitialized value warning on OVMF w/o efidisk * vzdump: fix backup of templates with stdout as output * cfg2cmd: set audiodev parameter only on qemu >= 4.2 * api: allow listing custom and default CPU models -- Proxmox Support Team Wed, 06 May 2020 17:16:56 +0200 qemu-server (6.1-19) pve; urgency=medium * clone: use new config_lock_shared -- Proxmox Support Team Tue, 5 May 2020 11:22:04 +0200 qemu-server (6.1-18) pve; urgency=medium * vzdump: assemble: fix skipping all pending and snapshot config entries * api/destroy: repeat early checks after locking * migrate: skip rescan for efidisk and shared volumes -- Proxmox Support Team Mon, 04 May 2020 17:36:40 +0200 qemu-server (6.1-17) pve; urgency=medium * backup: never try to freeze in stop mode backup * Fix #2124: Add support for zstd -- Proxmox Support Team Mon, 4 May 2020 14:11:50 +0200 qemu-server (6.1-16) pve; urgency=medium * spice audio: improve compatibility with QEMU versions newer than 4.2 * migrate: workaround issues with format switch on storage live migration * fix live migration with replicated unused volumes * importovf: improve compatibility with OVF files without default namespaces * backup restore: use correct storage for format check for cloudinit drives * handle stopping the storage migration NBD server better -- Proxmox Support Team Wed, 29 Apr 2020 16:23:24 +0200 qemu-server (6.1-14) pve; urgency=medium * Use foreach_volume instead of foreach_drive * Use new storage_migrate interface * migrate: update config with changed volume IDs * migrate: allow specifying targetstorage for offline migration * migrate: sync_disks: use allow_rename to avoid collisions on the target storage * migrate: sync_disks: log output of storage_migrate * migrate: also cleanup disks migrated by storage_migrate in case of failure -- Proxmox Support Team Thu, 9 Apr 2020 08:56:44 +0200 qemu-server (6.1-13) pve; urgency=medium * rescan: fix call to foreach_volume * migration: fix downtime limit auto-increase * migrate: allow arbitrary source->target storage maps * migrate: always check storage permissions and content type * Include full KVM/QEMU "-cpu" parameter with live-migration and snapshots/suspend to allow supporting custom CPU models * fix #2318: allow phys-bits CPU setting * allow custom CPU models * config: harmonize bridge pattern to match the same limits of containers * cpu config: add upcoming EPYC-Rome CPU type -- Proxmox Support Team Wed, 08 Apr 2020 17:08:13 +0200 qemu-server (6.1-12) pve; urgency=medium * CPUConfig: fix module load when pmxcfs is unavailable * migrate: fix replication false-positives * migrate: cleanup disk/bitmaps if 'qm start' failed * migration with targetstorage: check if target storage supports images * fix efidisks on storages with minimum sizes bigger than OVMF_VARS.fd * Implement volume-related helpers and use new foreach_volume -- Proxmox Support Team Mon, 30 Mar 2020 10:00:13 +0200 qemu-server (6.1-11) pve; urgency=medium * vzdump: fix regression in backups for specific storage * custom CPU models: add initial parser and verifier -- Proxmox Support Team Thu, 26 Mar 2020 09:00:24 +0100 qemu-server (6.1-10) pve; urgency=medium * version_guard: early out when major/minor version is high enough * drive-mirror: add support for incremental sync * migrate: add replication info to disk overview * migrate: add live-migration of replicated disks -- Proxmox Support Team Wed, 25 Mar 2020 15:16:21 +0100 qemu-server (6.1-8) pve; urgency=medium * cloudinit: make genisoimage quieter, only output errors * Append newline to all QGA commands for compatibility with non standard conforming Apple based guest agent implementation * add experimental support for proxmox backup server * fix #2580: api/delete: drop VM from HA resources if purge is set * improve drive mirror completion over NBD during migration * add secured unix socket support for NBD storage migration * Disable memory hotplug for custom NUMA topologies and die on misaligned memory for hotplug -- Proxmox Support Team Fri, 20 Mar 2020 11:11:31 +0100 qemu-server (6.1-7) pve; urgency=medium * vzdump: always exclude efidisks from backups of machines currently not set to use OVMF (UEFI) * Simplify QEMU version check and require at least 3.0+ * Align size to 1 KiB bytes before doing 'qmp block_resize' * fix #2611: use correct operation when calculating the migration bandwidth limit * fix #2612: allow input-data in guest exec and make command optional * cpu models: add icelake-server and icelake-client * already add models from future QEMU 4.2 release * fix #2264: allow one to add a virtio-rng device for improved entropy bandwidth in a VM * update_disksize: also update disk size if there was no old size at all -- Proxmox Support Team Mon, 09 Mar 2020 19:12:16 +0100 qemu-server (6.1-6) pve; urgency=medium * allow reading snapshot config for users with VM.Audit on a guest * fix #2566: increase scsi limit to 31 * fix #2578: check if $target is provided in clone * update QMP commands to reflect (future) depreacations and changes in QEMU * resize volume: always request new size from storage after resizing -- Proxmox Support Team Mon, 10 Feb 2020 06:40:43 +0100 qemu-server (6.1-5) pve; urgency=medium * Add QEMU CPU flag querying helpers * hotplug_pending: remove redundant write/load config calls * api: vm clone: unlink zombie target VM and firewall config at end of error cleanup * add timeout parameter to vm start API/CLI endpoint * fix #2070: vm_start: for a migrating VM, use current format of disk if possible * hotplug_pending: make 'ssd' option non-hotpluggable, it cannot be changed live on a plugged disk. -- Proxmox Support Team Thu, 30 Jan 2020 10:27:33 +0100 qemu-server (6.1-4) pve; urgency=medium * check if QEMU version is recent enough for requested machine type * suspend to disk: check and enforce more strict permissions * update disk size before local disk migration * hide very long commandline on vm_start/migrate failure * fix #2493: show QEMU errors in migration log * api/restore: do not trigger autostart-after-restored task from locked context -- Proxmox Support Team Mon, 16 Dec 2019 16:03:25 +0100 qemu-server (6.1-3) pve; urgency=medium * create efidisk: poll the real size after volume creation, as some storages need to create bigger volumes as requested, to cope with their internal alignment requirements. * fix #2469: fix qemu-img convert src_format detection, wrongly reverted. * fix #2510: hostpci: always check if specified device exists -- Proxmox Support Team Mon, 09 Dec 2019 11:44:14 +0100 qemu-server (6.1-2) pve; urgency=medium * api: allow one to remove (hibernation) vmstate * vzdump: log QGA FS freeze/thaw tries in task log * skip efidisk0 in hotplug -- Proxmox Support Team Sat, 30 Nov 2019 18:38:36 +0100 qemu-server (6.1-1) pve; urgency=medium * fix #2367: do not allow snapshot with name PENDING * fix #2469: fix qemu-img convert src_format detection * implement PVE Version addition for QEMU machine allowing to introduce new features while keeping migration compatibility more easily -- Proxmox Support Team Tue, 26 Nov 2019 13:06:21 +0100 qemu-server (6.0-17) pve; urgency=medium * PCI(e) pass-through: ensure we fallback to the previous default "0000" domain again. -- Proxmox Support Team Sat, 23 Nov 2019 09:52:09 +0100 qemu-server (6.0-16) pve; urgency=medium * fix #2473: use of uninitialized value -- Proxmox Support Team Fri, 22 Nov 2019 14:18:58 +0100 qemu-server (6.0-15) pve; urgency=medium * api/migration: fix autocomplete for targetstorage * add 'type' to guest agent format, allowing one to choose between VirtIO (default) and ISA * clone: pre-allocate cloud-init disk for destination * SPICE/QXL: tell Linux VMs that they can add up to 4 display when running with qemu 4.1 or newer * add support to tell showcmd helper to assume a specific forced machine version when assembling a command * refactor QemuServer to avoid cyclic module dependencies * fix #2436: pci: do not hardcode pci domain to 0000 * add 'tags' config option for adding meta information to a VM -- Proxmox Support Team Wed, 20 Nov 2019 19:41:01 +0100 qemu-server (6.0-14) pve; urgency=medium * use PVE::DataCenterConfig, use PVE::SSHInfo, use PVE::RRD for RRD data * refactor migration IP retrieval * add missing packages to (build-)dependencies * fix #2457: ga: set-user-password: increase maxLength of password * fix restoring old VM backups made with Promxox VE earlier than 2.3 * improve test mocking -- Proxmox Support Team Mon, 18 Nov 2019 12:12:03 +0100 qemu-server (6.0-13) pve; urgency=medium * fix #2434: extend machine regex to support stable release machine updates * prepare to fix #2408, #2355, #2380: use scsi-hd backend for iSCSI as well * fix deleting pending changes for not yet existing options * improve hugepage memory size detection * avoid a race for VMID reservation when importing an OVF manifest to a new VM * cleanup importidsk CLI command, and say to which exact disk we imported * add simple runtime heuristic for IOThread backup support, to ensure the VM to backup was started with a recent QEMU version. * QMPClient: ensure QMP connection is also closed in certain edge cases -- Proxmox Support Team Wed, 30 Oct 2019 17:43:41 +0100 qemu-server (6.0-12) pve; urgency=medium * fix regression from 6.0-10 with vmstate restore on RBD -- Proxmox Support Team Tue, 22 Oct 2019 16:31:46 +0200 qemu-server (6.0-11) pve; urgency=medium * fix #1071: VMs with IOThread enabled disks can now be backed up * fix regression from 6.0-10 with snapshot restore and statefile * fix regression with from 6.0-10 where forced was always assumed to be true for applying pending changes -- Proxmox Support Team Tue, 22 Oct 2019 12:50:18 +0200 qemu-server (6.0-10) pve; urgency=medium * fix #2344: ignore cloudinit in replication check * fix #1291: add option purge for vm_destroy api call * increase code re-use with pve-container for pending changes in configuration * fix #2412: only do the final configuration destroy after all the VMs resources, and references in other configurations like Firewall or resource pools was successfully cleaned up * fix #2171: ensure that non filesystem based statefiles get activated on VM start * fix #2395: improve QEMU image converter to cope better with pure file based sources and iSCSI source and destinations * fix #2402: allow 1GB hugepages if 2MB is unavailable * qemu 4.0 : add Cascadelake-Server and KnightsMill Intel CPU models * fix #2217: don't copy cloudinit disk on clone -- Proxmox Support Team Fri, 18 Oct 2019 22:04:50 +0200 qemu-server (6.0-9) pve; urgency=medium * fix issue where a SPICE remote viewer was disconnected during live migration * Add VM reboot API/CLI integration, allowing to reboot a VM and applying any pending changes in-between * CPU flags: allow one to add aes flag * fix #2263: die on live migration with local cloudinit disk * fix #2041, #2272: Add Spice enhancements * Add support for more (up to 16) PCI(e) devices * usb: Allow one to make SPICE USB ports USB3 capabile * allow one to use USB3 for SPICE USB ports with VMs started already with QEMU version 4.0.0, as live-migrations were not possible with this previous unsupported setup anyway. Live-snapshots from VMs with a SPICE USB device which was manually set (wasn't possible over Webinterface) to USB3 with a machine version of 4.0.0, need to remove the "usb3" flag again from the snapshot config when restoring it. * rework kvm_user_version cache mechanism * api: deletion: check also pending values for serial/usb * migration api: explicitly clear "online" flag if VM is stopped to avoid issues with storage migrations which are handled different for stopped VMs * abort resize disk if current size could not be determined * fix #2382: delete cloudinit disk before restoring -- Proxmox Support Team Thu, 26 Sep 2019 12:01:58 +0200 qemu-server (6.0-7) pve; urgency=medium * ensure new SPICE audio device works also with 'q35' based VMs -- Proxmox Support Team Wed, 24 Jul 2019 15:13:35 +0200 qemu-server (6.0-6) pve; urgency=medium * Add SPICE audio device support * fix #2275: die on invalid sendkey * Make sometimes problematic 'hv-tlbflush' and 'hv-evmcs' CPU flags optional -- Proxmox Support Team Tue, 23 Jul 2019 18:20:10 +0200 qemu-server (6.0-5) pve; urgency=medium * do not pass Proxmox VE internal startdate 'now' to QEMU, it does not understands it * use new pcie port hardware for 4.0 and newer q35 machine types -- Proxmox Support Team Thu, 11 Jul 2019 19:44:28 +0200 qemu-server (6.0-4) pve; urgency=medium * fix guest shutdown if agent is configured but no timeout was passed * cloudinit: set iso-level in genisoimage call * migrate pre-condition check: add size to volume attributes and handle storage not selected manually in storage config -- Proxmox Support Team Fri, 28 Jun 2019 20:35:09 +0200 qemu-server (6.0-3) pve; urgency=medium * do not add evmcs HyperV enlightment at all for now due to incompatibillity with AMD based hosts -- Proxmox Support Team Tue, 25 Jun 2019 14:33:01 +0200 qemu-server (6.0-2) pve; urgency=medium * add migration precondition api endpoint * wait for possible left over VM scopes to be gone through dbus based helper on VM start * fix #2244: Allow timeout for guest-agent shutdown * fix #2083: Add hv_tlbflush, hv_ipi, hv_evmcs enlightenments -- Proxmox Support Team Mon, 24 Jun 2019 17:46:51 +0200 qemu-server (6.0-1) pve; urgency=medium * SMBIOS: followup: allow now 512 characters for full format string * fix #2190: Base64 encode SMBIOS value strings in order to allow more characters * allow one to add md-clear cpu flag * add qm command for cloudinit config dump * drop vnc x509 param, deprecated in 2.5 removed in 4.0 * Fix #1999: cli: listsnapshot: handle multiple roots and mark orphaned as root * drop references to un-maintained sheepdog plugin * vm_resume: correctly honor $nocheck -- Proxmox Support Team Fri, 14 Jun 2019 20:59:07 +0200 qemu-server (6.0-0+1) pve; urgency=medium * rebuild for Debian Buster / PVE 6.0 -- Proxmox Support Team Wed, 22 May 2019 19:12:34 +0200 qemu-server (5.0-51) unstable; urgency=medium * fix #1811: allow non root user to edit serialX: socket entries * allow non root users to add spice usb port * fix #1829: do not ignore format parameter when creating cloudinit disk volume * fix: #1075: Restore VM template to VM and try to convert to template * fix #2173: use qemu-img to check cloudinit disk existence * cloudinit: use detected format in volume name parsing -- Proxmox Support Team Tue, 30 Apr 2019 14:07:59 +0000 qemu-server (5.0-50) unstable; urgency=medium * fix #2100: ignore cloudinit drive on offline migration * honor bandwidth limits (bwlimit) for migrate, drive-mirror, clone and add to API calls -- Proxmox Support Team Thu, 04 Apr 2019 16:22:10 +0200 qemu-server (5.0-49) unstable; urgency=medium * return config lock in vm status * move 'pve-snapshot-name' to common -- Proxmox Support Team Thu, 21 Mar 2019 12:55:03 +0100 qemu-server (5.0-48) unstable; urgency=medium * cloud-init: allow custom network/user data files via snippets * fix #2120: use hosts initiator name with qemu-img * fix #2131: get correct device when deleting iothreads * config: NIC macaddr: enforce unicast MAC addresses * implement suspend to disk for running VMs over API and CLI -- Proxmox Support Team Tue, 19 Mar 2019 13:25:38 +0100 qemu-server (5.0-47) unstable; urgency=medium * fix #2043: always stop existing systemd scopes on VM start * use nr_hugepages from /proc/cmdline * fix #2101: cloudinit: IPv6 ending in : not parsed as a string * fix #1891: Add zsh command completion for qm and qmrestore * fix #2097: allow one to set and pass the WWN parameter for IDE, SATA and SCSI disks * allow one to add IVSHMEM device to a VM configuration * fix #2114: always set correct link status on VM network adapter hotplug -- Proxmox Support Team Mon, 04 Mar 2019 10:11:00 +0100 qemu-server (5.0-46) unstable; urgency=medium * allow explicit hv-vendor-id * allow explicit set vga with gpu passthrough * fix #1924: add snapshot parameter * allow to add pre- start/stop hook scripts -- Proxmox Support Team Fri, 01 Feb 2019 13:04:19 +0100 qemu-server (5.0-45) unstable; urgency=medium * fix #2003: give qm terminal a terminal over ssh * migrate: fix local disk migration with online VMs -- Proxmox Support Team Mon, 21 Jan 2019 10:42:03 +0100 qemu-server (5.0-44) unstable; urgency=medium * fix #1013 : migrate : sync_disk : --targetstorage with offline disk * add Windows 7 PCIe quirk for adding 'hostpci' devices * fix #2032: check that type is set before assembling a VGA device * add configuration to command regression tests * allow 'none' as VGA option * fix #1267: move args to the end of qemu commandline * clone_disk : cloudinit drive: don't clone snapshot name (snapname) -- Proxmox Support Team Thu, 20 Dec 2018 10:17:47 +0100 qemu-server (5.0-43) unstable; urgency=medium * usb: fix check if machine type is q35 * qm cleanup: exit silently if VM config does not belong to node anymore -- Proxmox Support Team Thu, 29 Nov 2018 12:57:49 +0100 qemu-server (5.0-42) unstable; urgency=medium * add mediated devices support * use improved lspci -- Proxmox Support Team Thu, 22 Nov 2018 11:22:04 +0100 qemu-server (5.0-41) unstable; urgency=medium * fix #1959: add fallback for auto previously set by SLAAC * create_vm: don't add vmgenid for ARM machines by default * use qmeventd to execute qm cleanup * QemuServer: remove PCI sysfs helpers -- Proxmox Support Team Mon, 19 Nov 2018 14:38:42 +0100 qemu-server (5.0-40) unstable; urgency=medium * correctly check display vga type -- Proxmox Support Team Mon, 12 Nov 2018 17:27:53 +0100 qemu-server (5.0-39) unstable; urgency=medium * Add `ssd` property to IDE, SATA, and SCSI drives * fix #1952: make vga memory configurable * fix #1969: increase max unused disks -- Proxmox Support Team Fri, 09 Nov 2018 16:23:26 +0100 qemu-server (5.0-38) unstable; urgency=medium * add second qmp socket when running qemu >= 2.12 for pending event/reboot changes -- Proxmox Support Team Thu, 18 Oct 2018 12:28:49 +0200 qemu-server (5.0-37) unstable; urgency=medium * add virtio to the list of valid gpu types * lower qemu requirement for hv_synic and hv_stimer to 2.12 * prepare for deprecated functionality in qemu 3.0 -- Proxmox Support Team Tue, 16 Oct 2018 14:53:04 +0200 qemu-server (5.0-36) unstable; urgency=medium * fix #1908: add VM Generation ID (vmgenid) device by default on new VMs and allow to set on existing ones * fix version check in qemu_machine_feature_enabled * use qemu's internal blockdev-snapshot functions to avoid some issues with snapshots on qcow2 files -- Proxmox Support Team Mon, 24 Sep 2018 11:14:32 +0200 qemu-server (5.0-35) unstable; urgency=medium * add hv_synic and hv_stimer HyperV Enlightment flags on QEMU 3.0 and later * improve setting QEMU machine type on snapshot and rollback -- Proxmox Support Team Mon, 17 Sep 2018 15:29:18 +0200 qemu-server (5.0-34) unstable; urgency=medium * fix #1865: CloudInit doesn't add IPv6 * allow to add ibpb, ssbd, virt-ssbd, amd-ssbd, amd-no-ssb and pdpe1gb CPU flags * clone: improve message for ignoring pending changes -- Proxmox Support Team Thu, 13 Sep 2018 11:37:24 +0200 qemu-server (5.0-33) unstable; urgency=medium * fix #1242: allow one to setup an automatic QEMU guest agent fstrim command after cloning or moving a disk live * define return properties for vmstatus API call -- Proxmox Support Team Mon, 20 Aug 2018 14:40:46 +0200 qemu-server (5.0-32) unstable; urgency=medium * api/agent: do not dereference parameter hash before passing to agent_cmd method * api/agent: import check_agent_error method -- Proxmox Support Team Mon, 30 Jul 2018 10:55:42 +0200 qemu-server (5.0-31) unstable; urgency=medium * fix #1717: delete snapshot when vm running and drive not attached * fix qemu agent commands: add missing import of 'agent_cmd' method -- Proxmox Support Team Mon, 30 Jul 2018 10:24:27 +0200 qemu-server (5.0-30) unstable; urgency=medium * Fix SPICE multi-monitor mode on VMs using q35 machine type * extend QEMU guest agent API with file read/write, setting VM user password and executing arbitrary commands in VM * add guest command group to qm CLI tool * add 'dryrun' to qm rescan, which only list changes but does not actually saves them * add a workaround for bug #1650: add content type filter to rescan -- Proxmox Support Team Tue, 17 Jul 2018 11:39:01 +0200 qemu-server (5.0-29) unstable; urgency=medium * reserve new VM config with create lock early * api: allow auto vm start after create finished * use 'system_wakeup' to resume VMs suspended by their guest OS -- Proxmox Support Team Fri, 15 Jun 2018 12:11:29 +0200 qemu-server (5.0-28) unstable; urgency=medium * cloud-init: improve handling of network settings for broken versions (e.g., the one currently used by CentOS 7) -- Proxmox Support Team Mon, 11 Jun 2018 12:48:42 +0200 qemu-server (5.0-27) unstable; urgency=medium * qm agent: check if qga service is running * fix #1779: vzdump: ensure guest-fsfreeze-thaw is called on error * fix #1780: change datacenter.conf to datacenter.cfg in description * fix logic of deleting balloon config property * activate volume for cloudinit disk -- Proxmox Support Team Tue, 05 Jun 2018 15:24:14 +0200 qemu-server (5.0-26) unstable; urgency=medium * fix #1749: do not copy pending changes when cloning a vm * make q35 and virtio-scsi-single compatible * fix an issue where cleanup operations run as part of a VM's systemd scope * associate cloud-init options with with the VM.Config.Network permission -- Proxmox Support Team Mon, 14 May 2018 14:04:26 +0200 qemu-server (5.0-25) unstable; urgency=medium * fix #1697: only check machine type for pxe * fix scsi hotplug issue with q35 machines * fix live migration with local disk issue with nbd * switch to new method of passing disk serials to qemu * qemu-img convert: use cache=none for ZFS to not fill up the ARC -- Proxmox Support Team Fri, 13 Apr 2018 15:11:58 +0200 qemu-server (5.0-24) unstable; urgency=medium * use pve-edk2-firmware for supporting OVMF * restore: implement rate limiting * stop passing default '-k' (keyboard) QEMU option from datacenter.cfg * create linked clones of templates by default * remove legacy vm_monitor_command * start: always stop an existing $vmid.scope * implement cloudinit support -- Proxmox Support Team Thu, 22 Mar 2018 09:24:08 +0100 qemu-server (5.0-23) unstable; urgency=medium * fix migrate cache size parameter for qemu 2.11 * typo fixes in migrate task log * depend on pve-qemu-kvm >= 2.9.1-9 * allow virtio-scsi+iothread controller hot-unplug -- Proxmox Support Team Fri, 23 Feb 2018 13:21:08 +0100 qemu-server (5.0-22) unstable; urgency=medium * fix #1664: nbd mirror: remove socat tunnel now that it is not required anymore to avoid a 30s inactivity timeout * fix #1569: add shared flag to disks * fix #1662: make the 'snapshot' disk parameter work again * correct 'snapshot' flag description * CPU types: add EPYC and EPYC-IBPB * forward returned errors from guest-agent commands * add new guest-agent commands and add commands as separate api calls * add agent flag to vm status api call * clone/restore: make the smbios UUID unique if --unique is used * add serial:1 to vm-status when config has a serial device configured -- Proxmox Support Team Wed, 21 Feb 2018 09:56:08 +0100 qemu-server (5.0-21) unstable; urgency=medium * fix a case where with the upcoming introduction of sub-commands the spice ticket could be read from a file rather than received via stdin -- Proxmox Support Team Mon, 22 Jan 2018 15:13:23 +0100 qemu-server (5.0-20) unstable; urgency=medium * added spec-ctrl flag to cpu 'flags' property * added -IBRS cpu type variants * added Skylake-Server cpu type * added --pretty option to 'qm showcmd' * increase start timeout when hugepages are enabled to compensate for long allocation times -- Proxmox Support Team Tue, 16 Jan 2018 14:26:30 +0100 qemu-server (5.0-19) unstable; urgency=medium * add 'flags' property to cpu option to allow passing PCID support to VMs * fix efi disk format detection * use default values from load_defaults() when none is specified in conf -- Proxmox Support Team Tue, 9 Jan 2018 15:49:35 +0100 qemu-server (5.0-18) unstable; urgency=medium * fix #1570: fix template backup with pigz * fix #1471: change keyboard default to undef * fix efi disks in raw mode not being fully writable * qm terminal: add --escape option to change or disable the escape key * enable vncproxy with vncterm for serial ports * add termproxy api call * stop adding the same disk as unused multiple times when they are reachable through multiple storage definitions * check if the guest agent actually runs before a fsfreeze-freeze/thaw * update ostype documentation -- Proxmox Support Team Wed, 13 Dec 2017 14:56:59 +0100 qemu-server (5.0-17) unstable; urgency=medium * correct cpuunits range * check if base volumes are unused before deleting a template -- Proxmox Support Team Tue, 17 Oct 2017 14:58:00 +0200 qemu-server (5.0-16) unstable; urgency=medium * add new qm command 'importovf', to create VMs from an OVF manifest * snapshot: use explicitly configured vmstate storage * config: add vmstatestorage option * VM.Snapshot.Rollback privilege added * migration : enable mtunnel for insecure migration * ovmf: deprecate old legay ovmf image and refactor * improve efidisk creation * efidisk: do not hard code efivar base image size * fix #1441: do not unplug controllers when the mirroring is finished * do not overwrite global signal handlers * update_vm: sort logged parameters * remove legacy sparsecp * remove unused obsolete vmtar * fix #1125: check for KVM support before starting VM -- Proxmox Support Team Thu, 5 Oct 2017 11:17:26 +0200 qemu-server (5.0-15) unstable; urgency=medium * reduce migration downtime * fix freezing logic when saving vm states -- Proxmox Support Team Mon, 07 Aug 2017 10:46:33 +0200 qemu-server (5.0-14) unstable; urgency=medium * fix disk throttle parameters * improve compatibility when live-migrating from pve-4 -- Proxmox Support Team Mon, 17 Jul 2017 11:17:53 +0200 qemu-server (5.0-13) unstable; urgency=medium * Fix #1417: make sure the target storage allows disk images before importing -- Proxmox Support Team Thu, 13 Jul 2017 06:48:43 +0200 qemu-server (5.0-12) unstable; urgency=medium * Use default values when memory is not set in vm.conf when migrating -- Proxmox Support Team Mon, 03 Jul 2017 14:39:25 +0200 qemu-server (5.0-11) unstable; urgency=medium * Fix #1417: properly check for 'images' content type when allocating disks -- Proxmox Support Team Fri, 30 Jun 2017 09:33:17 +0200 qemu-server (5.0-10) unstable; urgency=medium * don't use cirrus by default for sane OS' * replication: remove guest states to ensure no old states are exists * allow disks on shared storages on replicated VMs * refuse to add non-replicatable disks to replicating VMs * config: has_feature() take default for backup into account -- Proxmox Support Team Wed, 28 Jun 2017 13:26:38 +0200 qemu-server (5.0-9) unstable; urgency=medium * migrate: use 'mtunnel' from pvecm -- Proxmox Support Team Fri, 23 Jun 2017 11:01:15 +0200 qemu-server (5.0-8) unstable; urgency=medium * migrate: pass the with_snapshots parameter -- Proxmox Support Team Thu, 22 Jun 2017 12:58:33 +0200 qemu-server (5.0-7) unstable; urgency=medium * PVE/QemuMigrate.pm: use new replication job helpers from AbstractMigrate * Change target in replication-state when replication direction is switched * PVE/QemuMigrate.pm: use replication job, transfer replication state * add regression tests for get_replicatable_volumes * get_replicatable_volumes: fix CDROM and local file/device handling * get_replicatable_volumes: add unused volumes * get_replicatable_volumes: skip volumes if we do not 'own' them * PVE::QemuServer::foreach_volid - pass $attr hash to callback * get_replicatable_volumes: skip volumes on shared storage * PVE/API2/Qemu.pm: acquire guest_migration_lock inside worker * Add a migration lock to avoid a replication on rollback-time * change from dpkg-deb to dpkg-buildpackage * fix #1405: sort pci ids by functions * Add new qm command 'importdisk' to import external disk images * migration: implement insecure offline migration * check also pending changes when reverting/deleting -- Proxmox Support Team Thu, 22 Jun 2017 08:50:45 +0200 qemu-server (5.0-6) unstable; urgency=medium * fix burst length parameter names and pass them to qemu * fix #1229: more explicit spice port allocation * migrate: acquire guest_migration_lock during migration * do not allow destroy if there are replication jobs * use new PVE::ReplicationConfig class * improve error on '{full, linked} clone not available' error * Fix #1384: add missing decrement to calculation of socket-id * migrate: pass ssh_info to storage_migrate * tests: fix broken snapshot create tests * tests: fix broken snapshot delete tests * use ReuseAddr for vncproxy * Fix #1361: create disk: stricter parsing of storage:size * PVE::QemuServer::create_disks - run code inside eval * implement get_replicatable_volumes * add description of read/writes statistics in vzdump output * start vncproxy worker in the background -- Proxmox Support Team Wed, 31 May 2017 09:21:27 +0200 qemu-server (5.0-5) unstable; urgency=medium * added rerror option to scsi drives * added storage replication settings -- Proxmox Support Team Fri, 28 Apr 2017 14:02:08 +0200 qemu-server (5.0-4) unstable; urgency=medium * fix target storage availability check for live migration with local storage -- Proxmox Support Team Fri, 21 Apr 2017 12:05:58 +0200 qemu-server (5.0-3) unstable; urgency=medium * fix vnc connections showing false errors * fix #1338: handle writer count limit in live migration with local disks with qemu 2.9 -- Proxmox Support Team Fri, 21 Apr 2017 11:47:07 +0200 qemu-server (5.0-2) unstable; urgency=medium * fix drive mirror readiness check * fix uninitialized warning when deleting unset VM option -- Proxmox Support Team Mon, 10 Apr 2017 16:27:59 +0200 qemu-server (5.0-1) unstable; urgency=medium * bump version for stretch -- Proxmox Support Team Mon, 13 Mar 2017 11:59:35 +0100 qemu-server (4.0-109) unstable; urgency=medium * drop netcat6 dependency * fix a case where VMs refuse to start with scsi disk ids >= 7 * add skylake to cpu models * improve error messages -- Proxmox Support Team Mon, 13 Mar 2017 11:56:14 +0100 qemu-server (4.0-109) unstable; urgency=medium * schema updates -- Proxmox Support Team Thu, 09 Feb 2017 11:40:53 +0100 qemu-server (4.0-108) unstable; urgency=medium * #1260: convert moved template disk to base * change TLS cipher suite to HIGH for spiceterm -- Proxmox Support Team Tue, 31 Jan 2017 13:54:42 +0100 qemu-server (4.0-107) unstable; urgency=medium * add dependency on libpve-guest-common-perl * add dependency on libpve-common-perl -- Proxmox Support Team Wed, 25 Jan 2017 09:47:09 +0100 qemu-server (4.0-106) unstable; urgency=medium * only use scsi-block with explicit opt-in -- Proxmox Support Team Wed, 25 Jan 2017 09:19:20 +0100 qemu-server (4.0-105) unstable; urgency=medium * use new PVE::Storage::check_volume_access() * Set zero $size and continue if volume_resize() returns false -- Proxmox Support Team Thu, 19 Jan 2017 09:19:45 +0100 qemu-server (4.0-104) unstable; urgency=medium * add setup_environment hook to CLIHandler classes * cleanup: drop superfluous condition in assignment * add romfile option to hostpci * add with-local-disks option for live storage migration * drive-mirror: bump timeout to 5s, add 30s inactivity timeout * live clone_vm: suspend or freezefs before block-job-cancel * add socat and unix socket for storage migration * add live storage migration with vm migration * clone live vm : add support for multiple jobs -- Proxmox Support Team Thu, 12 Jan 2017 14:15:01 +0100 qemu-server (4.0-103) unstable; urgency=medium * destroy_vm: allow vdisk_free to fail * Display volume size in log when doing a volume backup -- Proxmox Support Team Thu, 22 Dec 2016 12:40:40 +0100 qemu-server (4.0-102) unstable; urgency=medium * avoid "No balloon device has been activated" warnings in vmstatus -- Proxmox Support Team Tue, 20 Dec 2016 10:12:55 +0100 qemu-server (4.0-101) unstable; urgency=medium * allow migration of local qcow2 snapshots -- Proxmox Support Team Mon, 5 Dec 2016 12:38:54 +0100 qemu-server (4.0-100) unstable; urgency=medium * allow insecure migrations from older qemu-servers -- Proxmox Support Team Fri, 02 Dec 2016 18:50:54 +0100 qemu-server (4.0-99) unstable; urgency=medium * qm agent: remove 'guest-' prefix from commands * qm agent: pass command as second required argument * qm agent: add output formatter -- Proxmox Support Team Thu, 01 Dec 2016 07:58:34 +0100 qemu-server (4.0-98) unstable; urgency=medium * Add "qm agent" command * increase timeout for guest-fsfreeze-freeze -- Proxmox Support Team Wed, 30 Nov 2016 12:31:36 +0100 qemu-server (4.0-97) unstable; urgency=medium * Add entry for windows 10 and 2016 support -- Proxmox Support Team Tue, 29 Nov 2016 09:09:23 +0100 qemu-server (4.0-96) unstable; urgency=medium * restrict monitor API to Sys.Modify for most commands -- Proxmox Support Team Wed, 23 Nov 2016 10:02:37 +0100 qemu-server (4.0-95) unstable; urgency=medium * vm_shutdown: request 'stopped' state for HA enabled VMs * use ha-manager 'stopped' state instead of 'disabled' * switch to 'ha-manager set' (instead of 'ha-manager start/stop') * vmstate snapshot: activate|deactivate volume * qemu_volume_snapshot_delete : fix krbd snapshot delete * VZDump/QemuServer: set bless class correctly * cleanup windows version handling and hyperv enlightments * remove unnecessary format_description properties * register new standard option 'pve-qm-image-format' -- Proxmox Support Team Wed, 23 Nov 2016 08:11:08 +0100 qemu-server (4.0-94) unstable; urgency=medium * fix docs for iops/bps_max_length throttling options -- Proxmox Support Team Thu, 03 Nov 2016 12:53:44 +0100 qemu-server (4.0-93) unstable; urgency=medium * Close #1195: support iops/bps_max_length throttling options * fix a perl warning when failing to parse a new drive string * fix a warning: discard is not a number * fix #1177: allow dedicated migration network * change default value for cpuunits to 1024 -- Proxmox Support Team Thu, 03 Nov 2016 10:26:03 +0100 qemu-server (4.0-92) unstable; urgency=medium * Close #351: add more info to backup log * add qm listsnapshot call * snapshot_list: add bash completion for vmid * Fix #1174: remove pve-qm-drive * enable drive-mirror with iothread for qemu 2.7 v2 * add get_running_qemu_version * cpu hotplug : add new cpu hotplug method for qemu 2.7 * cpu hotplug : add coldplugged cpu to qemu command line * cpu hotplug : add print_cpu_device * numaX : use cpus option multiple time if cpulist -- Proxmox Support Team Fri, 21 Oct 2016 09:13:39 +0200 qemu-server (4.0-91) unstable; urgency=medium * fix #1131: activate volume before copying efidisk -- Proxmox Support Team Fri, 07 Oct 2016 08:21:19 +0200 qemu-server (4.0-90) unstable; urgency=medium * fix #1111: qm showcmd wrong escape sequence * Avoid to parse empty property string -- Proxmox Support Team Wed, 05 Oct 2016 07:13:33 +0200 qemu-server (4.0-89) unstable; urgency=medium * fix manual page (use efidisk0 instead of efidisk[N]) -- Proxmox Support Team Thu, 29 Sep 2016 12:21:07 +0200 qemu-server (4.0-88) unstable; urgency=medium * restore: better error handling for vdisk deletion * forbid migration of template with local base image * forbid restore into existing template -- Proxmox Support Team Fri, 16 Sep 2016 07:49:45 +0200 qemu-server (4.0-87) unstable; urgency=medium * add seabios bootsplash and use it * add efidisk0 to config, use efidisk0 for efivars -- Proxmox Support Team Thu, 08 Sep 2016 12:34:23 +0200 qemu-server (4.0-86) unstable; urgency=medium * hostpci: bring back multifunction pass-through shortcut * disable drive-mirror when iothread is enabled * hugepages: map numa node IDs to host and guest correctly * hugepages: use hostnodes value as numanode for topology * qemu-img convert : use default cache=unsafe instead writeback * Fix #1057: make protection a fast-plug option * add lock check for move_disk API call * deactivate new volumes after clone to other node * pass datacenter.cfg's mac_prefix to random_ether_addr -- Proxmox Support Team Mon, 29 Aug 2016 10:11:07 +0200 qemu-server (4.0-85) unstable; urgency=medium * Fix #1051: typo: vpcus -> vcpus - so non-root users can change vcpus again -- Proxmox Support Team Mon, 11 Jul 2016 14:46:43 +0200 qemu-server (4.0-84) unstable; urgency=medium * fix #1046: add non-snapshotted disks as unused * fix #1040: warn early about moving a snapshotted disk * disable usb hotplug for now * remove old move disk snapshot check * improve errors when trying to migrate disks with snapshots -- Proxmox Support Team Mon, 11 Jul 2016 11:50:01 +0200 qemu-server (4.0-83) unstable; urgency=medium * Add hugepages option * Refactor various parts out of QemuServer * Include USB devices in vm_devices_list * Add usb hotplugging * Fix syntax in pve-q35.cfg * Fix #146: add name to backup log -- Proxmox Support Team Wed, 22 Jun 2016 11:29:01 +0200 qemu-server (4.0-82) unstable; urgency=medium * add dependency on DBus -- Proxmox Support Team Mon, 20 Jun 2016 11:48:45 +0200 qemu-server (4.0-81) unstable; urgency=medium * Add LVM and LVMThin to QemuMigration * add check for snapshots at migration -- Proxmox Support Team Thu, 16 Jun 2016 12:10:48 +0200 qemu-server (4.0-80) unstable; urgency=medium * split old style pipe open call * fix #1020: remove unnecessary root check for unused disks -- Proxmox Support Team Thu, 09 Jun 2016 18:13:03 +0200 qemu-server (4.0-79) unstable; urgency=medium * migrate: collect migration tunnel child process (avoid zombies) * migrate: use ssh forwarded UNIX socket tunnel * migrate: add some more log output -- Proxmox Support Team Fri, 03 Jun 2016 12:16:31 +0200 qemu-server (4.0-78) unstable; urgency=medium * use enter_systemd_scope instead of systemd-run * fix #1010: whitelist options for permissions -- Proxmox Support Team Fri, 03 Jun 2016 11:42:25 +0200 qemu-server (4.0-77) unstable; urgency=medium * do not ignore hotplug parse errors -- Proxmox Support Team Tue, 31 May 2016 12:15:55 +0200 qemu-server (4.0-76) unstable; urgency=medium * allow VLAN 1 tag in qemu-kvm vms -- Proxmox Support Team Wed, 18 May 2016 11:25:54 +0200 qemu-server (4.0-75) unstable; urgency=medium * fix #975, use new keyAlias feature. * add --description to systemd scope unit -- Proxmox Support Team Wed, 11 May 2016 11:15:56 +0200 qemu-server (4.0-74) unstable; urgency=medium * correctly set cpu vendor * fix #971: don't activate shared storage in offline migration * migrate: check if storage is available -- Proxmox Support Team Sun, 01 May 2016 09:25:06 +0200 qemu-server (4.0-73) unstable; urgency=medium * restore: pass format to vma extract -- Proxmox Support Team Fri, 29 Apr 2016 09:03:12 +0200 qemu-server (4.0-72) unstable; urgency=medium * vm_start : force systemctl stop if orphan scope exist -- Proxmox Support Team Fri, 22 Apr 2016 11:09:52 +0200 qemu-server (4.0-71) unstable; urgency=medium * Fix #643: activate volumes before resizing * fix #947: re-enable disk/cdrom passthrough -- Proxmox Support Team Thu, 21 Apr 2016 10:34:42 +0200 qemu-server (4.0-70) unstable; urgency=medium * vm_status: return more verbose HA state -- Proxmox Support Team Tue, 19 Apr 2016 09:01:24 +0200 qemu-server (4.0-69) unstable; urgency=medium * Fix #848: deactivate old volume after clone before deletion -- Proxmox Support Team Wed, 13 Apr 2016 08:24:49 +0200 qemu-server (4.0-68) unstable; urgency=medium * change shutdown behaviour on suspended vm -- Proxmox Support Team Tue, 12 Apr 2016 17:19:20 +0200 qemu-server (4.0-67) unstable; urgency=medium * use pve-doc-generator to generate man pages -- Proxmox Support Team Fri, 08 Apr 2016 07:37:16 +0200 qemu-server (4.0-66) unstable; urgency=medium * use generic property string parser * drive schema: allow 'none' again -- Proxmox Support Team Fri, 01 Apr 2016 09:33:16 +0200 qemu-server (4.0-65) unstable; urgency=medium * cfg: use the 'urlencoded' format for drive model and serial * clone: use the zeroinit filter for sparseinit storages -- Proxmox Support Team Mon, 21 Mar 2016 09:09:07 +0100 qemu-server (4.0-64) unstable; urgency=medium * Use AbstractConfig for Qemu * fix #909: pass rate to tap_plug() -- Proxmox Support Team Tue, 08 Mar 2016 11:43:28 +0100 qemu-server (4.0-63) unstable; urgency=medium * only perform scsi inquiry on device nodes -- Proxmox Support Team Sat, 27 Feb 2016 10:20:50 +0100 qemu-server (4.0-62) unstable; urgency=medium * fix undefined value when starting a q35 machine VM * kvm_user_version: update code to use our framework * Refactor has_feature -- Proxmox Support Team Fri, 26 Feb 2016 17:02:34 +0100 qemu-server (4.0-61) unstable; urgency=medium * clone_vm : only deactivate sources volume if source vm if offline * Refactor snapshot_create to match LXC.pm -- Proxmox Support Team Thu, 25 Feb 2016 08:48:59 +0100 qemu-server (4.0-60) unstable; urgency=medium * change check for write-zeros -- Proxmox Support Team Wed, 24 Feb 2016 17:19:12 +0100 qemu-server (4.0-59) unstable; urgency=medium * qemu_machine_pxe : return $machine if no pxe -- Proxmox Support Team Mon, 22 Feb 2016 17:37:11 +0100 qemu-server (4.0-58) unstable; urgency=medium * deactivate volumes if vm start command fails -- Proxmox Support Team Sat, 20 Feb 2016 10:26:28 +0100 qemu-server (4.0-57) unstable; urgency=medium * delete ram state files when restoring a backup * Refactor snapshot code * disable vnc server and add -nographic is no vga card is present * passthrough : re-enable hyperv and add hv_vendor_id for windows -- Proxmox Support Team Thu, 18 Feb 2016 12:47:51 +0100 qemu-server (4.0-56) unstable; urgency=medium * If we freeze the fs with the Qemu-Guest-Agent test if QGA is running. -- Proxmox Support Team Mon, 15 Feb 2016 12:51:19 +0100 qemu-server (4.0-55) unstable; urgency=medium * improve usb config parsing * Refactor update_config_nolock -> write_config -- Proxmox Support Team Fri, 12 Feb 2016 12:07:56 +0100 qemu-server (4.0-54) unstable; urgency=medium * restore: deal with new backup=0 property string * add usb3 option for usb-devices -- Proxmox Support Team Wed, 10 Feb 2016 17:49:19 +0100 qemu-server (4.0-53) unstable; urgency=medium * pass $skiplock all the way through to destroy_vm -- Proxmox Support Team Mon, 08 Feb 2016 11:53:03 +0100 qemu-server (4.0-52) unstable; urgency=medium * close tunnel after migration is finish. -- Proxmox Support Team Tue, 02 Feb 2016 18:16:47 +0100 qemu-server (4.0-51) unstable; urgency=medium * Fix #878: disk-size format * Fix #879: exclusion of disk for backup -- Proxmox Support Team Fri, 29 Jan 2016 10:04:41 +0100 qemu-server (4.0-50) unstable; urgency=medium * add hidden option to cpu type * fix PVE::HA use clause so HA resources get registered * Create firewall dir on VM restore -- Proxmox Support Team Tue, 26 Jan 2016 16:57:43 +0100 qemu-server (4.0-49) unstable; urgency=medium * increase version to update test environment -- Proxmox Support Team Tue, 26 Jan 2016 12:50:07 +0100 qemu-server (4.0-48) unstable; urgency=medium * use safe_string_ne for trunks -- Proxmox Support Team Mon, 18 Jan 2016 16:58:23 +0100 qemu-server (4.0-47) unstable; urgency=medium * add support for network trunks -- Proxmox Support Team Fri, 15 Jan 2016 17:27:32 +0100 qemu-server (4.0-46) unstable; urgency=medium * Closes #787: add Haswell-noTSX and Broadwell-noTSX cpu types -- Proxmox Support Team Wed, 13 Jan 2016 16:42:05 +0100 qemu-server (4.0-45) unstable; urgency=medium * io throttle: pass pool parameters (*_max) -- Proxmox Support Team Wed, 13 Jan 2016 06:17:52 +0100 qemu-server (4.0-44) unstable; urgency=medium * ovmf : don't pass x-vga to vfio-pci * check for quorum when starting a VM -- Proxmox Support Team Sun, 10 Jan 2016 15:11:43 +0100 qemu-server (4.0-43) unstable; urgency=medium * disable hyper-v enlightment when xvga is enabled -- Proxmox Support Team Mon, 04 Jan 2016 06:26:31 +0100 qemu-server (4.0-42) unstable; urgency=medium * add detect_zeroes option -- Proxmox Support Team Fri, 18 Dec 2015 09:20:18 +0100 qemu-server (4.0-41) unstable; urgency=medium * add new option to select BIOS (--bios ) * fix bug #841: replace get_used_paths with is_volume_in_use -- Proxmox Support Team Thu, 10 Dec 2015 11:15:23 +0100 qemu-server (4.0-40) unstable; urgency=medium * qm: Add VMID auto completion to some commands -- Proxmox Support Team Wed, 09 Dec 2015 12:22:25 +0100 qemu-server (4.0-39) unstable; urgency=medium * fix bug #828: activate disks before hotplugging them * fix bug #783: set KillMode=none, so that systemd don't kill them at shutdown * re-enable steal time * backup/restore firewall config with vzdump -- Proxmox Support Team Tue, 24 Nov 2015 16:50:11 +0100 qemu-server (4.0-38) unstable; urgency=medium * Don't treat serial devices as a local resource if they point to a socket. * qemu_img_convert: activate source volume -- Proxmox Support Team Fri, 13 Nov 2015 07:02:01 +0100 qemu-server (4.0-37) unstable; urgency=medium * add pve-bridge-hotplug script -- Proxmox Support Team Fri, 06 Nov 2015 16:24:44 +0100 qemu-server (4.0-36) unstable; urgency=medium * use qom-get to check if old pxe network rom file are used * migration: improve ipv6 case -- Proxmox Support Team Fri, 06 Nov 2015 07:56:14 +0100 qemu-server (4.0-35) unstable; urgency=medium * clone: use a fullclone hash instead of $drive->{full} -- Proxmox Support Team Fri, 30 Oct 2015 07:06:47 +0100 qemu-server (4.0-34) unstable; urgency=medium * use old netdevice bios files for older machine types (<= 2.4) -- Proxmox Support Team Wed, 28 Oct 2015 09:08:49 +0100 qemu-server (4.0-31) unstable; urgency=medium * migrate : add nocheck for resume -- Proxmox Support Team Thu, 15 Oct 2015 12:41:43 +0200 qemu-server (4.0-30) unstable; urgency=medium * qmrestore: implement bash completion * create: add better check for unused IDs -- Proxmox Support Team Mon, 05 Oct 2015 13:13:07 +0200 qemu-server (4.0-29) unstable; urgency=medium * support serial numbers and models for disks -- Proxmox Support Team Wed, 30 Sep 2015 10:56:30 +0200 qemu-server (4.0-28) unstable; urgency=medium * disable kvm_steal_time, because it's currently buggy with live migration -- Proxmox Support Team Tue, 29 Sep 2015 07:14:44 +0200 qemu-server (4.0-27) unstable; urgency=medium * allow to change boot order with VM.Config.Options privileges -- Proxmox Support Team Sat, 26 Sep 2015 11:07:11 +0200 qemu-server (4.0-26) unstable; urgency=medium * migration: disable compress (already default) * migration: enable xbzrle -- Proxmox Support Team Fri, 25 Sep 2015 17:59:30 +0200 qemu-server (4.0-25) unstable; urgency=medium * fix kvm version parser for CVE stable releases -- Proxmox Support Team Wed, 23 Sep 2015 11:48:09 +0200 qemu-server (4.0-24) unstable; urgency=medium * improve VM protection mode * pci passthough : make vfio default * fix error message: allocate to much v-CPUs -- Proxmox Support Team Mon, 21 Sep 2015 12:28:19 +0200 qemu-server (4.0-23) unstable; urgency=medium * destroy: avoid warning about undefined 'protection' value -- Proxmox Support Team Thu, 10 Sep 2015 09:38:30 +0200 qemu-server (4.0-22) unstable; urgency=medium * add possibility to restore backup on rbd in krbd mode * add krbd support to online snapshot -- Proxmox Support Team Wed, 09 Sep 2015 07:55:31 +0200 qemu-server (4.0-21) unstable; urgency=medium * qm: implement bash completion * add VM protection mode * fix move_disk on RBD * add HA resources check before destroying a VM -- Proxmox Support Team Tue, 08 Sep 2015 10:51:05 +0200 qemu-server (4.0-20) unstable; urgency=medium * fix start kvm with os type 'other' * spice console: fix 'uninitialized value in concat' due to unnamed VM -- Proxmox Support Team Fri, 28 Aug 2015 11:39:52 +0200 qemu-server (4.0-19) unstable; urgency=medium * clone: also copy vm firewall config file -- Proxmox Support Team Tue, 25 Aug 2015 06:50:04 +0200 qemu-server (4.0-18) unstable; urgency=medium * fix bug #688: if vm is not owner of this disk remove from config -- Proxmox Support Team Thu, 20 Aug 2015 12:30:52 +0200 qemu-server (4.0-17) unstable; urgency=medium * fix bug #603: vmid.fw file not deleted * fix bug #517: improve error message * adapt /config and /pending API calls to force-delete -- Proxmox Support Team Wed, 19 Aug 2015 15:44:53 +0200 qemu-server (4.0-16) unstable; urgency=medium * remove vm access permissions after destroy * pending-delete: remember force-deletes * correctly handle empty description in pending section -- Proxmox Support Team Fri, 14 Aug 2015 08:09:44 +0200 qemu-server (4.0-15) unstable; urgency=medium * Fixed wrong UUID in Qemu VZDump backup * add memory_unplug support V2 -- Proxmox Support Team Thu, 30 Jul 2015 11:32:51 +0200 qemu-server (4.0-14) unstable; urgency=medium * fixed bug 662: wrong subroutine for parsing startup order * cpuflags : don't enforce with tcg mode * cpuflags : remove enforce for cpumodel=host * cpuflags : remove -rdtscp for Opteron cpu models * vzdump : abort backup if disk have iothread enabled -- Proxmox Support Team Mon, 27 Jul 2015 13:24:31 +0200 qemu-server (4.0-13) unstable; urgency=medium * drive-mirror: now allow interrupts at the scanning bitmap phase -- Proxmox Support Team Wed, 01 Jul 2015 06:40:52 +0200 qemu-server (4.0-12) unstable; urgency=medium * remove outdated host_device format * parse_drive: do not overwrite configured format * fix aio O_DIRECT check for cdrom drives -- Proxmox Support Team Wed, 10 Jun 2015 10:32:09 +0200 qemu-server (4.0-11) unstable; urgency=medium * activate-noawait pve-api-updates -- Proxmox Support Team Mon, 01 Jun 2015 12:36:15 +0200 qemu-server (4.0-10) unstable; urgency=medium * implement cgroups through systemd-run * implement hotplug for cpuunits * remove old openvz fairscheduler code -- Proxmox Support Team Fri, 29 May 2015 08:24:33 +0200 qemu-server (4.0-9) unstable; urgency=medium * qmp drive-mirror : set big timeout * qemu-mirror : block job complete : use ready flag -- Proxmox Support Team Tue, 12 May 2015 08:19:38 +0200 qemu-server (4.0-8) unstable; urgency=medium * trigger pve-api-updates event -- Proxmox Support Team Tue, 05 May 2015 14:57:55 +0200 qemu-server (4.0-7) unstable; urgency=medium * cleanup: use new standard option 'pve-startup-order' -- Proxmox Support Team Wed, 22 Apr 2015 10:04:04 +0200 qemu-server (4.0-6) unstable; urgency=medium * fix ha resource names -- Proxmox Support Team Fri, 17 Apr 2015 13:11:12 +0200 qemu-server (4.0-5) unstable; urgency=medium * use new pve-ha-manager -- Proxmox Support Team Fri, 27 Mar 2015 12:48:41 +0100 qemu-server (4.0-4) unstable; urgency=medium * add qemu cpu flags optimisations for kernel 3.10 -- Proxmox Support Team Tue, 17 Mar 2015 08:58:56 +0100 qemu-server (4.0-3) unstable; urgency=medium * vmstatus: use vcpus if defined * balloon: use qom-get for guest balloon statistics -- Proxmox Support Team Mon, 09 Mar 2015 08:22:47 +0100 qemu-server (4.0-2) unstable; urgency=medium * drive_add: escape \ character * bugfix: allow manual balloning if shares = 0 -- Proxmox Support Team Fri, 06 Mar 2015 10:21:52 +0100 qemu-server (4.0-1) unstable; urgency=medium * updated for debian jessie -- Proxmox Support Team Fri, 27 Feb 2015 13:00:56 +0100 qemu-server (3.3-20) unstable; urgency=low * correctly set an remove lock (fix "qm unlock") -- Proxmox Support Team Sun, 15 Feb 2015 09:05:40 +0100 qemu-server (3.3-19) unstable; urgency=low * commit pending values when changing CDROM -- Proxmox Support Team Sat, 14 Feb 2015 09:24:03 +0100 qemu-server (3.3-18) unstable; urgency=low * bugfix : add missing queues nic option in print_net -- Proxmox Support Team Fri, 13 Feb 2015 09:05:21 +0100 qemu-server (3.3-17) unstable; urgency=low * fix CDROM hotplug * This fix hotplug for devices behind bridges * check possibility to rollback a snapshot early -- Proxmox Support Team Thu, 12 Feb 2015 08:19:02 +0100 qemu-server (3.3-16) unstable; urgency=low * QemuServer: fix wrong binding of pci root ports, bridges or switches to vfio -- Proxmox Support Team Wed, 11 Feb 2015 06:32:14 +0100 qemu-server (3.3-15) unstable; urgency=low * code cleanup: add foreach_dimm sub * hotplug: memory hotplug option is not hotpluggable * fix bug 597: hotplug fix -- Proxmox Support Team Mon, 09 Feb 2015 07:05:05 +0100 qemu-server (3.3-14) unstable; urgency=low * include memory hotplug patch -- Proxmox Support Team Wed, 28 Jan 2015 07:11:10 +0100 qemu-server (3.3-13) unstable; urgency=low * hotplug config: allow to enable specific features -- Proxmox Support Team Tue, 27 Jan 2015 12:39:21 +0100 qemu-server (3.3-12) unstable; urgency=low * add new vcpus option -- Proxmox Support Team Fri, 23 Jan 2015 08:04:44 +0100 qemu-server (3.3-11) unstable; urgency=low * enable hotplug by default -- Proxmox Support Team Wed, 21 Jan 2015 08:53:10 +0100 qemu-server (3.3-10) unstable; urgency=low * Support additional e1000 variants for VM machines * Add link_down flag to network config -- Proxmox Support Team Tue, 20 Jan 2015 07:15:48 +0100 qemu-server (3.3-9) unstable; urgency=low * pending api : fix parsing 0 value * fix test for ballon hotplug * set boot strict=on to prevent booting from not listed boot devices -- Proxmox Support Team Thu, 15 Jan 2015 06:23:10 +0100 qemu-server (3.3-8) unstable; urgency=low * new 'pending changes' API -- Proxmox Support Team Thu, 08 Jan 2015 13:32:46 +0100 qemu-server (3.3-7) unstable; urgency=low * vzdump: use qga freeze in vzdump in snapshot mode * add custom numa topology support -- Proxmox Support Team Mon, 22 Dec 2014 17:24:24 +0100 qemu-server (3.3-6) unstable; urgency=low * API change: snapshot_create - remove unused freezefs flag. * snapshot_create: call fsfreeze if agent flag is set * qmpclient: use guest-sync-delimited -- Proxmox Support Team Thu, 04 Dec 2014 12:34:50 +0100 qemu-server (3.3-5) unstable; urgency=low * drive-mirror: cleanups, avoid division by zero bug * shutdown by Qemu Guest Agent if the agent flag is set * savevm-end : wait that savevm is finished -- Proxmox Support Team Mon, 01 Dec 2014 09:31:17 +0100 qemu-server (3.3-4) unstable; urgency=low * serial: allow to pass arbitrary device names * Add check if host has enough real CPUs for starting VM -- Proxmox Support Team Mon, 24 Nov 2014 07:42:10 +0100 qemu-server (3.3-3) unstable; urgency=low * drive-mirror: further improvements -- Proxmox Support Team Mon, 10 Nov 2014 06:32:04 +0100 qemu-server (3.3-2) unstable; urgency=low * drive-mirror: wait that busy eq false before block-job-complete -- Proxmox Support Team Fri, 07 Nov 2014 15:35:55 +0100 qemu-server (3.3-1) unstable; urgency=low * enable write zeroes optimisations -- Proxmox Support Team Mon, 13 Oct 2014 10:35:49 +0200 qemu-server (3.1-35) unstable; urgency=low * fix bug #542: return VMID as integer -- Proxmox Support Team Wed, 17 Sep 2014 15:52:34 +0200 qemu-server (3.1-34) unstable; urgency=low * hotplug: allow scsi and virtio-scsi disk hotplug|unplug -- Proxmox Support Team Mon, 01 Sep 2014 11:35:49 +0200 qemu-server (3.1-33) unstable; urgency=low * clone_vm: auto generate new uuid -- Proxmox Support Team Tue, 26 Aug 2014 09:23:58 +0200 qemu-server (3.1-32) unstable; urgency=low * add Broadwell cpu model -- Proxmox Support Team Wed, 20 Aug 2014 12:21:08 +0200 qemu-server (3.1-31) unstable; urgency=low * fix bug 529: fix virtio-serial when using q35 machines -- Proxmox Support Team Wed, 13 Aug 2014 06:17:08 +0200 qemu-server (3.1-30) unstable; urgency=low * bump max hostpci to 4 * set vga=none if x-vga passthrough is enabled -- Proxmox Support Team Wed, 06 Aug 2014 09:41:36 +0200 qemu-server (3.1-29) unstable; urgency=low * vm_stop: do not use ha commands if $migratedfrom is set -- Proxmox Support Team Tue, 29 Jul 2014 06:53:05 +0200 qemu-server (3.1-28) unstable; urgency=low * disable kvm cpu signature if x-vga is enabled -- Proxmox Support Team Thu, 24 Jul 2014 06:52:25 +0200 qemu-server (3.1-27) unstable; urgency=low * pci passthrough: Reset device only if has_fl_reset is defined -- Proxmox Support Team Wed, 23 Jul 2014 06:11:49 +0200 qemu-server (3.1-26) unstable; urgency=low * enable linked clones from snapshots (ceph) -- Proxmox Support Team Thu, 17 Jul 2014 09:28:47 +0200 qemu-server (3.1-25) unstable; urgency=low * allow resize of virtio windows boot disk (virtio-win-0.1-74 have fixed the resize bug) -- Proxmox Support Team Wed, 16 Jul 2014 12:48:03 +0200 qemu-server (3.1-24) unstable; urgency=low * new option smbios1: specify SMBIOS type 1 fields (uuid, ...) -- Proxmox Support Team Thu, 26 Jun 2014 11:13:27 +0200 qemu-server (3.1-23) unstable; urgency=low * vncproxy: remove check if VM is running -- Proxmox Support Team Wed, 25 Jun 2014 09:56:02 +0200 qemu-server (3.1-22) unstable; urgency=low * hostpci: add pci multifunction support * hostpci: add pcie and x-vga passthrough -- Proxmox Support Team Wed, 25 Jun 2014 09:31:30 +0200 qemu-server (3.1-21) unstable; urgency=low * protect websocket API with vncticket -- Proxmox Support Team Tue, 24 Jun 2014 17:43:25 +0200 qemu-server (3.1-20) unstable; urgency=low * vncwebsocket: do not proxy connection (not required) -- Proxmox Support Team Wed, 18 Jun 2014 12:45:39 +0200 qemu-server (3.1-19) unstable; urgency=low * do not depend on novnc-pve package (use new HTTPServer feature instead) -- Proxmox Support Team Wed, 18 Jun 2014 11:04:48 +0200 qemu-server (3.1-18) unstable; urgency=low * depend on novnc-pve package * support webockets for VNC console * migration : add setup state * add virtio-net multiqueue support * add firewall option to qemu network interface * add initiator-name to iscsi drives if configured -- Proxmox Support Team Tue, 17 Jun 2014 09:00:03 +0200 qemu-server (3.1-17) unstable; urgency=low * depend on pve-firewall package -- Proxmox Support Team Tue, 06 May 2014 11:28:26 +0200 qemu-server (3.1-16) unstable; urgency=low * fix bug #502: allow creation of empty vma archives * fix bug #510: move_disk - don't delete disk if used in a previous snasphot * qmrestore: removed short timeout -- Proxmox Support Team Thu, 17 Apr 2014 09:29:26 +0200 qemu-server (3.1-15) unstable; urgency=low * correctly disable unwanted migrate features (xbzrle). * QemuMigrate: print migration xbzrle if enabled -- Proxmox Support Team Mon, 10 Feb 2014 08:06:29 +0100 qemu-server (3.1-14) unstable; urgency=low * Add title of VM to Spice Client * deactivate volume after move_disk * add cpu_hotplug (and maxcpus config) * migration : enable auto-converge capability -- Proxmox Support Team Wed, 29 Jan 2014 06:49:16 +0100 qemu-server (3.1-13) unstable; urgency=low * pci passthrough: new option to disable device ROM (rombar=off) -- Proxmox Support Team Fri, 13 Dec 2013 11:45:46 +0100 qemu-server (3.1-12) unstable; urgency=low * spiceproxy: use POST instead of GET -- Proxmox Support Team Tue, 10 Dec 2013 10:49:30 +0100 qemu-server (3.1-11) unstable; urgency=low * depend on pve-qemu-kvm >= 1.7-1 * fix 'qm unlink' command syntax * add 'pvscsi' to the list of scsi controllers (emulate the VMware PVSCSI device) * add 'lsi53c810' to the list of scsi controllers (supported on some very old Windows NT versions) * add 'vmxnet3' to the list of available network card models (emulate VMware paravirtualized network card) * add drive option 'discard' * add support for new qemu throttling burst max parameters. -- Proxmox Support Team Tue, 03 Dec 2013 10:48:33 +0100 qemu-server (3.1-10) unstable; urgency=low * add +lahf_lm flag to kvm64 cpudef -- Proxmox Support Team Fri, 29 Nov 2013 09:18:39 +0100 qemu-server (3.1-9) unstable; urgency=low * clone: deactivate volumes after clone to other node -- Proxmox Support Team Tue, 19 Nov 2013 08:17:35 +0100 qemu-server (3.1-8) unstable; urgency=low * clone: correctly check drive options (avoid feature is not available bug) -- Proxmox Support Team Mon, 14 Oct 2013 07:36:26 +0200 qemu-server (3.1-7) unstable; urgency=low * spice: add multi-monitors support -- Proxmox Support Team Wed, 02 Oct 2013 09:13:23 +0200 qemu-server (3.1-6) unstable; urgency=low * use new PVE::Storage::abs_filesystem_path() * use warnings instead of global -w flag -- Proxmox Support Team Tue, 01 Oct 2013 12:42:42 +0200 qemu-server (3.1-5) unstable; urgency=low * introduce new ostype 'solaris' (run without x2apic). -- Proxmox Support Team Tue, 24 Sep 2013 06:55:33 +0200 qemu-server (3.1-4) unstable; urgency=low * qemu migrate : only wait for spice server online + eval -- Proxmox Support Team Thu, 19 Sep 2013 06:29:39 +0200 qemu-server (3.1-3) unstable; urgency=low * speedup restore on glusterfs (do not write zero bytes) -- Proxmox Support Team Tue, 17 Sep 2013 09:13:34 +0200 qemu-server (3.1-2) unstable; urgency=low * Allow VMAdmin/DatastoreUser to delete disk -- Proxmox Support Team Thu, 05 Sep 2013 07:48:07 +0200 qemu-server (3.1-1) unstable; urgency=low * allow pass through of usb parallel devices (--parallel0 /dev/usb/lp0) -- Proxmox Support Team Wed, 14 Aug 2013 12:20:23 +0200 qemu-server (3.0-30) unstable; urgency=low * fix bugs in migration code (wrong qw() usage) -- Proxmox Support Team Mon, 12 Aug 2013 09:49:20 +0200 qemu-server (3.0-29) unstable; urgency=low * vncproxy: load config from correct node -- Proxmox Support Team Tue, 06 Aug 2013 08:15:59 +0200 qemu-server (3.0-28) unstable; urgency=low * allow to use a socket for serial devices * implement 'qm terminal' to open terminal via serial device * add ability to run without graphic card ('vga: serial[n]') -- Proxmox Support Team Fri, 02 Aug 2013 10:05:35 +0200 qemu-server (3.0-27) unstable; urgency=low * add support for insecure/fast migration (setting in datacenter.cfg) -- Proxmox Support Team Fri, 26 Jul 2013 11:24:36 +0200 qemu-server (3.0-26) unstable; urgency=low * remove spice cert paths (depend on pve-qemu-kvm >= 1.4-16) * implement spice seamless migration -- Proxmox Support Team Tue, 23 Jul 2013 10:08:33 +0200 qemu-server (3.0-25) unstable; urgency=low * support usb redirection devices for spice (usb[n]: spice) * disable tablet device by default for spice -- Proxmox Support Team Fri, 19 Jul 2013 09:38:25 +0200 qemu-server (3.0-24) unstable; urgency=low * spiceproxy API: allow client to choose proxy address * spiceproxy API: read cert subject name directly using Net::SSLeay * spice: use TLS (encrypt whole traffic) -- Proxmox Support Team Thu, 18 Jul 2013 08:14:46 +0200 qemu-server (3.0-23) unstable; urgency=low * allow to pass SCSI generic devices to guests, for example "scsi0: /dev/sg5" -- Proxmox Support Team Tue, 16 Jul 2013 06:49:45 +0200 qemu-server (3.0-22) unstable; urgency=low * cpu flags optimization -- Proxmox Support Team Mon, 15 Jul 2013 09:14:49 +0200 qemu-server (3.0-21) unstable; urgency=low * add support for SPICE -- Proxmox Support Team Wed, 26 Jun 2013 13:15:48 +0200 qemu-server (3.0-20) unstable; urgency=low * new API to update VM config: this one is fully asynchronous. * snapshot rollback: use pc-i440fx-1.4 as default -- Proxmox Support Team Fri, 07 Jun 2013 11:44:16 +0200 qemu-server (3.0-19) unstable; urgency=low * config: implement new 'machine' configuration * migrate: pass --machine parameter to remote 'qm start' command * snapshot: store/use 'machine' confifiguration -- Proxmox Support Team Wed, 05 Jun 2013 10:27:47 +0200 qemu-server (3.0-18) unstable; urgency=low * implement delete flag for move_disk * API: rename move to move_disk -- Proxmox Support Team Fri, 31 May 2013 10:57:50 +0200 qemu-server (3.0-17) unstable; urgency=low * implement storage migration ("qm move") -- Proxmox Support Team Wed, 29 May 2013 13:01:16 +0200 qemu-server (3.0-16) unstable; urgency=low * fix bug 395: correctly handle unused disk with storage alias * fix unused disk handling (do not hide unused disks when used with snapshot). -- Proxmox Support Team Tue, 28 May 2013 12:23:12 +0200 qemu-server (3.0-15) unstable; urgency=low * qemu_img_format : use raw for as default for other storage -- Proxmox Support Team Thu, 23 May 2013 11:33:47 +0200 qemu-server (3.0-14) unstable; urgency=low * fix bug #389: avoid error if balloon is undefined -- Proxmox Support Team Wed, 22 May 2013 07:17:38 +0200 qemu-server (3.0-13) unstable; urgency=low * fix bug #391 - restore: test if requested format is supported -- Proxmox Support Team Tue, 21 May 2013 12:03:56 +0200 qemu-server (3.0-12) unstable; urgency=low * use add_vm_to_pool/remove_vm_from_pool from PVE::AccessControl -- Proxmox Support Team Tue, 14 May 2013 12:02:41 +0200 qemu-server (3.0-11) unstable; urgency=low * clone disk : keep source volume params -- Proxmox Support Team Tue, 14 May 2013 10:18:21 +0200 qemu-server (3.0-10) unstable; urgency=low * fix bug #381: use PVE::Tools::next_migrate_port() * clone: check is we can clone to target storage -- Proxmox Support Team Mon, 13 May 2013 07:31:47 +0200 qemu-server (3.0-9) unstable; urgency=low * restore: do not restore template flag -- Proxmox Support Team Wed, 08 May 2013 10:23:49 +0200 qemu-server (3.0-8) unstable; urgency=low * qemu_img_format: use 'raw' for lvm * drive-mirror : die if stats are empty * set long timeout for query-block-jobs -- Proxmox Support Team Tue, 07 May 2013 10:19:19 +0200 qemu-server (3.0-7) unstable; urgency=low * has_feature: return list of allowed nodes -- Proxmox Support Team Mon, 06 May 2013 08:58:35 +0200 qemu-server (3.0-6) unstable; urgency=low * fix bug #379 - restore: allow to overwrite existing VMs if user has VM.Backup permissions -- Proxmox Support Team Fri, 03 May 2013 07:53:40 +0200 qemu-server (3.0-5) unstable; urgency=low * implement shared file locks -- Proxmox Support Team Thu, 25 Apr 2013 11:35:01 +0200 qemu-server (3.0-4) unstable; urgency=low * fix bug 377: make qm rescan work properly * avoid endless loop in QMPClient * use vm create permissions for templates -- Proxmox Support Team Wed, 24 Apr 2013 07:59:28 +0200 qemu-server (3.0-3) unstable; urgency=low * allow sparse restore for sheepdog and rbd -- Proxmox Support Team Thu, 18 Apr 2013 06:21:40 +0200 qemu-server (3.0-2) unstable; urgency=low * fix warnings in syslog -- Proxmox Support Team Fri, 22 Mar 2013 06:25:12 +0100 qemu-server (3.0-1) unstable; urgency=low * start 3.0 development -- Proxmox Support Team Tue, 05 Mar 2013 11:39:08 +0100 qemu-server (2.3-17) unstable; urgency=low * fix backup-cancel timeout -- Proxmox Support Team Fri, 01 Mar 2013 10:58:58 +0100 qemu-server (2.3-16) unstable; urgency=low * vncproxy (another try): wait max 10s for the socket if it does not exist -- Proxmox Support Team Fri, 01 Mar 2013 06:42:36 +0100 qemu-server (2.3-15) unstable; urgency=low * revert vncproxy change because it breaks console on remote hosts. -- Proxmox Support Team Thu, 28 Feb 2013 12:53:59 +0100 qemu-server (2.3-14) unstable; urgency=low * bugfix #340 : don't set cache=none to cdrom * Migrate: fix check if a backing file exist * vncproxy: wait max 10s for the socket if it does not exist * vzdump: improve error reporting -- Proxmox Support Team Thu, 28 Feb 2013 06:21:30 +0100 qemu-server (2.3-13) unstable; urgency=low * set cache=none by default -- Proxmox Support Team Mon, 25 Feb 2013 06:12:14 +0100 qemu-server (2.3-12) unstable; urgency=low * disable hotplug by default (revert previous change) * disable usb2 controller by default (revert previous change) -- Proxmox Support Team Fri, 22 Feb 2013 09:52:13 +0100 qemu-server (2.3-11) unstable; urgency=low * fix backup parameters for pve-qemu-kvm 1.4-4 -- Proxmox Support Team Wed, 20 Feb 2013 10:47:55 +0100 qemu-server (2.3-10) unstable; urgency=low * enable hotplug by default * enable usb2 controller by default -- Proxmox Support Team Tue, 19 Feb 2013 10:45:28 +0100 qemu-server (2.3-9) unstable; urgency=low * depend on pve-qemu-kvm >= 1.4-1 * qemu 1.4 fix : rename stats-polling-interval to guest-stats-polling-interval * remove expected_downtime from migration status (not set in qemu 1.4, always 0) * do not set cache=none for .raw files - use qemu default instead (writeback) -- Proxmox Support Team Wed, 13 Feb 2013 10:38:13 +0100 qemu-server (2.3-8) unstable; urgency=low * fix tar restore: correctly check if VM config already exists -- Proxmox Support Team Mon, 28 Jan 2013 09:53:12 +0100 qemu-server (2.3-7) unstable; urgency=low * allow to suspend/resume VM during backup -- Proxmox Support Team Thu, 17 Jan 2013 10:25:17 +0100 qemu-server (2.3-6) unstable; urgency=low * fix bug #315: cancel backup before stopping the VM -- Proxmox Support Team Wed, 16 Jan 2013 13:22:58 +0100 qemu-server (2.3-5) unstable; urgency=low * fix bug #307: correctly restore disk settings -- Proxmox Support Team Mon, 07 Jan 2013 06:48:48 +0100 qemu-server (2.3-4) unstable; urgency=low * qmrestore vma files: when restoring to same VMID, only remove volumes contained in backup - other volumes will be still available as 'unused' disks. Note: tar restore still removes all volumes. -- Proxmox Support Team Fri, 04 Jan 2013 07:02:00 +0100 qemu-server (2.3-3) unstable; urgency=low * fix Bug #293: CDROM size not reset when set to use no media * set migrate_downtime default value to 0.1 (and use float instead of int) * implement dynamic migration_downtime * allow manual ballooning if shares is set to zero * new api2 call: vm_feature (test support for snapshots/clone feature) -- Proxmox Support Team Wed, 02 Jan 2013 06:31:38 +0100 qemu-server (2.3-2) unstable; urgency=low * use memory info from ballon driver (if enabled) -- Proxmox Support Team Tue, 18 Dec 2012 12:51:57 +0100 qemu-server (2.3-1) unstable; urgency=low * include new qemu backup feature -- Proxmox Support Team Wed, 12 Dec 2012 15:14:59 +0100 qemu-server (2.0-71) unstable; urgency=low * show better error message if bridge does not exist * QMPClient: support fdsets -- Proxmox Support Team Tue, 11 Dec 2012 11:17:47 +0100 qemu-server (2.0-70) unstable; urgency=low * fix version parser for qemu 1.3 * add new cpu types: SandyBridge Haswell Opteron_G4 Opteron_G5 * remove rhel specific cpu types: cpu64-rhel6 cpu64-rhel5 -- Proxmox Support Team Fri, 23 Nov 2012 07:45:19 +0100 qemu-server (2.0-69) unstable; urgency=low * fix ballon monitor command (use bytes instead of MBs) -- Proxmox Support Team Fri, 16 Nov 2012 06:13:46 +0100 qemu-server (2.0-68) unstable; urgency=low * vzdump: store drive in correct order (sort) to avoid confusion -- Proxmox Support Team Mon, 05 Nov 2012 06:25:33 +0100 qemu-server (2.0-67) unstable; urgency=low * fix allocation size in qmrestore -- Proxmox Support Team Sat, 03 Nov 2012 07:54:06 +0100 qemu-server (2.0-66) unstable; urgency=low * vzdump: restore sata drives correctly -- Proxmox Support Team Fri, 02 Nov 2012 07:43:16 +0100 qemu-server (2.0-65) unstable; urgency=low * remove hardcoded blowfish cipher -- Proxmox Support Team Wed, 31 Oct 2012 13:58:27 +0100 qemu-server (2.0-64) unstable; urgency=low * fix memory leak in QMP Client (many thanks to Stefan!) -- Proxmox Support Team Mon, 29 Oct 2012 12:14:51 +0100 qemu-server (2.0-63) unstable; urgency=low * fix bug in vmtar -- Proxmox Support Team Thu, 25 Oct 2012 10:00:50 +0200 qemu-server (2.0-62) unstable; urgency=low * vncproxy: wait until vnc port is ready -- Proxmox Support Team Wed, 24 Oct 2012 08:58:52 +0200 qemu-server (2.0-61) unstable; urgency=low * add 'win8' ostype (same defaults as win7 for now) -- Proxmox Support Team Tue, 23 Oct 2012 09:35:27 +0200 qemu-server (2.0-60) unstable; urgency=low * fix bug 258: use bridge mtu for tap interfaces -- Proxmox Support Team Mon, 22 Oct 2012 12:26:42 +0200 qemu-server (2.0-59) unstable; urgency=low * disable vzdump for VM containing snapshots -- Proxmox Support Team Thu, 27 Sep 2012 09:41:19 +0200 qemu-server (2.0-58) unstable; urgency=low * fix bug 251: use new command line syntax -- Proxmox Support Team Wed, 26 Sep 2012 12:43:17 +0200 qemu-server (2.0-57) unstable; urgency=low * fix migrating VMs with snapshots -- Proxmox Support Team Tue, 25 Sep 2012 08:12:37 +0200 qemu-server (2.0-56) unstable; urgency=low * include shanpshot support -- Proxmox Support Team Mon, 24 Sep 2012 15:59:40 +0200 qemu-server (2.0-55) unstable; urgency=low * migrate: disable xbzrle (not stable) -- Proxmox Support Team Fri, 31 Aug 2012 11:04:35 +0200 qemu-server (2.0-54) unstable; urgency=low * use mbps instead of bps * try to make migrate more stable -- Proxmox Support Team Thu, 30 Aug 2012 12:16:39 +0200 qemu-server (2.0-53) unstable; urgency=low * livemigrate : activate xbzrle cache * scsihw: add megasas controller * partly fix bug 247: retry qmp open -- Proxmox Support Team Mon, 27 Aug 2012 13:45:18 +0200 qemu-server (2.0-52) unstable; urgency=low * fix bug 217: stop VM after failed migrate -- Proxmox Support Team Thu, 23 Aug 2012 10:33:57 +0200 qemu-server (2.0-51) unstable; urgency=low * support up to 32 network devices * support up to 16 virtio devices -- Proxmox Support Team Tue, 21 Aug 2012 10:06:39 +0200 qemu-server (2.0-50) unstable; urgency=low * implement 'qm rescan' to update disk sizes and unused disk info -- Proxmox Support Team Mon, 20 Aug 2012 09:22:29 +0200 qemu-server (2.0-49) unstable; urgency=low * fix bug 242: re-add old monitor code -- Proxmox Support Team Fri, 17 Aug 2012 10:33:37 +0200 qemu-server (2.0-48) unstable; urgency=low * add size hint to drive options * new 'qm resize' command -- Proxmox Support Team Tue, 14 Aug 2012 06:55:39 +0200 qemu-server (2.0-47) unstable; urgency=low * implement virtio-scsi-pci controller -- Proxmox Support Team Wed, 01 Aug 2012 08:57:52 +0200 qemu-server (2.0-46) unstable; urgency=low * bug fix: allow to set devices directly (-ide1 /dev/XYZ) -- Proxmox Support Team Fri, 27 Jul 2012 11:59:14 +0200 qemu-server (2.0-45) unstable; urgency=low * migrate: only scan available storages -- Proxmox Support Team Mon, 16 Jul 2012 10:18:22 +0200 qemu-server (2.0-44) unstable; urgency=low * migrate: only scan necessary storage for unused volumes -- Proxmox Support Team Mon, 16 Jul 2012 06:59:23 +0200 qemu-server (2.0-43) unstable; urgency=low * use new QMPClient code -- Proxmox Support Team Fri, 13 Jul 2012 07:05:28 +0200 qemu-server (2.0-42) unstable; urgency=low * fix pool permission checks on create -- Proxmox Support Team Wed, 30 May 2012 10:13:11 +0200 qemu-server (2.0-41) unstable; urgency=low * more fixes for newer pve-storage versions -- Proxmox Support Team Thu, 24 May 2012 07:24:00 +0200 qemu-server (2.0-40) unstable; urgency=low * minor fixes for newer pve-storage versions -- Proxmox Support Team Wed, 23 May 2012 07:23:31 +0200 qemu-server (2.0-39) unstable; urgency=low * new startup option to define startup order. * do not start VMs with /etc/init.d/qemu-server. This is now done with /etc/init.d/pve-manager * qm: removed startall/stopall commands -- Proxmox Support Team Thu, 19 Apr 2012 14:26:04 +0200 qemu-server (2.0-38) unstable; urgency=low * add directsync cache mode * fix bug #147: allow to set migrate_downtime to 0 -- Proxmox Support Team Wed, 11 Apr 2012 08:51:56 +0200 qemu-server (2.0-37) unstable; urgency=low * fix bug in storage availability check (migrate) -- Proxmox Support Team Sat, 07 Apr 2012 08:25:59 +0200 qemu-server (2.0-36) unstable; urgency=low * use '-no-kvm-pit-reinjection -no-hpet' for win7 and w2k8 guests. -- Proxmox Support Team Thu, 05 Apr 2012 12:34:00 +0200 qemu-server (2.0-35) unstable; urgency=low * fix bug #134: allow to pass file names to qmrestore and 'qm set' -- Proxmox Support Team Mon, 02 Apr 2012 10:51:41 +0200 qemu-server (2.0-34) unstable; urgency=low * fix bug #12: check storage availability early (migrate) -- Proxmox Support Team Fri, 30 Mar 2012 09:12:46 +0200 qemu-server (2.0-33) unstable; urgency=low * fix bug #121: activate volumes correctly -- Proxmox Support Team Thu, 29 Mar 2012 11:09:36 +0200 qemu-server (2.0-32) unstable; urgency=low * fix usb host syntax (correctly pass hexadecimal numbers with prefix '0x' to kvm) * document rerror/werror options -- Proxmox Support Team Thu, 29 Mar 2012 07:11:51 +0200 qemu-server (2.0-31) unstable; urgency=low * use new network setup code from PVE::Network (libpve-common-perl) -- Proxmox Support Team Wed, 28 Mar 2012 10:39:12 +0200 qemu-server (2.0-30) unstable; urgency=low * fix ha migration -- Proxmox Support Team Tue, 27 Mar 2012 12:20:33 +0200 qemu-server (2.0-29) unstable; urgency=low * fix bug #97: execute 'clusvcadm' commands for HA managed VMs -- Proxmox Support Team Tue, 27 Mar 2012 10:36:23 +0200 qemu-server (2.0-28) unstable; urgency=low * use Digest::SHA instead of Digest::SHA1 -- Proxmox Support Team Tue, 20 Mar 2012 12:24:01 +0100 qemu-server (2.0-27) unstable; urgency=low * make startall wait up to 10 seconds for quorum -- Proxmox Support Team Tue, 20 Mar 2012 09:32:53 +0100 qemu-server (2.0-26) unstable; urgency=low * fix bug 109: use scsi inquiry to test if we can use the scsi-block driver. -- Proxmox Support Team Mon, 19 Mar 2012 10:39:30 +0100 qemu-server (2.0-25) unstable; urgency=low * fix bug 102: remove stale status file on stop -- Proxmox Support Team Thu, 01 Mar 2012 12:53:32 +0100 qemu-server (2.0-24) unstable; urgency=low * save description as comment. -- Proxmox Support Team Thu, 01 Mar 2012 08:12:29 +0100 qemu-server (2.0-23) unstable; urgency=low * fix lvremove call: avoid 'Not a CODE reference' warning -- Proxmox Support Team Thu, 01 Mar 2012 06:36:33 +0100 qemu-server (2.0-22) unstable; urgency=low * revert tablet mice fix - does not work reliable -- Proxmox Support Team Wed, 29 Feb 2012 09:45:30 +0100 qemu-server (2.0-21) unstable; urgency=low * fix tablet mice as default when live migrate -- Proxmox Support Team Wed, 29 Feb 2012 06:51:09 +0100 qemu-server (2.0-20) unstable; urgency=low * fix bug 96: fix vzdump on stopped vm -- Proxmox Support Team Fri, 24 Feb 2012 07:38:45 +0100 qemu-server (2.0-19) unstable; urgency=low * support more CPU models -- Proxmox Support Team Wed, 22 Feb 2012 07:18:53 +0100 qemu-server (2.0-18) unstable; urgency=low * fix cdrom (/dev/cdrom) permission check -- Proxmox Support Team Mon, 20 Feb 2012 07:16:36 +0100 qemu-server (2.0-17) unstable; urgency=low * fix cdrom removal bug -- Proxmox Support Team Wed, 15 Feb 2012 10:48:30 +0100 qemu-server (2.0-16) unstable; urgency=low * ignore -tdf (avoid kvm warning) - this is no longer needed -- Proxmox Support Team Mon, 13 Feb 2012 11:18:01 +0100 qemu-server (2.0-15) unstable; urgency=low * online migration fix: close tunnel later, wait for connection close * fix bug #81: do no deactivate volumes in vzdump stop mode -- Proxmox Support Team Tue, 17 Jan 2012 11:24:56 +0100 qemu-server (2.0-14) unstable; urgency=low * use 'da' instead of 'dk' for Danish keyboard (qemu use that name) -- Proxmox Support Team Mon, 09 Jan 2012 11:24:40 +0100 qemu-server (2.0-13) unstable; urgency=low * load vhost_net module -- Proxmox Support Team Tue, 20 Dec 2011 12:25:22 +0100 qemu-server (2.0-12) unstable; urgency=low * implement forceStop option for shutdown -- Proxmox Support Team Thu, 15 Dec 2011 12:58:00 +0100 qemu-server (2.0-11) unstable; urgency=low * do not use ehci by default -- Proxmox Support Team Thu, 08 Dec 2011 10:26:36 +0100 qemu-server (2.0-10) unstable; urgency=low * set qm exit codes correctly * fix 'qm shutdown ' -- Proxmox Support Team Wed, 30 Nov 2011 09:35:43 +0100 qemu-server (2.0-9) unstable; urgency=low * fix 'qm stopall' -- Proxmox Support Team Tue, 29 Nov 2011 11:14:01 +0100 qemu-server (2.0-8) unstable; urgency=low * be more careful when removing snapshots * do not call check_lock() for sendkey * try to detect errors before starting the background task -- Proxmox Support Team Tue, 29 Nov 2011 08:08:44 +0100 qemu-server (2.0-7) unstable; urgency=low * activate/deactivate LVs more carefully * avoid syslog whenever possible * code cleanups -- Proxmox Support Team Fri, 25 Nov 2011 08:08:04 +0100 qemu-server (2.0-6) unstable; urgency=low * set correct migrate speed -- Proxmox Support Team Wed, 23 Nov 2011 09:12:12 +0100 qemu-server (2.0-5) unstable; urgency=low * fix vzdump stop mode -- Proxmox Support Team Mon, 21 Nov 2011 06:38:03 +0100 qemu-server (2.0-4) unstable; urgency=low * bump version -- Proxmox Support Team Sat, 19 Nov 2011 09:54:19 +0100 qemu-server (2.0-3) unstable; urgency=low * implement monitor API * implement qmrestore * fix vzdump plugin for 2.0 -- Proxmox Support Team Wed, 09 Nov 2011 11:35:58 +0100 qemu-server (2.0-2) unstable; urgency=low * cleanups -- Proxmox Support Team Wed, 05 Oct 2011 10:15:37 +0200 qemu-server (2.0-1) unstable; urgency=low * see Changelog for details -- Proxmox Support Team Thu, 26 Aug 2010 13:48:12 +0200 qemu-server (1.1-18) unstable; urgency=low * small bug fix im qmigrate -- Proxmox Support Team Fri, 20 Aug 2010 08:05:21 +0200 qemu-server (1.1-17) unstable; urgency=low * carefully catch write errors -- Proxmox Support Team Mon, 19 Jul 2010 09:00:48 +0200 qemu-server (1.1-16) unstable; urgency=low * add rerror/werror options (patch from l.mierzwa) -- Proxmox Support Team Tue, 29 Jun 2010 08:49:00 +0200 qemu-server (1.1-15) unstable; urgency=low * fix vmtar bug (endless growing archive) -- Proxmox Support Team Fri, 25 Jun 2010 12:22:17 +0200 qemu-server (1.1-14) unstable; urgency=low * correct order of config option (prevent virtio reordering) -- Proxmox Support Team Wed, 28 Apr 2010 09:05:15 +0200 qemu-server (1.1-13) unstable; urgency=low * allow vlan1-to vlan4094 (Since 802.1q allows VLAN identifiers up to 4094). -- Proxmox Support Team Wed, 21 Apr 2010 10:19:37 +0200 qemu-server (1.1-12) unstable; urgency=low * minor fixes for new qemu-kvm 0.12.3 -- Proxmox Support Team Fri, 16 Apr 2010 12:01:17 +0200 qemu-server (1.1-11) unstable; urgency=low * experimental support for pci pass-through (option 'hostpci') -- Proxmox Support Team Fri, 08 Jan 2010 13:03:44 +0100 qemu-server (1.1-10) unstable; urgency=low * add compatibility code for older kvm versions (0.9) * only use fairsched when the kernel has openvz support -- Proxmox Support Team Fri, 04 Dec 2009 15:17:18 +0100 qemu-server (1.1-9) unstable; urgency=low * always display boot menu (Press F12...) -- Proxmox Support Team Wed, 28 Oct 2009 10:28:23 +0100 qemu-server (1.1-8) unstable; urgency=low * fix 'stopall' timeout -- Proxmox Support Team Fri, 23 Oct 2009 12:55:23 +0200 qemu-server (1.1-7) unstable; urgency=low * do not set fairsched --id when using virtio -- Proxmox Support Team Thu, 22 Oct 2009 11:57:57 +0200 qemu-server (1.1-6) unstable; urgency=low * disable fairsched when option 'cpuunits' is set to 0 (zero) * disable fairsched when the VM uses virtio devices. -- Proxmox Support Team Thu, 15 Oct 2009 15:06:48 +0200 qemu-server (1.1-5) unstable; urgency=low * suppress syslog when setting migrate downtime/speed -- Proxmox Support Team Tue, 06 Oct 2009 10:10:55 +0200 qemu-server (1.1-4) unstable; urgency=low * depend on stable pve-qemu-kvm * new migrate_speed and migrate_downtime settings -- Proxmox Support Team Mon, 28 Sep 2009 11:18:08 +0200 qemu-server (1.1-3) unstable; urgency=low * support up to 1000 vlans -- Proxmox Support Team Fri, 18 Sep 2009 09:54:35 +0200 qemu-server (1.1-2) unstable; urgency=low * introduce new 'sockets' and 'cores' settings -- Proxmox Support Team Fri, 18 Sep 2009 09:54:18 +0200 qemu-server (1.1-1) unstable; urgency=low * use new pve-storage framework * delete unused disk on destroy * fix cache= option (none|writethrough|writeback) * use rtc-td-hack for windows when acpi is enabled -- Proxmox Support Team Thu, 25 Jun 2009 08:49:53 +0200 qemu-server (1.0-14) unstable; urgency=low * add new tablet option (to disable --usbdevice tablet, which generate many interrupts, which is bad when you run many VMs) (Patch was provided by Tomasz Chmielewski) -- Proxmox Support Team Wed, 27 May 2009 12:50:45 +0200 qemu-server (1.0-13) unstable; urgency=low * Conflict with netcat-openbsd -- Proxmox Support Team Wed, 13 May 2009 10:20:54 +0200 qemu-server (1.0-12) unstable; urgency=low * fixes for debian lenny -- Proxmox Support Team Tue, 21 Apr 2009 14:28:42 +0200 qemu-server (1.0-11) unstable; urgency=low * allow white spaces inside args - use normal shell quoting -- Proxmox Support Team Thu, 26 Feb 2009 11:31:36 +0100 qemu-server (1.0-10) unstable; urgency=low * add 'args' option * bug fix for 'lost description' -- Proxmox Support Team Wed, 11 Feb 2009 08:18:29 +0100 qemu-server (1.0-9) unstable; urgency=low * add 'parallel' option * add 'startdate' option * fix manual page -- Proxmox Support Team Mon, 2 Feb 2009 08:53:26 +0100 qemu-server (1.0-8) unstable; urgency=low * add 'serial' option -- Proxmox Support Team Mon, 20 Jan 2009 08:52:24 +0100 qemu-server (1.0-7) unstable; urgency=low * use new syntax for kvm vga option (needed for kvm newer than kvm77) -- Proxmox Support Team Wed, 7 Jan 2009 14:46:09 +0100 qemu-server (1.0-6) unstable; urgency=low * use predefined names for tap devices -- Proxmox Support Team Fri, 19 Dec 2008 13:00:44 +0100 qemu-server (1.0-5) unstable; urgency=low * added host usb device support -- Proxmox Support Team Mon, 17 Nov 2008 11:26:04 +0100 qemu-server (1.0-4) unstable; urgency=low * fix status problem -- Proxmox Support Team Thu, 13 Nov 2008 13:13:43 +0100 qemu-server (1.0-3) unstable; urgency=low * small bug fixes -- Proxmox Support Team Tue, 11 Nov 2008 08:29:23 +0100 qemu-server (1.0-1) unstable; urgency=low * update for kvm-75, support vm migration -- Proxmox Support Team Wed, 22 Oct 2008 11:04:03 +0200 qemu-server (0.9) unstable; urgency=low * initial release -- Proxmox Support Team Mon, 4 Feb 2008 09:10:13 +0100 ================================================ FILE: debian/control ================================================ Source: qemu-server Section: admin Priority: optional Maintainer: Proxmox Support Team Build-Depends: debhelper-compat (= 13), libclass-methodmaker-perl, libglib2.0-dev, libio-multiplex-perl, libjson-c-dev, libnet-dbus-perl, libpve-apiclient-perl, libpve-cluster-perl, libpve-common-perl (>= 9.1.9), libpve-guest-common-perl (>= 5.2.2), libpve-network-perl, libpve-storage-perl (>= 9.0.16), libtest-mockmodule-perl, liburi-perl, libuuid-perl, lintian, perl, pkgconf, pve-cluster, pve-doc-generator (>= 6.2-5), pve-edk2-firmware-ovmf (>= 4.2025.05-2), pve-firewall, pve-ha-manager , pve-qemu-kvm (>= 10.1~), Standards-Version: 4.5.1 Homepage: https://www.proxmox.com Package: qemu-server Architecture: any Depends: conntrack, dbus, genisoimage, libclass-methodmaker-perl, libio-multiplex-perl, libjson-perl, libjson-xs-perl, libnet-dbus-perl, libnet-ssleay-perl, libpve-access-control (>= 9.0.7~), libpve-apiclient-perl, libpve-cluster-perl, libpve-common-perl (>= 9.1.11~), libpve-guest-common-perl (>= 5.2.2), libpve-storage-perl (>= 9.0.16), libterm-readline-gnu-perl, liburi-perl, libuuid-perl, perl (>= 5.10.0-19), proxmox-termproxy (>= 2.1.0), proxmox-websocket-tunnel, pve-cluster, # TODO: make legacy edk2 optional (suggests) for PVE 9 and warn explicitly about it pve-edk2-firmware-legacy | pve-edk2-firmware (<< 4~), pve-edk2-firmware-ovmf (>= 4.2025.05-2), pve-firewall (>= 6.0.3), pve-ha-manager (>= 5.0.3), pve-qemu-kvm (>= 7.1~), python3-virt-firmware, socat, swtpm, swtpm-tools, ${misc:Depends}, ${perl:Depends}, ${shlibs:Depends}, Recommends: libpve-network-perl (>= 0.8.3), proxmox-backup-file-restore (>= 2.1.9-2), virtiofsd, Suggests: pve-edk2-firmware-aarch64, pve-edk2-firmware-riscv, proxmox-firewall (>= 1.1.1), Breaks: pve-ha-manager (<< 4.0.1), pve-manager (<= 6.0-13), Description: Qemu Server Tools This package contains the Qemu Server tools used by Proxmox VE ================================================ FILE: debian/copyright ================================================ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Files: * Copyright: 2011 - 2026 Proxmox Server Solutions GmbH License: AGPL-3+ License: AGPL-3+ This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. . This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. . You should have received a copy of the GNU Affero General Public License along with this program. If not, see . ================================================ FILE: debian/docs ================================================ debian/SOURCE ================================================ FILE: debian/rules ================================================ #!/usr/bin/make -f # -*- makefile -*- # Sample debian/rules that uses debhelper. # This file was originally written by Joey Hess and Craig Small. # As a special exception, when this file is copied by dh-make into a # dh-make output file, you may use that output file without restriction. # This special exception was added by Craig Small in version 0.37 of dh-make. # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 %: dh $@ ================================================ FILE: debian/source/format ================================================ 3.0 (native) ================================================ FILE: debian/triggers ================================================ activate-noawait pve-api-updates ================================================ FILE: src/Makefile ================================================ all: .PHONY: install install: $(MAKE) -C PVE install $(MAKE) -C bin install $(MAKE) -C qmeventd install $(MAKE) -C query-machine-capabilities install $(MAKE) -C usr install .PHONY: test test: $(MAKE) -C test $(MAKE) -C bin $@ .PHONY: clean clean: $(MAKE) -C test $@ $(MAKE) -C bin $@ .PHONY: distclean distclean: clean ================================================ FILE: src/PVE/API2/Makefile ================================================ DESTDIR= PREFIX=/usr PERLDIR=$(PREFIX)/share/perl5 .PHONY: install install: install -d -m 0755 $(DESTDIR)$(PERLDIR)/PVE/API2 install -D -m 0644 Qemu.pm $(DESTDIR)$(PERLDIR)/PVE/API2/Qemu.pm $(MAKE) -C Qemu install $(MAKE) -C NodeCapabilities install ================================================ FILE: src/PVE/API2/NodeCapabilities/Makefile ================================================ DESTDIR= PREFIX=/usr PERLDIR=$(PREFIX)/share/perl5 SOURCES := Qemu/Migration.pm .PHONY: install install: $(SOURCES) for i in $(SOURCES); do install -D -m 0644 $$i $(DESTDIR)$(PERLDIR)/PVE/API2/NodeCapabilities/$$i; done ================================================ FILE: src/PVE/API2/NodeCapabilities/Qemu/Migration.pm ================================================ package PVE::API2::NodeCapabilities::Qemu::Migration; use strict; use warnings; use JSON; use PVE::JSONSchema qw(get_standard_option); use PVE::RESTHandler; use base qw(PVE::RESTHandler); __PACKAGE__->register_method({ name => 'capabilities', path => '', method => 'GET', proxyto => 'node', description => 'Get node-specific QEMU migration capabilities of the node.' . " Requires the 'Sys.Audit' permission on '/nodes/'.", permissions => { check => ['perm', '/nodes/{node}', ['Sys.Audit']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), }, }, returns => { type => 'object', additionalProperties => 0, properties => { 'has-dbus-vmstate' => { type => 'boolean', description => 'Whether the host supports live-migrating additional' . ' VM state via the dbus-vmstate helper.', }, }, }, code => sub { return { 'has-dbus-vmstate' => -f '/usr/libexec/qemu-server/dbus-vmstate' ? JSON::true : JSON::false, }; }, }); 1; ================================================ FILE: src/PVE/API2/Qemu/Agent.pm ================================================ package PVE::API2::Qemu::Agent; use strict; use warnings; use JSON; use MIME::Base64 qw(encode_base64 decode_base64); use PVE::JSONSchema qw(get_standard_option); use PVE::RESTHandler; use PVE::QemuConfig; use PVE::QemuServer; use PVE::QemuServer::Agent qw(agent_cmd check_agent_error); use PVE::QemuServer::Monitor qw(mon_cmd); use base qw(PVE::RESTHandler); # max size for file-read over the api my $MAX_READ_SIZE = 16 * 1024 * 1024; # 16 MiB # list of commands # will generate one api endpoint per command # needs a 'method' property and a 'perms' property my $guest_agent_commands = { 'ping' => { method => 'POST', perms => 'VM.GuestAgent.Audit', }, 'get-time' => { method => 'GET', perms => 'VM.GuestAgent.Audit', }, 'info' => { method => 'GET', perms => 'VM.GuestAgent.Audit', }, 'fsfreeze-status' => { method => 'POST', perms => { check => [ 'perm', '/vms/{vmid}', [ 'VM.GuestAgent.Audit', 'VM.GuestAgent.FileSystemMgmt', 'VM.GuestAgent.Unrestricted', ], any => 1, ], }, }, 'fsfreeze-freeze' => { method => 'POST', perms => 'VM.GuestAgent.FileSystemMgmt', }, 'fsfreeze-thaw' => { method => 'POST', perms => 'VM.GuestAgent.FileSystemMgmt', }, 'fstrim' => { method => 'POST', perms => 'VM.GuestAgent.FileSystemMgmt', }, 'network-get-interfaces' => { method => 'GET', perms => 'VM.GuestAgent.Audit', }, 'get-vcpus' => { method => 'GET', perms => 'VM.GuestAgent.Audit', }, 'get-fsinfo' => { method => 'GET', perms => 'VM.GuestAgent.Audit', }, 'get-memory-blocks' => { method => 'GET', perms => 'VM.GuestAgent.Audit', }, 'get-memory-block-info' => { method => 'GET', perms => 'VM.GuestAgent.Audit', }, 'suspend-hybrid' => { method => 'POST', perms => 'VM.PowerMgmt', }, 'suspend-ram' => { method => 'POST', perms => 'VM.PowerMgmt', }, 'suspend-disk' => { method => 'POST', perms => 'VM.PowerMgmt', }, 'shutdown' => { method => 'POST', perms => 'VM.PowerMgmt', }, # added since qemu 2.9 'get-host-name' => { method => 'GET', perms => 'VM.GuestAgent.Audit', }, 'get-osinfo' => { method => 'GET', perms => 'VM.GuestAgent.Audit', }, 'get-users' => { method => 'GET', perms => 'VM.GuestAgent.Audit', }, 'get-timezone' => { method => 'GET', perms => 'VM.GuestAgent.Audit', }, }; __PACKAGE__->register_method({ name => 'index', path => '', proxyto => 'node', method => 'GET', description => "QEMU Guest Agent command index.", permissions => { user => 'all', }, parameters => { additionalProperties => 1, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), }, }, returns => { type => 'array', items => { type => "object", properties => {}, }, links => [{ rel => 'child', href => '{name}' }], description => "Returns the list of QEMU Guest Agent commands", }, code => sub { my ($param) = @_; my $result = []; my $cmds = [keys %$guest_agent_commands]; push @$cmds, qw( exec exec-status file-read file-write set-user-password ); for my $cmd (sort @$cmds) { push @$result, { name => $cmd }; } return $result; }, }); sub register_command { my ($class, $command, $method, $perm) = @_; die "no method given\n" if !$method; die "no command given\n" if !defined($command); my $permission; if (ref($perm) eq 'HASH') { $permission = $perm; } else { die "internal error: missing permission for $command" if !$perm; $permission = { check => ['perm', '/vms/{vmid}', [$perm, 'VM.GuestAgent.Unrestricted'], any => 1], }; } my $parameters = { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running, }, ), command => { type => 'string', description => "The QGA command.", enum => [sort keys %$guest_agent_commands], }, }, }; my $description = "Execute QEMU Guest Agent commands."; my $name = 'agent'; if ($command ne '') { $description = "Execute $command."; $name = $command; delete $parameters->{properties}->{command}; } __PACKAGE__->register_method({ name => $name, path => $command, method => $method, protected => 1, proxyto => 'node', description => $description, permissions => $permission, parameters => $parameters, returns => { type => 'object', description => "Returns an object with a single `result` property.", }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; my $conf = PVE::QemuConfig->load_config($vmid); # check if VM exists PVE::QemuServer::Agent::assert_agent_available($vmid, $conf); my $cmd = $param->{command} // $command; my $res = mon_cmd($vmid, "guest-$cmd"); return { result => $res }; }, }); } # old {vmid}/agent POST endpoint, here for compatibility __PACKAGE__->register_command('', 'POST', 'VM.GuestAgent.Unrestricted'); for my $cmd (sort keys %$guest_agent_commands) { my $props = $guest_agent_commands->{$cmd}; __PACKAGE__->register_command($cmd, $props->{method}, $props->{perms}); } # commands with parameters are complicated and we want to register them manually __PACKAGE__->register_method({ name => 'set-user-password', path => 'set-user-password', method => 'POST', protected => 1, proxyto => 'node', description => "Sets the password for the given user to the given password", permissions => { check => ['perm', '/vms/{vmid}', ['VM.GuestAgent.Unrestricted']] }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), username => { type => 'string', description => 'The user to set the password for.', }, password => { type => 'string', description => 'The new password.', minLength => 5, maxLength => 1024, }, crypted => { type => 'boolean', description => 'set to 1 if the password has already been passed through crypt()', optional => 1, default => 0, }, }, }, returns => { type => 'object', description => "Returns an object with a single `result` property.", }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; my $conf = PVE::QemuConfig->load_config($vmid); my $crypted = $param->{crypted} // 0; my $args = { username => $param->{username}, password => encode_base64($param->{password}), crypted => $crypted ? JSON::true : JSON::false, }; my $res = agent_cmd($vmid, $conf, "set-user-password", $args, 'cannot set user password'); return { result => $res }; }, }); __PACKAGE__->register_method({ name => 'exec', path => 'exec', method => 'POST', protected => 1, proxyto => 'node', description => "Executes the given command in the vm via the guest-agent and returns an object with the pid.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.GuestAgent.Unrestricted']] }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), command => { type => 'array', description => 'The command as a list of program + arguments.', items => { type => 'string', description => 'A single part of the program + arguments.', }, }, 'input-data' => { type => 'string', maxLength => 64 * 1024, description => "Data to pass as 'input-data' to the guest. Usually treated as STDIN to 'command'.", optional => 1, }, }, }, returns => { type => 'object', properties => { pid => { type => 'integer', description => "The PID of the process started by the guest-agent.", }, }, }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; my $conf = PVE::QemuConfig->load_config($vmid); my $cmd = $param->{command}; my $res = PVE::QemuServer::Agent::qemu_exec($vmid, $conf, $param->{'input-data'}, $cmd); return $res; }, }); __PACKAGE__->register_method({ name => 'exec-status', path => 'exec-status', method => 'GET', protected => 1, proxyto => 'node', description => "Gets the status of the given pid started by the guest-agent", permissions => { check => ['perm', '/vms/{vmid}', ['VM.GuestAgent.Unrestricted']] }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), pid => { type => 'integer', description => 'The PID to query', }, }, }, returns => { type => 'object', properties => { exited => { type => 'boolean', description => 'Tells if the given command has exited yet.', }, exitcode => { type => 'integer', optional => 1, description => 'process exit code if it was normally terminated.', }, signal => { type => 'integer', optional => 1, description => 'signal number or exception code if the process was abnormally terminated.', }, 'out-data' => { type => 'string', optional => 1, description => 'stdout of the process', }, 'err-data' => { type => 'string', optional => 1, description => 'stderr of the process', }, 'out-truncated' => { type => 'boolean', optional => 1, description => 'true if stdout was not fully captured', }, 'err-truncated' => { type => 'boolean', optional => 1, description => 'true if stderr was not fully captured', }, }, }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; my $conf = PVE::QemuConfig->load_config($vmid); my $pid = int($param->{pid}); my $res = PVE::QemuServer::Agent::qemu_exec_status($vmid, $conf, $pid); return $res; }, }); __PACKAGE__->register_method({ name => 'file-read', path => 'file-read', method => 'GET', protected => 1, proxyto => 'node', description => "Reads the given file via guest agent. Is limited to $MAX_READ_SIZE bytes.", permissions => { check => [ 'perm', '/vms/{vmid}', ['VM.GuestAgent.FileRead', 'VM.GuestAgent.Unrestricted'], any => 1, ], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), count => { type => 'integer', optional => 1, minimum => 1, maximum => $MAX_READ_SIZE, default => $MAX_READ_SIZE, description => "Number of bytes to read.", }, decode => { type => 'boolean', optional => 1, default => 1, description => "Data received from the QEMU Guest-Agent is base64 encoded." . " If this is set to true, the data is decoded." . " Otherwise the content is forwarded with base64 encoding." . " Defaults to true.", }, file => { type => 'string', description => 'The path to the file', }, offset => { type => 'integer', optional => 1, minimum => 0, default => 0, description => "Offset to start reading at", }, }, }, returns => { type => 'object', description => "Returns an object with a `content` property.", properties => { content => { type => 'string', description => "The content of the file, maximum $MAX_READ_SIZE", }, truncated => { type => 'boolean', optional => 1, description => "If set to 1, the read did not reach the end of the file.", }, }, }, code => sub { my ($param) = @_; my $count = $param->{count} // $MAX_READ_SIZE; my $decode = $param->{decode} // 1; my $offset = $param->{offset} // 0; my $vmid = $param->{vmid}; my $conf = PVE::QemuConfig->load_config($vmid); my $qgafh = agent_cmd($vmid, $conf, "file-open", { path => $param->{file} }, "can't open file"); if ($offset > 0) { my $seek = mon_cmd( $vmid, "guest-file-seek", handle => $qgafh, offset => int($offset), whence => 'set', ); check_agent_error($seek, "can't seek to offset position"); } my $bytes_read = 0; my $eof = 0; my $read_size = 1024 * 1024; my $content = ""; while ($bytes_read < $count && !$eof) { my $bytes_left = $count - $bytes_read; my $chunk_size = $bytes_left < $read_size ? $bytes_left : $read_size; my $read = mon_cmd($vmid, "guest-file-read", handle => $qgafh, count => int($chunk_size)); check_agent_error($read, "can't read from file"); my $chunk = $read->{'buf-b64'}; $chunk = decode_base64($chunk) if $decode; $content .= $chunk; $bytes_read += $read->{count}; $eof = $read->{eof} // 0; } my $res = mon_cmd($vmid, "guest-file-close", handle => $qgafh); check_agent_error($res, "can't close file", 1); my $result = { content => $content, 'bytes-read' => $bytes_read, }; if (!$eof) { if (!defined($param->{count})) { warn "agent file-read: reached maximum read size: $MAX_READ_SIZE bytes." . " output might be truncated.\n"; } $result->{truncated} = 1; } return $result; }, }); __PACKAGE__->register_method({ name => 'file-write', path => 'file-write', method => 'POST', protected => 1, proxyto => 'node', description => "Writes the given file via guest agent.", permissions => { check => [ 'perm', '/vms/{vmid}', ['VM.GuestAgent.FileWrite', 'VM.GuestAgent.Unrestricted'], any => 1, ], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), file => { type => 'string', description => 'The path to the file.', }, content => { type => 'string', maxLength => 60 * 1024, # 60k, smaller than our 64k POST limit description => "The content to write into the file.", }, encode => { type => 'boolean', description => "If set, the content will be encoded as base64 (required by QEMU)." . "Otherwise the content needs to be encoded beforehand - defaults to true.", optional => 1, default => 1, }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; my $conf = PVE::QemuConfig->load_config($vmid); my $buf = ($param->{encode} // 1) ? encode_base64($param->{content}) : $param->{content}; my $qgafh = agent_cmd( $vmid, $conf, "file-open", { path => $param->{file}, mode => 'wb' }, "can't open file", ); agent_cmd( $vmid, $conf, "file-write", { handle => $qgafh, 'buf-b64' => $buf }, "can't write to file", ); agent_cmd($vmid, $conf, "file-close", { handle => $qgafh }, "can't close file"); return; }, }); 1; ================================================ FILE: src/PVE/API2/Qemu/CPU.pm ================================================ package PVE::API2::Qemu::CPU; use strict; use warnings; use PVE::JSONSchema qw(get_standard_option); use PVE::RPCEnvironment; use PVE::RESTHandler; use PVE::Tools qw(extract_param); use PVE::QemuServer::CPUConfig; use base qw(PVE::RESTHandler); __PACKAGE__->register_method({ name => 'index', path => '', method => 'GET', description => 'List all custom and default CPU models.', permissions => { user => 'all', description => 'Only returns custom models when the current user has' . ' Sys.Audit on /nodes.', }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), arch => get_standard_option('pve-qm-cpu-arch', { optional => 1 }), }, }, returns => { type => 'array', items => { type => 'object', properties => { name => { type => 'string', description => "Name of the CPU model. Identifies it for" . " subsequent API calls. Prefixed with" . " 'custom-' for custom models.", }, custom => { type => 'boolean', description => "True if this is a custom CPU model.", }, vendor => { type => 'string', description => "CPU vendor visible to the guest when this" . " model is selected. Vendor of" . " 'reported-model' in case of custom models.", }, }, }, links => [{ rel => 'child', href => '{name}' }], }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $include_custom = $rpcenv->check($authuser, "/nodes", ['Sys.Audit'], 1); my $arch = extract_param($param, 'arch'); return PVE::QemuServer::CPUConfig::get_cpu_models($include_custom, $arch); }, }); 1; ================================================ FILE: src/PVE/API2/Qemu/CPUFlags.pm ================================================ package PVE::API2::Qemu::CPUFlags; use v5.36; use PVE::JSONSchema qw(get_standard_option); use PVE::RESTHandler; use PVE::Tools qw(extract_param); use PVE::QemuServer::CPUConfig; use base qw(PVE::RESTHandler); __PACKAGE__->register_method({ name => 'index', path => '', method => 'GET', description => 'List of available VM-specific CPU flags.', permissions => { user => 'all' }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), arch => get_standard_option('pve-qm-cpu-arch', { optional => 1 }), }, }, returns => { type => 'array', items => { type => 'object', properties => { name => { type => 'string', description => "Name of the CPU flag.", }, description => { type => 'string', description => "Description of the CPU flag.", }, }, }, }, code => sub { my ($param) = @_; my $arch = extract_param($param, 'arch'); return PVE::QemuServer::CPUConfig::get_supported_cpu_flags($arch); }, }); 1; ================================================ FILE: src/PVE/API2/Qemu/HMPPerms.pm ================================================ package PVE::API2::Qemu::HMPPerms; use strict; use warnings; # List of monitor commands and associated required permission. Listed explicitly to be future-proof. # # Currently permissions are: # 'root' - for root-only commands # 'Sys.Modify' - commands that can be issued with 'Sys.Modify' on '/' # 'none' - no permissions required (i.e. help and info) our $hmp_command_perms = { help => 'none', # show the help '?' => 'none', # short-form of 'help' info => 'none', # show various information about the system state # root-only: backup to arbitrary target file (although currently, not overwriting existing file) backup => 'root', # create a VM backup (VMA format). # root-only: requires the stream source in the backing chain currently, but better be safe block_stream => 'root', # copy data from a backing file into a block device # root-only: allows changing the path a removable medium points to change => 'root', # change a removable medium # root-only: among others, there is a 'file' driver 'chardev-add' => 'root', # add chardev # root-only: among others, there is a 'file' driver (e.g. modify backend for serial device) 'chardev-change' => 'root', # change chardev # root-only: because chardev-add is 'chardev-remove' => 'root', # remove chardev # root-only: after migration SPICE client will attempt to connect to arbitrarily set host client_migrate_info => 'root', # set migration information for remote display # root-only: like '-device' on the commandline device_add => 'root', # add device, like -device on the command line # root-only: because device_add is device_del => 'root', # remove device # root-only: like '-drive' on the commandline drive_add => 'root', # add drive to PCI storage controller # root-only: backup to arbitrary target file drive_backup => 'root', # initiates a point-in-time copy for a device. # root-only: because drive_add is drive_del => 'root', # remove host block device # root-only: mirror to arbitrary target file drive_mirror => 'root', # initiates live storage migration for a device. # root-only: dump guest memory into arbitrary target file 'dump-guest-memory' => 'root', # dump guest memory into file 'filename'. # root-only: dumps into arbitrary target file dumpdtb => 'root', # dump the FDT in dtb format to 'filename' # root-only: starts GDB server on the host gdbserver => 'root', # start gdbserver on given device (default 'tcp::1234'), stop with 'none' # root-only: host information leak gpa2hpa => 'Sys.Modify', # print the host physical address corresponding to a guest physical address # root-only: host information leak gpa2hva => 'Sys.Modify', # print the host virtual address corresponding to a guest physical address # root-only: redirect TCP or UDP connections from host to guest hostfwd_add => 'root', # redirect TCP or UDP connections from host to guest (requires -net user) # root-only: because hostfwd_add is hostfwd_remove => 'root', # remove host-to-guest TCP or UDP redirection # root-only: read from IO adress space (e.g. PCI devices) i => 'Sys.Modify', # I/O port read # root-only: log to arbitrary target file logfile => 'root', # output logs to 'filename' # root-only: no guarantee there are no KVM bugs that could afffect the real CPU mce => 'root', # inject a MCE on the given CPU [and broadcast to other CPUs with -b option] # root-only: allows to save to arbitrary file memsave => 'root', # save to disk virtual memory dump starting at 'addr' of size 'size' # root-only: could specify arbitrary host, also there is 'exec' and 'file' migrations migrate => 'root', # migrate to URI (using -d to not wait for completion) # root-only: allows setting arbitrary URI migrate_incoming => 'root', # Continue an incoming migration from an -incoming defer # root-only: allows setting arbitrary URI migrate_recover => 'root', # Continue a paused incoming postcopy migration # root-only: because nbd_server_start is nbd_server_add => 'root', # export a block device via NBD # root-only: because nbd_server_start is nbd_server_remove => 'root', # remove an export previously exposed via NBD # root-only: start NBD server on the host nbd_server_start => 'root', # serve block devices on the given host and port # root-only: because nbd_server_start is nbd_server_stop => 'root', # stop serving block devices using the NBD protocol # root-only: add host network device netdev_add => 'root', # add host network device # root-only: because netdev_add is netdev_del => 'root', # remove host network device # root-only: no guarantee there are no KVM bugs that could afffect the real CPU nmi => 'root', # inject an NMI # root-only: write to IO adress space (e.g. PCI devices) o => 'root', # I/O port write # root-only: create arbitrary objects, e.g. serial object_add => 'root', # create QOM object # root-only: because object_del is object_del => 'root', # destroy QOM object # root-only: inject error on PCIe devices pcie_aer_inject_error => 'root', # inject pcie aer error # root-only: save to arbitrary file pmemsave => 'root', # save to disk physical memory dump starting at 'addr' of size 'size' # root-only: modify arbitrary object properties 'qom-set' => 'root', # set QOM property. # root-only: because savevm-start is 'savevm-end' => 'root', # Resume VM after snaphot. # root-only: save VM state to arbitrary target file 'savevm-start' => 'root', # Prepare for snapshot and halt VM. Save VM state to statefile. # root-only: dump to arbitrary target file screendump => 'root', # save screen # root-only: allows specifying arbitrary target file snapshot_blkdev => 'root', # initiates a live snapshot of device # root-only: allows inject-nmi watchdog_action => 'root', # change watchdog action # root-only: saves to arbitrary target file wavcapture => 'root', # capture audio to a wave file # root-only: not relevant for Proxmox VE 'xen-event-inject' => 'root', # inject event channel # root-only: not relevant for Proxmox VE 'xen-event-list' => 'root', # list event channel state announce_self => 'Sys.Modify', # Trigger GARP/RARP announcements backup_cancel => 'Sys.Modify', # cancel the current VM backup balloon => 'Sys.Modify', # request VM to change its memory allocation (in MB) block_job_cancel => 'Sys.Modify', # stop an active background block operation block_job_complete => 'Sys.Modify', # stop an active background block operation block_job_pause => 'Sys.Modify', # pause an active background block operation block_job_resume => 'Sys.Modify', # resume a paused background block operation block_job_set_speed => 'Sys.Modify', # set maximum speed for a background block operation block_resize => 'Sys.Modify', # resize a block image block_set_io_throttle => 'Sys.Modify', # change I/O throttle limits for a block drive boot_set => 'Sys.Modify', # define new values for the boot device list calc_dirty_rate => 'Sys.Modify', # start a round of guest dirty rate measurement cancel_vcpu_dirty_limit => 'Sys.Modify', # cancel dirty page rate limit 'chardev-send-break' => 'Sys.Modify', # send a break on chardev closefd => 'Sys.Modify', # close a file descriptor previously passed via SCM rights commit => 'Sys.Modify', # commit changes to the disk images or backing files cont => 'Sys.Modify', # resume emulation c => 'Sys.Modify', # short-form of 'cont' cpu => 'Sys.Modify', # set the default CPU delvm => 'Sys.Modify', # delete a VM snapshot from its tag eject => 'Sys.Modify', # eject a removable medium (use -f to force it) exit_preconfig => 'Sys.Modify', # exit the preconfig state expire_password => 'Sys.Modify', # set spice/vnc password expire-time getfd => 'Sys.Modify', # receive a file descriptor via SCM rights and assign it a name gva2gpa => 'Sys.Modify', # print the guest physical address corresponding to a guest virtual address loadvm => 'Sys.Modify', # restore a VM snapshot from its tag log => 'Sys.Modify', # activate logging of the specified items migrate_cancel => 'Sys.Modify', # cancel the current VM migration migrate_continue => 'Sys.Modify', # Continue migration from the given paused state migrate_pause => 'Sys.Modify', # Pause an ongoing migration (postcopy-only) migrate_set_capability => 'Sys.Modify', # Enable/Disable the usage of a capability for migration migrate_set_parameter => 'Sys.Modify', # Set the parameter for migration migrate_start_postcopy => 'Sys.Modify', # Switch the migration to postcopy mode. mouse_button => 'Sys.Modify', # change mouse button state (1=L, 2=M, 4=R) mouse_move => 'Sys.Modify', # send mouse move events mouse_set => 'Sys.Modify', # set which mouse device receives events 'one-insn-per-tb' => 'Sys.Modify', # run emulation with one guest instruction per translation block print => 'Sys.Modify', # print expression value (use $reg for CPU register access) p => 'Sys.Modify', # alias for 'print' 'qemu-io' => 'Sys.Modify', # run a qemu-io command on a block device # decidedly not root-only even if qom-set ist, because it is just too useful 'qom-get' => 'Sys.Modify', # print QOM property 'qom-list' => 'Sys.Modify', # list QOM properties quit => 'Sys.Modify', # quit the emulator q => 'Sys.Modify', # short-form of 'quit' replay_break => 'Sys.Modify', # set breakpoint at the specified instruction count replay_delete_break => 'Sys.Modify', # remove replay breakpoint replay_seek => 'Sys.Modify', # replay execution to the specified instruction count ringbuf_read => 'Sys.Modify', # Read from a ring buffer character device ringbuf_write => 'Sys.Modify', # Write to a ring buffer character device savevm => 'Sys.Modify', # save a VM snapshot. If no tag is provided, a new snapshot is created sendkey => 'Sys.Modify', # send keys to the VM set_link => 'Sys.Modify', # change the link status of a network adapter set_password => 'Sys.Modify', # set spice/vnc password set_vcpu_dirty_limit => 'Sys.Modify', # set dirty page rate limit snapshot_blkdev_internal => 'Sys.Modify', # take an internal snapshot of device. snapshot_delete_blkdev_internal => 'Sys.Modify', # delete an internal snapshot of device. stopcapture => 'Sys.Modify', # stop capture stop => 'Sys.Modify', # stop emulation s => 'Sys.Modify', # short-form of 'stop' sum => 'Sys.Modify', # compute the checksum of a memory region 'sync-profile' => 'Sys.Modify', # enable, disable or reset synchronization profiling. system_powerdown => 'Sys.Modify', # send system power down event system_reset => 'Sys.Modify', # reset the system system_wakeup => 'Sys.Modify', # wakeup guest from suspend 'trace-event' => 'Sys.Modify', # changes status of a specific trace event x => 'Sys.Modify', # virtual memory dump starting at 'addr' x_colo_lost_heartbeat => 'Sys.Modify', # Tell COLO that heartbeat is lost xp => 'Sys.Modify', # physical memory dump starting at 'addr' }; sub generate_description { my $cmd_by_priv = {}; for my $cmd (sort keys $hmp_command_perms->%*) { my $priv = $hmp_command_perms->{$cmd}; $cmd_by_priv->{$priv} = [] if !exists($cmd_by_priv->{$priv}); push $cmd_by_priv->{$priv}->@*, $cmd; } my $none_cmds = delete($cmd_by_priv->{none}) or die "internal error - no commands for 'none' found"; my $root_only_cmds = delete($cmd_by_priv->{'root'}) or die "internal error no commands for 'root' found"; my $text = ''; $text .= "The following commands do not require any additional privilege: " . join(', ', $none_cmds->@*) . "\n\n"; for my $priv (sort keys $cmd_by_priv->%*) { $text .= "The following commands require '$priv': " . join(', ', $cmd_by_priv->{$priv}->@*) . "\n\n"; } $text .= "The following commands are root-only: " . join(', ', $root_only_cmds->@*) . "\n"; } 1; ================================================ FILE: src/PVE/API2/Qemu/Machine.pm ================================================ package PVE::API2::Qemu::Machine; use strict; use warnings; use JSON; use PVE::JSONSchema qw(get_standard_option); use PVE::RESTHandler; use PVE::Tools qw(extract_param file_get_contents); use PVE::QemuServer::Machine; use PVE::QemuServer::Helpers qw(get_host_arch); use base qw(PVE::RESTHandler); __PACKAGE__->register_method({ name => 'types', path => '', method => 'GET', proxyto => 'node', description => "Get available QEMU/KVM machine types.", permissions => { user => 'all', }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), arch => get_standard_option('pve-qm-cpu-arch', { optional => 1 }), }, }, returns => { type => 'array', items => { type => 'object', additionalProperties => 1, properties => { id => { type => 'string', description => "Full name of machine type and version.", }, type => { type => 'string', enum => ['q35', 'i440fx'], description => "The machine type.", }, version => { type => 'string', description => "The machine version.", }, changes => { type => 'string', optional => 1, description => 'Notable changes of a version, currently only set for +pveX versions.', }, }, }, }, code => sub { my ($param) = @_; my $arch = extract_param($param, 'arch') // get_host_arch(); my $supported_machine_list = eval { my $raw = file_get_contents("/usr/share/kvm/machine-versions-$arch.json"); my $machines = from_json($raw, { utf8 => 1 }); my $pve_machines = []; for my $machine ($machines->@*) { my $pve_machine = PVE::QemuServer::Machine::get_machine_pve_revisions($machine->{version}) or next; for my $pve_revision (sort keys $pve_machine->{revisions}->%*) { my $entry = { id => $machine->{id} . $pve_revision, type => $machine->{type}, version => $machine->{version} . $pve_revision, }; if (defined(my $changes = $pve_machine->{revisions}->{$pve_revision})) { $entry->{changes} = $changes; } push $pve_machines->@*, $entry; } } return [ sort { PVE::QemuServer::Machine::machine_version_cmp($b->{id}, $a->{id}) } ($machines->@*, $pve_machines->@*) ]; # merge & sort }; die "could not load supported machine versions - $@\n" if $@; return $supported_machine_list; }, }); 1; ================================================ FILE: src/PVE/API2/Qemu/Makefile ================================================ DESTDIR= PREFIX=/usr PERLDIR=$(PREFIX)/share/perl5 SOURCES=Agent.pm CPU.pm CPUFlags.pm HMPPerms.pm Machine.pm .PHONY: install install: install -d -m 0755 $(DESTDIR)$(PERLDIR)/PVE/API2/Qemu for i in $(SOURCES); do install -D -m 0644 $$i $(DESTDIR)$(PERLDIR)/PVE/API2/Qemu/$$i; done ================================================ FILE: src/PVE/API2/Qemu.pm ================================================ package PVE::API2::Qemu; use strict; use warnings; use Cwd 'abs_path'; use Fcntl qw(F_GETFD F_SETFD FD_CLOEXEC); use Net::SSLeay; use IO::Socket::IP; use IO::Socket::SSL; use IO::Socket::UNIX; use IPC::Open3; use JSON; use URI::Escape; use Socket qw(SOCK_STREAM); use PVE::APIClient::LWP; use PVE::CGroup; use PVE::Cluster qw (cfs_read_file cfs_write_file); use PVE::RRD; use PVE::SafeSyslog; use PVE::Tools qw(extract_param run_command); use PVE::Exception qw(raise raise_param_exc raise_perm_exc); use PVE::Storage; use PVE::JSONSchema qw(get_standard_option); use PVE::RESTHandler; use PVE::ReplicationConfig; use PVE::GuestHelpers qw(assert_tag_permissions); use PVE::GuestImport; use PVE::QemuConfig; use PVE::QemuServer; use PVE::QemuServer::Agent; use PVE::QemuServer::Blockdev; use PVE::QemuServer::BlockJob; use PVE::QemuServer::Cloudinit; use PVE::QemuServer::CPUConfig; use PVE::QemuServer::Drive qw(checked_volume_format checked_parse_volname); use PVE::QemuServer::Helpers; use PVE::QemuServer::ImportDisk; use PVE::QemuServer::Monitor qw(mon_cmd vm_qmp_peer); use PVE::QemuServer::Machine; use PVE::QemuServer::Memory qw(get_current_memory); use PVE::QemuServer::MetaInfo; use PVE::QemuServer::Network; use PVE::QemuServer::OVMF; use PVE::QemuServer::PCI; use PVE::QemuServer::QMPHelpers; use PVE::QemuServer::RNG; use PVE::QemuServer::RunState; use PVE::QemuServer::USB; use PVE::QemuServer::Virtiofs qw(max_virtiofs); use PVE::QemuServer::DBusVMState; use PVE::QemuMigrate; use PVE::QemuMigrate::Helpers; use PVE::RPCEnvironment; use PVE::AccessControl; use PVE::INotify; use PVE::Network; use PVE::Firewall; use PVE::API2::Firewall::VM; use PVE::API2::Qemu::Agent; use PVE::API2::Qemu::HMPPerms; use PVE::VZDump::Plugin; use PVE::DataCenterConfig; use PVE::ProcFSTools; use PVE::SSHInfo; use PVE::Replication; use PVE::ReplicationState; use PVE::StorageTunnel; use PVE::RESTEnvironment qw(log_warn); use PVE::Ticket; BEGIN { if (!$ENV{PVE_GENERATING_DOCS}) { require PVE::HA::Env::PVE2; import PVE::HA::Env::PVE2; require PVE::HA::Config; import PVE::HA::Config; } } use base qw(PVE::RESTHandler); my $opt_force_description = "Force physical removal. Without this, we simple remove the disk from the config file and create an additional configuration entry called 'unused[n]', which contains the volume ID. Unlink of unused[n] always cause physical removal."; my $resolve_cdrom_alias = sub { my $param = shift; if (my $value = $param->{cdrom}) { $value .= ",media=cdrom" if $value !~ m/media=/; $param->{ide2} = $value; delete $param->{cdrom}; } }; # Used in import-enabled API endpoints. Parses drives using the extended '_with_alloc' schema. my $foreach_volume_with_alloc = sub { my ($param, $func) = @_; for my $opt (sort keys $param->%*) { next if !PVE::QemuServer::is_valid_drivename($opt); my $drive = PVE::QemuServer::Drive::parse_drive($opt, $param->{$opt}, 1); next if !$drive; $func->($opt, $drive); } }; my $check_drive_param = sub { my ($param, $storecfg, $extra_checks) = @_; for my $opt (sort keys $param->%*) { next if !PVE::QemuServer::is_valid_drivename($opt); my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1); raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive; if ($drive->{'import-from'}) { if ($drive->{file} !~ $PVE::QemuServer::Drive::NEW_DISK_RE || $3 != 0) { raise_param_exc({ $opt => "'import-from' requires special syntax - " . "use :0,import-from=", }); } if ($opt eq 'efidisk0') { for my $required (qw(efitype pre-enrolled-keys)) { if (!defined($drive->{$required})) { raise_param_exc({ $opt => "need to specify '$required' when using 'import-from'", }); } } } elsif ($opt eq 'tpmstate0') { raise_param_exc({ $opt => "need to specify 'version' when using 'import-from'" }) if !defined($drive->{version}); } } PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive); my $volid = $drive->{file}; my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); if ( $storeid && $volid !~ $PVE::QemuServer::Drive::NEW_DISK_RE && defined($volname) && $volname ne 'cloudinit' ) { my $vtype = (PVE::Storage::parse_volname($storecfg, $volid))[0]; raise_param_exc({ $opt => "explicit 'media=cdrom' is required for iso images" }) if $vtype eq 'iso' && !(defined($drive->{media}) && $drive->{media} eq 'cdrom'); } $extra_checks->($drive) if $extra_checks; $param->{$opt} = PVE::QemuServer::print_drive($drive, 1); } }; my $check_storage_access = sub { my ($rpcenv, $authuser, $storecfg, $vmid, $settings, $default_storage, $extraction_storage) = @_; $foreach_volume_with_alloc->( $settings, sub { my ($ds, $drive) = @_; my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive); my $volid = $drive->{file}; my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); if ( !$volid || ($volid eq 'none' || $volid eq 'cloudinit' || (defined($volname) && $volname eq 'cloudinit')) ) { # nothing to check } elsif ($isCDROM && ($volid eq 'cdrom')) { $rpcenv->check($authuser, "/", ['Sys.Console']); } elsif (!$isCDROM && ($volid =~ $PVE::QemuServer::Drive::NEW_DISK_RE)) { my $storeid = $2 || $default_storage; die "no storage ID specified (and no default storage)\n" if !$storeid; $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']); my $scfg = PVE::Storage::storage_config($storecfg, $storeid); raise_param_exc({ storage => "storage '$storeid' does not support vm images" }) if !$scfg->{content}->{images}; } else { PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid); if ($storeid) { my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid); raise_param_exc({ $ds => "content type needs to be 'images' or 'iso'" }) if $vtype ne 'images' && $vtype ne 'iso'; } } if (my $src_image = $drive->{'import-from'}) { my $src_vmid; my ($storeid) = PVE::Storage::parse_volume_id($src_image, 1); if ($storeid) { # PVE-managed volume my $scfg = PVE::Storage::storage_config($storecfg, $storeid); (my $vtype, undef, $src_vmid, undef, undef, undef, my $fmt) = checked_parse_volname($storecfg, $src_image); raise_param_exc( { $ds => "$src_image has wrong type '$vtype' - needs to be 'images' or 'import'", }, ) if $vtype ne 'images' && $vtype ne 'import'; if (PVE::QemuServer::Helpers::needs_extraction($vtype, $fmt)) { my $extraction_scfg = defined($extraction_storage) ? PVE::Storage::storage_config($storecfg, $extraction_storage) : $scfg; my $extraction_param = defined($extraction_storage) ? 'import-working-storage' : $ds; my $extraction_id = $extraction_storage // $storeid; if ( !$extraction_scfg->{content}->{images} || !$extraction_scfg->{path} ) { raise_param_exc({ $extraction_param => "import working storage '$extraction_id' does not support" . " 'images' content type or is not file based.", }); } $rpcenv->check( $authuser, "/storage/$extraction_id", ['Datastore.AllocateSpace'], ); } } if ($src_vmid) { # might be actively used by VM and will be copied via clone_disk() $rpcenv->check($authuser, "/vms/${src_vmid}", ['VM.Clone']); } else { PVE::Storage::check_volume_access( $rpcenv, $authuser, $storecfg, $vmid, $src_image, ); } } }, ); $rpcenv->check( $authuser, "/storage/$settings->{vmstatestorage}", ['Datastore.AllocateSpace'], ) if defined($settings->{vmstatestorage}); }; my $check_storage_access_clone = sub { my ($rpcenv, $authuser, $storecfg, $conf, $storage) = @_; my $sharedvm = 1; PVE::QemuConfig->foreach_volume( $conf, sub { my ($ds, $drive) = @_; my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive); my $volid = $drive->{file}; return if !$volid || $volid eq 'none'; if ($isCDROM) { if ($volid eq 'cdrom') { $rpcenv->check($authuser, "/", ['Sys.Console']); } else { # we simply allow access my ($sid, $volname) = PVE::Storage::parse_volume_id($volid); my $scfg = PVE::Storage::storage_config($storecfg, $sid); $sharedvm = 0 if !$scfg->{shared}; } } else { my ($sid, $volname) = PVE::Storage::parse_volume_id($volid); my $scfg = PVE::Storage::storage_config($storecfg, $sid); $sharedvm = 0 if !$scfg->{shared}; $sid = $storage if $storage; $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']); } }, ); $rpcenv->check($authuser, "/storage/$conf->{vmstatestorage}", ['Datastore.AllocateSpace']) if defined($conf->{vmstatestorage}); return $sharedvm; }; my $check_storage_access_migrate = sub { my ($rpcenv, $authuser, $storecfg, $storage, $node) = @_; PVE::Storage::storage_check_enabled($storecfg, $storage, $node); $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']); my $scfg = PVE::Storage::storage_config($storecfg, $storage); die "storage '$storage' does not support vm images\n" if !$scfg->{content}->{images}; }; my $import_from_volid = sub { my ($storecfg, $src_volid, $dest_info, $vollist) = @_; die "could not get size of $src_volid\n" if !PVE::Storage::volume_size_info($storecfg, $src_volid, 10); die "cannot import from cloudinit disk\n" if PVE::QemuServer::Drive::drive_is_cloudinit({ file => $src_volid }); my $src_vmid = (PVE::Storage::parse_volname($storecfg, $src_volid))[2]; my $src_vm_state = sub { my $exists = $src_vmid && PVE::Cluster::get_vmlist()->{ids}->{$src_vmid} ? 1 : 0; my $runs = 0; if ($exists) { eval { PVE::QemuConfig::assert_config_exists_on_node($src_vmid); }; die "owner VM $src_vmid not on local node\n" if $@; $runs = PVE::QemuServer::Helpers::vm_running_locally($src_vmid) || 0; } return ($exists, $runs); }; my ($src_vm_exists, $running) = $src_vm_state->(); die "cannot import from '$src_volid' - full clone feature is not supported\n" if !PVE::Storage::volume_has_feature($storecfg, 'copy', $src_volid, undef, $running); my $clonefn = sub { my ($src_vm_exists_now, $running_now) = $src_vm_state->(); die "owner VM $src_vmid changed state unexpectedly\n" if $src_vm_exists_now != $src_vm_exists || $running_now != $running; my $src_conf = $src_vm_exists_now ? PVE::QemuConfig->load_config($src_vmid) : {}; my $src_drive = { file => $src_volid }; my $src_drivename; PVE::QemuConfig->foreach_volume( $src_conf, sub { my ($ds, $drive) = @_; return if $src_drivename; if ($drive->{file} eq $src_volid) { $src_drive = $drive; $src_drivename = $ds; } }, ); my $source_info = { vmid => $src_vmid, running => $running_now, drivename => $src_drivename, drive => $src_drive, snapname => undef, }; my ($src_storeid) = PVE::Storage::parse_volume_id($src_volid); my $fs_freeze = PVE::QemuServer::Agent::should_fs_freeze($src_conf); return PVE::QemuServer::clone_disk( $storecfg, $source_info, $dest_info, 1, $vollist, undef, undef, $fs_freeze, PVE::Storage::get_bandwidth_limit('clone', [$src_storeid, $dest_info->{storage}]), ); }; my $cloned; if ($running) { $cloned = PVE::QemuConfig->lock_config_full($src_vmid, 30, $clonefn); } elsif ($src_vmid) { $cloned = PVE::QemuConfig->lock_config_shared($src_vmid, 30, $clonefn); } else { $cloned = $clonefn->(); } return $cloned->@{qw(file size)}; }; my sub prohibit_tpm_version_change { my ($old, $new) = @_; return if !$old || !$new; my $old_drive = PVE::QemuServer::parse_drive('tpmstate0', $old); my $new_drive = PVE::QemuServer::parse_drive('tpmstate0', $new); return if $old_drive->{file} ne $new_drive->{file}; my $old_version = $old_drive->{version} // 'v1.2'; my $new_version = $new_drive->{version} // 'v1.2'; die "cannot change TPM state version after creation\n" if $old_version ne $new_version; return; } # Note: $pool is only needed when creating a VM, because pool permissions # are automatically inherited if VM already exists inside a pool. my sub create_disks : prototype($$$$$$$$$$$) { my ( $rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $settings, $default_storage, $is_live_import, $extraction_storage, ) = @_; my $vollist = []; my $res = {}; my $live_import_mapping = {}; my $code = sub { my ($ds, $disk) = @_; my $volid = $disk->{file}; my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); if (!$volid || $volid eq 'none' || $volid eq 'cdrom') { delete $disk->{size}; $res->{$ds} = PVE::QemuServer::print_drive($disk); } elsif (defined($volname) && $volname eq 'cloudinit') { $storeid = $storeid // $default_storage; die "no storage ID specified (and no default storage)\n" if !$storeid; if ( my $ci_key = PVE::QemuConfig->has_cloudinit($conf, $ds) || PVE::QemuConfig->has_cloudinit($conf->{pending} || {}, $ds) || PVE::QemuConfig->has_cloudinit($res, $ds) ) { die "$ds - cloud-init drive is already attached at '$ci_key'\n"; } my $scfg = PVE::Storage::storage_config($storecfg, $storeid); my $name = "vm-$vmid-cloudinit"; my $fmt = undef; if ($scfg->{path}) { $fmt = $disk->{format} // "qcow2"; $name .= ".$fmt"; } else { $fmt = $disk->{format} // "raw"; } # Initial disk created with 4 MB and aligned to 4MB on regeneration my $ci_size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE; my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, $name, $ci_size / 1024); $disk->{file} = $volid; $disk->{media} = 'cdrom'; push @$vollist, $volid; delete $disk->{format}; # no longer needed $res->{$ds} = PVE::QemuServer::print_drive($disk); print "$ds: successfully created disk '$res->{$ds}'\n"; } elsif ($volid =~ $PVE::QemuServer::Drive::NEW_DISK_RE) { my ($storeid, $size) = ($2 || $default_storage, $3); die "no storage ID specified (and no default storage)\n" if !$storeid; $size = PVE::Tools::convert_size($size, 'gb' => 'kb'); # vdisk_alloc uses kb my $live_import = $is_live_import && $ds ne 'efidisk0'; my $needs_creation = 1; if (my $source = delete $disk->{'import-from'}) { my $dst_volid; $needs_creation = $live_import; my ($source_storage, $source_volid) = PVE::Storage::parse_volume_id($source, 1); if ($source_storage) { # PVE-managed volume my ($vtype, $source_format) = (checked_parse_volname($storecfg, $source))[0, 6]; my $needs_extraction = PVE::QemuServer::Helpers::needs_extraction($vtype, $source_format); my $untrusted = $vtype eq 'import' ? 1 : 0; if ($needs_extraction) { print "extracting $source\n"; my $extracted_volid = PVE::GuestImport::extract_disk_from_import_file( $source, $vmid, $extraction_storage, ); print "finished extracting to $extracted_volid\n"; push @$vollist, $extracted_volid; $source = $extracted_volid; $source_format = checked_volume_format($storecfg, $source); my (undef, undef, undef, $parent) = PVE::Storage::volume_size_info($storecfg, $source); die "importing from extracted images with backing file ($parent) not allowed\n" if $parent; } if ($live_import && $ds ne 'efidisk0') { my $path = PVE::Storage::path($storecfg, $source) or die "failed to get a path for '$source'\n"; # check potentially untrusted image file for import vtype $size = PVE::Storage::file_size_info($path, undef, $source_format, $untrusted); die "could not get file size of $path\n" if !$size; $live_import_mapping->{$ds} = { path => $path, format => $source_format, volid => $source, }; $live_import_mapping->{$ds}->{'delete-after-finish'} = $source if $needs_extraction; } else { # check potentially untrusted image file for import vtype if ($untrusted) { my $path = PVE::Storage::path($storecfg, $source); PVE::Storage::file_size_info($path, undef, $source_format, 1); } my $dest_info = { vmid => $vmid, drivename => $ds, storage => $storeid, format => $disk->{format}, }; $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf, $disk) if $ds eq 'efidisk0'; eval { ($dst_volid, $size) = $import_from_volid->($storecfg, $source, $dest_info, $vollist); # remove extracted volumes after importing if ($needs_extraction) { PVE::Storage::vdisk_free($storecfg, $source); print "cleaned up extracted image $source\n"; } @$vollist = grep { $_ ne $source } @$vollist; }; die "cannot import from '$source' - $@" if $@; } } else { $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1); # check potentially untrusted image file! ($size, my $source_format) = PVE::Storage::file_size_info($source, undef, 'auto-detect', 1); die "could not get file size of $source\n" if !$size; if ($live_import && $ds ne 'efidisk0') { $live_import_mapping->{$ds} = { path => $source, format => $source_format, volid => $source, }; } else { (undef, $dst_volid) = PVE::QemuServer::ImportDisk::do_import( $source, $size, $vmid, $storeid, { drive_name => $ds, format => $disk->{format}, 'skip-config-update' => 1, }, ); # change imported disk to a base volume in case the VM is a template $dst_volid = PVE::Storage::vdisk_create_base($storecfg, $dst_volid) if PVE::QemuConfig->is_template($conf); push @$vollist, $dst_volid; } } if ($needs_creation) { $size = PVE::Tools::convert_size($size, 'b' => 'kb'); # vdisk_alloc uses kb } else { $disk->{file} = $dst_volid; $disk->{size} = $size; delete $disk->{format}; # no longer needed $res->{$ds} = PVE::QemuServer::print_drive($disk); } } if ($needs_creation) { my $defformat = PVE::Storage::storage_default_format($storecfg, $storeid); my $fmt = $disk->{format} || $defformat; my $volid; if ($ds eq 'efidisk0') { my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf); my $cvm_type = PVE::QemuServer::CPUConfig::get_cvm_type($conf); die "SEV-SNP uses consolidated read-only firmware and does not require an EFI disk\n" if $cvm_type && $cvm_type eq 'snp'; ($volid, $size) = PVE::QemuServer::OVMF::create_efidisk( $storecfg, $storeid, $vmid, $fmt, $arch, $disk, $smm, $cvm_type, ); } elsif ($ds eq 'tpmstate0') { # A fixed size is used for TPM state volumes $size = PVE::Tools::convert_size( PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE, 'b' => 'kb', ); $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size); } else { $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size); } # change created disk to a base volume in case the VM is a template $volid = PVE::Storage::vdisk_create_base($storecfg, $volid) if PVE::QemuConfig->is_template($conf); push @$vollist, $volid; $disk->{file} = $volid; $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b'); delete $disk->{format}; # no longer needed $res->{$ds} = PVE::QemuServer::print_drive($disk); } print "$ds: successfully created disk '$res->{$ds}'\n"; } else { PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid); if ($storeid) { my ($vtype, $volume_format) = (checked_parse_volname($storecfg, $volid))[0, 6]; die "cannot use volume $volid - content type needs to be 'images' or 'iso'" if $vtype ne 'images' && $vtype ne 'iso'; # TODO PVE 9 - consider disallowing setting an explicit format for managed volumes. if ($disk->{format} && $disk->{format} ne $volume_format) { die "drive '$ds' - volume '$volid' - 'format=$disk->{format}' option different" . " from storage format '$volume_format'\n"; } if (PVE::QemuServer::Drive::drive_is_cloudinit($disk)) { if ( my $ci_key = PVE::QemuConfig->has_cloudinit($conf, $ds) || PVE::QemuConfig->has_cloudinit($conf->{pending} || {}, $ds) || PVE::QemuConfig->has_cloudinit($res, $ds) ) { die "$ds - cloud-init drive is already attached at '$ci_key'\n"; } } } PVE::Storage::activate_volumes($storecfg, [$volid]) if $storeid; my $size = PVE::Storage::volume_size_info($storecfg, $volid); die "volume $volid does not exist\n" if !$size; $disk->{size} = $size; $res->{$ds} = PVE::QemuServer::print_drive($disk); } }; eval { $foreach_volume_with_alloc->($settings, $code); }; # free allocated images on error if (my $err = $@) { syslog('err', "VM $vmid creating disks failed"); foreach my $volid (@$vollist) { eval { PVE::Storage::vdisk_free($storecfg, $volid); }; warn $@ if $@; } die $err; } # don't return empty import mappings $live_import_mapping = undef if !%$live_import_mapping; return ($vollist, $res, $live_import_mapping); } my $check_cpu_model_access = sub { my ($rpcenv, $authuser, $new, $existing) = @_; return if !defined($new->{cpu}); my $cpu = PVE::JSONSchema::check_format('pve-vm-cpu-conf', $new->{cpu}); return if !$cpu || !$cpu->{cputype}; # always allow default my $cputype = $cpu->{cputype}; if ($existing && $existing->{cpu}) { # changing only other settings doesn't require permissions for CPU model my $existingCpu = PVE::JSONSchema::check_format('pve-vm-cpu-conf', $existing->{cpu}); return if $existingCpu->{cputype} eq $cputype; } if (PVE::QemuServer::CPUConfig::is_custom_model($cputype)) { $rpcenv->check($authuser, "/nodes", ['Sys.Audit']); } }; # The top-most snapshot for a FUSE-exported TPM state cannot be removed live, because exporting # unshares the 'resize' permission, which would be required by both 'block-commit' and # 'block-stream'. my sub assert_tpm_snapshot_delete_possible { my ($vmid, $conf, $snap_conf, $snap_name) = @_; return if !$conf->{tpmstate0}; return if !PVE::QemuServer::Helpers::vm_running_locally($vmid); my $drive = PVE::QemuServer::Drive::parse_drive('tpmstate0', $conf->{tpmstate0}); my $volid = $drive->{file}; my $storecfg = PVE::Storage::config(); return if ($conf->{parent} // '') ne $snap_name; # allowed if not top-most snapshot return if !$snap_conf->{tpmstate0}; my $snap_drive = PVE::QemuServer::Drive::parse_drive('tpmstate0', $snap_conf->{tpmstate0}); return if $volid ne $snap_drive->{file}; my $format = PVE::QemuServer::Drive::checked_volume_format($storecfg, $volid); my ($storeid) = PVE::Storage::parse_volume_id($volid, 1); if ($storeid && $format eq 'qcow2') { my $scfg = PVE::Storage::storage_config($storecfg, $storeid); if ($scfg && $scfg->{'snapshot-as-volume-chain'}) { die "top-most snapshot of TPM state '$volid' on storage with 'snapshot-as-volume-chain'" . " cannot be removed while the VM is running.\n"; } } } my $cpuoptions = { 'cores' => 1, 'cpu' => 1, 'runningcpu' => 1, 'cpulimit' => 1, 'cpuunits' => 1, 'numa' => 1, 'smp' => 1, 'sockets' => 1, 'vcpus' => 1, }; my $memoryoptions = { 'memory' => 1, 'balloon' => 1, 'shares' => 1, 'allow-ksm' => 1, }; my $hwtypeoptions = { 'acpi' => 1, 'hotplug' => 1, 'kvm' => 1, 'machine' => 1, 'runningmachine' => 1, 'scsihw' => 1, 'smbios1' => 1, 'tablet' => 1, 'vga' => 1, 'watchdog' => 1, 'audio0' => 1, 'rng0' => 1, }; my $generaloptions = { 'agent' => 1, 'autostart' => 1, 'bios' => 1, 'description' => 1, 'keyboard' => 1, 'localtime' => 1, 'migrate_downtime' => 1, 'migrate_speed' => 1, 'name' => 1, 'onboot' => 1, 'ostype' => 1, 'protection' => 1, 'reboot' => 1, 'startdate' => 1, 'startup' => 1, 'tdf' => 1, 'template' => 1, }; my $vmpoweroptions = { 'freeze' => 1, }; my $diskoptions = { 'boot' => 1, 'bootdisk' => 1, 'vmstatestorage' => 1, }; my $cloudinitoptions = { cicustom => 1, cipassword => 1, citype => 1, ciuser => 1, ciupgrade => 1, nameserver => 1, searchdomain => 1, sshkeys => 1, }; my $check_vm_create_serial_perm = sub { my ($rpcenv, $authuser, $vmid, $pool, $param) = @_; return 1 if $authuser eq 'root@pam'; foreach my $opt (keys %{$param}) { next if $opt !~ m/^serial\d+$/; if ($param->{$opt} eq 'socket') { $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']); } else { die "only root can set '$opt' config for real devices\n"; } } return 1; }; my sub check_usb_perm { my ($rpcenv, $authuser, $vmid, $pool, $opt, $value) = @_; return 1 if $authuser eq 'root@pam'; $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']); my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $value); if ($device->{host}) { if ($device->{host} =~ m/^spice$/i) { # already checked generic permission above } else { die "only root can set '$opt' config for real devices\n"; } } elsif ($device->{mapping}) { $rpcenv->check_full($authuser, "/mapping/usb/$device->{mapping}", ['Mapping.Use']); } else { die "either 'host' or 'mapping' must be set.\n"; } return 1; } my sub check_vm_create_usb_perm { my ($rpcenv, $authuser, $vmid, $pool, $param) = @_; return 1 if $authuser eq 'root@pam'; foreach my $opt (keys %{$param}) { next if $opt !~ m/^usb\d+$/; check_usb_perm($rpcenv, $authuser, $vmid, $pool, $opt, $param->{$opt}); } return 1; } my sub check_hostpci_perm { my ($rpcenv, $authuser, $vmid, $pool, $opt, $value) = @_; return 1 if $authuser eq 'root@pam'; my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $value); if ($device->{host}) { die "only root can set '$opt' config for non-mapped devices\n"; } elsif ($device->{mapping}) { $rpcenv->check_full($authuser, "/mapping/pci/$device->{mapping}", ['Mapping.Use']); $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']); } else { die "either 'host' or 'mapping' must be set.\n"; } return 1; } my sub check_vm_create_hostpci_perm { my ($rpcenv, $authuser, $vmid, $pool, $param) = @_; return 1 if $authuser eq 'root@pam'; foreach my $opt (keys %{$param}) { next if $opt !~ m/^hostpci\d+$/; check_hostpci_perm($rpcenv, $authuser, $vmid, $pool, $opt, $param->{$opt}); } return 1; } my sub check_rng_perm { my ($rpcenv, $authuser, $vmid, $pool, $opt, $value) = @_; return 1 if $authuser eq 'root@pam'; $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']); my $device = PVE::JSONSchema::parse_property_string('pve-qm-rng', $value); if ($device->{source} && $device->{source} eq '/dev/hwrng') { $rpcenv->check_full($authuser, "/mapping/hwrng", ['Mapping.Use']); } return 1; } my sub check_dir_perm { my ($rpcenv, $authuser, $vmid, $pool, $opt, $value) = @_; return 1 if $authuser eq 'root@pam'; $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']); my $virtiofs = PVE::JSONSchema::parse_property_string('pve-qm-virtiofs', $value); $rpcenv->check_full($authuser, "/mapping/dir/$virtiofs->{dirid}", ['Mapping.Use']); return 1; } my sub check_vm_create_dir_perm { my ($rpcenv, $authuser, $vmid, $pool, $param) = @_; return 1 if $authuser eq 'root@pam'; for (my $i = 0; $i < max_virtiofs(); $i++) { my $opt = "virtiofs$i"; next if !$param->{$opt}; check_dir_perm($rpcenv, $authuser, $vmid, $pool, $opt, $param->{$opt}); } return 1; } my $check_vm_modify_config_perm = sub { my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_; return 1 if $authuser eq 'root@pam'; foreach my $opt (@$key_list) { # some checks (e.g., disk, serial port, usb) need to be done somewhere # else, as there the permission can be value dependent next if PVE::QemuServer::is_valid_drivename($opt); next if $opt eq 'cdrom'; next if $opt =~ m/^(?:unused|serial|usb|hostpci|virtiofs)\d+$/; next if $opt eq 'tags'; if ($cpuoptions->{$opt} || $opt =~ m/^numa\d+$/) { $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']); } elsif ($memoryoptions->{$opt}) { $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']); } elsif ($hwtypeoptions->{$opt}) { $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']); } elsif ($generaloptions->{$opt}) { $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']); # special case for startup since it changes host behaviour if ($opt eq 'startup') { $rpcenv->check_full($authuser, "/", ['Sys.Modify']); } } elsif ($vmpoweroptions->{$opt}) { $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.PowerMgmt']); } elsif ($diskoptions->{$opt}) { $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']); } elsif ($opt =~ m/^net\d+$/ || $opt eq 'running-nets-host-mtu') { $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']); } elsif ($cloudinitoptions->{$opt} || $opt =~ m/^ipconfig\d+$/) { $rpcenv->check_vm_perm( $authuser, $vmid, $pool, ['VM.Config.Cloudinit', 'VM.Config.Network'], 1, ); } elsif ($opt eq 'vmstate') { # the user needs Disk and PowerMgmt privileges to change the vmstate # also needs privileges on the storage, that will be checked later $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk', 'VM.PowerMgmt']); } else { # catches args, lock, etc. # new options will be checked here die "only root can set '$opt' config\n"; } } return 1; }; sub assert_scsi_feature_compatibility { my ($opt, $conf, $storecfg, $drive_attributes) = @_; my $drive = PVE::QemuServer::Drive::parse_drive($opt, $drive_attributes, 1); my $machine_type = PVE::QemuServer::Machine::get_vm_machine($conf); my $machine_version = PVE::QemuServer::Machine::extract_version( $machine_type, PVE::QemuServer::Helpers::kvm_user_version(), ); my $drivetype = PVE::QemuServer::Drive::get_scsi_device_type($drive, $storecfg, $machine_version); if ($drivetype ne 'hd' && $drivetype ne 'cd') { if ($drive->{product}) { raise_param_exc({ $opt => "Passing of product information is only supported for 'scsi-hd' and " . "'scsi-cd' devices (e.g. not pass-through).", }); } if ($drive->{vendor}) { raise_param_exc({ $opt => "Passing of vendor information is only supported for 'scsi-hd' and " . "'scsi-cd' devices (e.g. not pass-through).", }); } } } __PACKAGE__->register_method({ name => 'vmlist', path => '', method => 'GET', description => "Virtual machine index (per node).", permissions => { description => "Only list VMs where you have VM.Audit permissions on /vms/.", user => 'all', }, proxyto => 'node', protected => 1, # qemu pid files are only readable by root parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), full => { type => 'boolean', optional => 1, description => "Determine the full status of active VMs.", }, }, }, returns => { type => 'array', items => { type => "object", properties => $PVE::QemuServer::vmstatus_return_properties, }, links => [{ rel => 'child', href => "{vmid}" }], }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $vmstatus = PVE::QemuServer::vmstatus(undef, $param->{full}); my $res = []; foreach my $vmid (keys %$vmstatus) { next if !$rpcenv->check($authuser, "/vms/$vmid", ['VM.Audit'], 1); my $data = $vmstatus->{$vmid}; push @$res, $data; } return $res; }, }); my $classify_restore_archive = sub { my ($storecfg, $archive) = @_; my ($archive_storeid, $archive_volname) = PVE::Storage::parse_volume_id($archive, 1); my $res = {}; if (defined($archive_storeid)) { my $scfg = PVE::Storage::storage_config($storecfg, $archive_storeid); $res->{volid} = $archive; if ($scfg->{type} eq 'pbs') { $res->{type} = 'pbs'; return $res; } if (PVE::Storage::storage_has_feature($storecfg, $archive_storeid, 'backup-provider')) { my $log_function = sub { my ($log_level, $message) = @_; my $prefix = $log_level eq 'err' ? 'ERROR' : uc($log_level); print "$prefix: $message\n"; }; my $backup_provider = PVE::Storage::new_backup_provider( $storecfg, $archive_storeid, $log_function, ); $res->{type} = 'external'; $res->{'backup-provider'} = $backup_provider; return $res; } } my $path = PVE::Storage::abs_filesystem_path($storecfg, $archive); $res->{type} = 'file'; $res->{path} = $path; return $res; }; __PACKAGE__->register_method({ name => 'create_vm', path => '', method => 'POST', description => "Create or restore a virtual machine.", permissions => { description => "You need 'VM.Allocate' permissions on /vms/{vmid} or on the VM pool /pool/{pool}. " . "For restore (option 'archive'), it is enough if the user has 'VM.Backup' permission and the VM already exists. " . "If you create disks you need 'Datastore.AllocateSpace' on any used storage." . "If you use a bridge/vlan, you need 'SDN.Use' on any used bridge/vlan.", user => 'all', # check inside }, protected => 1, proxyto => 'node', parameters => { additionalProperties => 0, properties => PVE::QemuServer::json_config_properties( { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }, ), archive => { description => "The backup archive. Either the file system path to a .tar or .vma file (use '-' to pipe data from stdin) or a proxmox storage backup volume identifier.", type => 'string', optional => 1, maxLength => 255, completion => \&PVE::QemuServer::complete_backup_archives, }, storage => get_standard_option( 'pve-storage-id', { description => "Default storage.", optional => 1, completion => \&PVE::QemuServer::complete_storage, }, ), force => { optional => 1, type => 'boolean', description => "Allow to overwrite existing VM.", requires => 'archive', }, unique => { optional => 1, type => 'boolean', description => "Assign a unique random ethernet address.", requires => 'archive', }, 'live-restore' => { optional => 1, type => 'boolean', description => "Start the VM immediately while importing or restoring in the background.", }, pool => { optional => 1, type => 'string', format => 'pve-poolid', description => "Add the VM to the specified pool.", }, bwlimit => { description => "Override I/O bandwidth limit (in KiB/s).", optional => 1, type => 'integer', minimum => '0', default => 'restore limit from datacenter or storage config', }, start => { optional => 1, type => 'boolean', default => 0, description => "Start VM after it was created successfully.", }, 'ha-managed' => { optional => 1, type => 'boolean', default => 0, description => "Add the VM as a HA resource after it was created.", }, 'import-working-storage' => get_standard_option( 'pve-storage-id', { description => "A file-based storage with 'images' content-type enabled, which" . " is used as an intermediary extraction storage during import. Defaults to" . " the source storage.", optional => 1, completion => \&PVE::QemuServer::complete_storage, }, ), }, 1, # with_disk_alloc ), }, returns => { type => 'string', }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $archive = extract_param($param, 'archive'); my $is_restore = !!$archive; my $bwlimit = extract_param($param, 'bwlimit'); my $force = extract_param($param, 'force'); my $pool = extract_param($param, 'pool'); my $start_after_create = extract_param($param, 'start'); my $ha_managed = extract_param($param, 'ha-managed'); my $storage = extract_param($param, 'storage'); my $unique = extract_param($param, 'unique'); my $live_restore = extract_param($param, 'live-restore'); my $extraction_storage = extract_param($param, 'import-working-storage'); if (defined(my $ssh_keys = $param->{sshkeys})) { $ssh_keys = URI::Escape::uri_unescape($ssh_keys); PVE::Tools::validate_ssh_public_keys($ssh_keys); } $param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits}) if defined($param->{cpuunits}); # clamp value depending on cgroup version PVE::Cluster::check_cfs_quorum(); my $filename = PVE::QemuConfig->config_file($vmid); my $storecfg = PVE::Storage::config(); if (defined($pool)) { $rpcenv->check_pool_exist($pool); } $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']) if defined($storage); $rpcenv->check($authuser, "/", ['Sys.Console']) if $ha_managed; if ($rpcenv->check($authuser, "/vms/$vmid", ['VM.Allocate'], 1)) { # OK } elsif ($pool && $rpcenv->check($authuser, "/pool/$pool", ['VM.Allocate'], 1)) { # OK } elsif ( $archive && $force && (-f $filename) && $rpcenv->check($authuser, "/vms/$vmid", ['VM.Backup'], 1) ) { # OK: user has VM.Backup permissions and wants to restore an existing VM } else { raise_perm_exc(); } if ($archive) { for my $opt (sort keys $param->%*) { if (PVE::QemuServer::Drive::is_valid_drivename($opt)) { raise_param_exc({ $opt => "option conflicts with option 'archive'" }); } } if ($archive eq '-') { die "pipe requires cli environment\n" if $rpcenv->{type} ne 'cli'; $archive = { type => 'pipe' }; } else { PVE::Storage::check_volume_access( $rpcenv, $authuser, $storecfg, $vmid, $archive, 'backup', ); $archive = $classify_restore_archive->($storecfg, $archive); } } if (scalar(keys $param->%*) > 0) { &$resolve_cdrom_alias($param); &$check_storage_access( $rpcenv, $authuser, $storecfg, $vmid, $param, $storage, $extraction_storage, ); &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, $pool, [keys %$param]); &$check_vm_create_serial_perm($rpcenv, $authuser, $vmid, $pool, $param); check_vm_create_usb_perm($rpcenv, $authuser, $vmid, $pool, $param); check_vm_create_hostpci_perm($rpcenv, $authuser, $vmid, $pool, $param); check_rng_perm($rpcenv, $authuser, $vmid, $pool, 'rng0', $param->{rng0}) if $param->{rng0}; check_vm_create_dir_perm($rpcenv, $authuser, $vmid, $pool, $param); PVE::QemuServer::check_bridge_access($rpcenv, $authuser, $param); &$check_cpu_model_access($rpcenv, $authuser, $param); $check_drive_param->($param, $storecfg); PVE::QemuServer::Network::add_random_macs($param); } my $emsg = $is_restore ? "unable to restore VM $vmid -" : "unable to create VM $vmid -"; eval { PVE::QemuConfig->create_and_lock_config($vmid, $force) }; die "$emsg $@" if $@; my $restored_data = 0; my $restorefn = sub { my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_protection($conf, $emsg); die "$emsg vm is running\n" if PVE::QemuServer::check_running($vmid); my $realcmd = sub { my $restore_options = { storage => $storage, pool => $pool, unique => $unique, bwlimit => $bwlimit, live => $live_restore, override_conf => $param, }; if (my $volid = $archive->{volid}) { # best effort, real check is after restoring! my $merged = eval { my $old_conf = PVE::Storage::extract_vzdump_config($storecfg, $volid); PVE::QemuServer::restore_merge_config( "backup/qemu-server/$vmid.conf", $old_conf, $param, ); }; if ($@) { warn "Could not extract backed up config: $@\n"; warn "Skipping early checks!\n"; } else { PVE::QemuServer::check_restore_permissions($rpcenv, $authuser, $merged); } } if (my $backup_provider = $archive->{'backup-provider'}) { PVE::QemuServer::restore_external_archive( $backup_provider, $archive->{volid}, $vmid, $authuser, $restore_options, ); } elsif ($archive->{type} eq 'file' || $archive->{type} eq 'pipe') { die "live-restore is only compatible with backup images from a Proxmox Backup Server\n" if $live_restore; PVE::QemuServer::restore_file_archive( $archive->{path} // '-', $vmid, $authuser, $restore_options, ); } elsif ($archive->{type} eq 'pbs') { PVE::QemuServer::restore_proxmox_backup_archive( $archive->{volid}, $vmid, $authuser, $restore_options, ); } else { die "unknown backup archive type\n"; } $restored_data = 1; my $restored_conf = PVE::QemuConfig->load_config($vmid); # Convert restored VM to template if backup was VM template if (PVE::QemuConfig->is_template($restored_conf)) { warn "Convert to template.\n"; eval { PVE::QemuServer::template_create($vmid, $restored_conf) }; warn $@ if $@; } PVE::QemuServer::Network::create_ifaces_ipams_ips($restored_conf, $vmid) if $unique; }; # ensure no old replication state are exists PVE::ReplicationState::delete_guest_states($vmid); PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd); if ($start_after_create && !$live_restore) { print "Execute autostart\n"; eval { PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node }) }; warn $@ if $@; } if ($ha_managed) { my $resource_exists = PVE::HA::Config::service_is_configured("vm:$vmid"); my $state = $start_after_create || $live_restore ? 'started' : 'stopped'; my $ha_cmd = $resource_exists ? 'set' : 'add'; print ucfirst("$ha_cmd HA resource with request-state '$state'\n"); eval { run_command(['ha-manager', $ha_cmd, "vm:$vmid", '--state', $state]) }; warn $@ if $@; } }; my $createfn = sub { my $live_import_mapping = {}; # ensure no old replication state are exists PVE::ReplicationState::delete_guest_states($vmid); my $realcmd = sub { my $conf = $param; my $arch = PVE::QemuServer::Helpers::get_vm_arch($conf); for my $opt (sort keys $param->%*) { next if $opt !~ m/^scsi\d+$/; assert_scsi_feature_compatibility($opt, $conf, $storecfg, $param->{$opt}); } $conf->{meta} = PVE::QemuServer::MetaInfo::new_meta_info_string(); my $vollist = []; eval { ($vollist, my $created_opts, $live_import_mapping) = create_disks( $rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $param, $storage, $live_restore, $extraction_storage, ); $conf->{$_} = $created_opts->{$_} for keys $created_opts->%*; if (!$conf->{boot}) { my $devs = PVE::QemuServer::get_default_bootdevices($conf); $conf->{boot} = PVE::QemuServer::print_bootorder($devs); } my $vga = PVE::QemuServer::parse_vga($conf->{vga}); PVE::QemuServer::assert_clipboard_config($vga); # auto generate uuid if user did not specify smbios1 option if (!$conf->{smbios1}) { $conf->{smbios1} = PVE::QemuServer::generate_smbios1_uuid(); } if ( (!defined($conf->{vmgenid}) || $conf->{vmgenid} eq '1') && $arch ne 'aarch64' ) { $conf->{vmgenid} = PVE::QemuServer::generate_uuid(); } # always pin Windows' machine version on create, they get confused too easily my $machine_string = PVE::QemuServer::Machine::check_and_pin_machine_string( $conf->{machine}, $conf->{ostype}, $arch, ); $conf->{machine} = $machine_string if $machine_string; $conf->{lock} = 'import' if $live_import_mapping; PVE::QemuConfig->write_config($vmid, $conf); }; my $err = $@; if ($err) { foreach my $volid (@$vollist) { eval { PVE::Storage::vdisk_free($storecfg, $volid); }; warn $@ if $@; } die "$emsg $err"; } PVE::AccessControl::add_vm_to_pool($vmid, $pool) if $pool; PVE::QemuServer::Network::create_ifaces_ipams_ips($conf, $vmid); }; PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd); if ($ha_managed) { print "Add as HA resource\n"; my $state = $start_after_create ? 'started' : 'stopped'; my $cmd = ['ha-manager', 'add', "vm:$vmid", '--state', $state]; eval { run_command($cmd) }; warn $@ if $@; } if ($start_after_create && !$live_restore) { print "Execute autostart\n"; eval { PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node }) }; warn $@ if $@; return; } else { return $live_import_mapping; } }; my ($code, $worker_name); if ($is_restore) { $worker_name = 'qmrestore'; $code = sub { eval { $restorefn->() }; if (my $err = $@) { eval { PVE::QemuConfig->remove_lock($vmid, 'create') }; warn $@ if $@; if ($restored_data) { warn "error after data was restored, VM disks should be OK but config may " . "require adaptions. VM $vmid state is NOT cleaned up.\n"; } else { warn "error before or during data restore, some or all disks were not " . "completely restored. VM $vmid state is NOT cleaned up.\n"; } die $err; } }; } else { $worker_name = 'qmcreate'; $code = sub { # If a live import was requested the create function returns # the mapping for the startup. my $live_import_mapping = eval { $createfn->() }; if (my $err = $@) { eval { my $conffile = PVE::QemuConfig->config_file($vmid); unlink($conffile) or die "failed to remove config file: $!\n"; }; warn $@ if $@; die $err; } if ($live_import_mapping) { my $import_options = { bwlimit => $bwlimit, live => 1, }; my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuServer::live_import_from_files( $live_import_mapping, $vmid, $conf, $import_options, ); } }; } return $rpcenv->fork_worker($worker_name, $vmid, $authuser, $code); }, }); __PACKAGE__->register_method({ name => 'vmdiridx', path => '{vmid}', method => 'GET', proxyto => 'node', description => "Directory index", permissions => { user => 'all', }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), }, }, returns => { type => 'array', items => { type => "object", properties => { subdir => { type => 'string' }, }, }, links => [{ rel => 'child', href => "{subdir}" }], }, code => sub { my ($param) = @_; my $res = [ { subdir => 'config' }, { subdir => 'cloudinit' }, { subdir => 'pending' }, { subdir => 'status' }, { subdir => 'unlink' }, { subdir => 'vncproxy' }, { subdir => 'termproxy' }, { subdir => 'migrate' }, { subdir => 'resize' }, { subdir => 'move' }, { subdir => 'rrd' }, { subdir => 'rrddata' }, { subdir => 'monitor' }, { subdir => 'agent' }, { subdir => 'snapshot' }, { subdir => 'spiceproxy' }, { subdir => 'sendkey' }, { subdir => 'firewall' }, { subdir => 'mtunnel' }, { subdir => 'remote_migrate' }, ]; return $res; }, }); __PACKAGE__->register_method({ subclass => "PVE::API2::Firewall::VM", path => '{vmid}/firewall', }); __PACKAGE__->register_method({ subclass => "PVE::API2::Qemu::Agent", path => '{vmid}/agent', }); __PACKAGE__->register_method({ name => 'rrd', path => '{vmid}/rrd', method => 'GET', protected => 1, # fixme: can we avoid that? permissions => { check => ['perm', '/vms/{vmid}', ['VM.Audit']], }, description => "Read VM RRD statistics (returns PNG)", parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), timeframe => { description => "Specify the time frame you are interested in.", type => 'string', enum => ['hour', 'day', 'week', 'month', 'year'], }, ds => { description => "The list of datasources you want to display.", type => 'string', format => 'pve-configid-list', }, cf => { description => "The RRD consolidation function", type => 'string', enum => ['AVERAGE', 'MAX'], optional => 1, }, }, }, returns => { type => "object", properties => { filename => { type => 'string' }, }, }, code => sub { my ($param) = @_; return PVE::RRD::create_rrd_graph( "pve-vm-9.0/$param->{vmid}", $param->{timeframe}, $param->{ds}, $param->{cf}, ); }, }); __PACKAGE__->register_method({ name => 'rrddata', path => '{vmid}/rrddata', method => 'GET', protected => 1, # fixme: can we avoid that? permissions => { check => ['perm', '/vms/{vmid}', ['VM.Audit']], }, description => "Read VM RRD statistics", parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), timeframe => { description => "Specify the time frame you are interested in.", type => 'string', enum => ['hour', 'day', 'week', 'month', 'year'], }, cf => { description => "The RRD consolidation function", type => 'string', enum => ['AVERAGE', 'MAX'], optional => 1, }, }, }, returns => { type => "array", items => { type => "object", properties => {}, }, }, code => sub { my ($param) = @_; return PVE::RRD::create_rrd_data( "pve-vm-9.0/$param->{vmid}", $param->{timeframe}, $param->{cf}, ); }, }); __PACKAGE__->register_method({ name => 'vm_config', path => '{vmid}/config', method => 'GET', proxyto => 'node', description => "Get the virtual machine configuration with pending configuration " . "changes applied. Set the 'current' parameter to get the current configuration instead.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Audit']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), current => { description => "Get current values (instead of pending values).", optional => 1, default => 0, type => 'boolean', }, snapshot => get_standard_option( 'pve-snapshot-name', { description => "Fetch config values from given snapshot.", optional => 1, completion => sub { my ($cmd, $pname, $cur, $args) = @_; PVE::QemuConfig->snapshot_list($args->[0]); }, }, ), }, }, returns => { description => "The VM configuration.", type => "object", properties => PVE::QemuServer::json_config_properties( { digest => { type => 'string', description => 'SHA1 digest of configuration file. This can be used to prevent concurrent modifications.', }, }, 0, 1, ), }, code => sub { my ($param) = @_; raise_param_exc({ snapshot => "cannot use 'snapshot' parameter with 'current'", current => "cannot use 'snapshot' parameter with 'current'", }) if ($param->{snapshot} && $param->{current}); my $conf; if ($param->{snapshot}) { $conf = PVE::QemuConfig->load_snapshot_config($param->{vmid}, $param->{snapshot}); } else { $conf = PVE::QemuConfig->load_current_config($param->{vmid}, $param->{current}); } $conf->{cipassword} = '**********' if $conf->{cipassword}; return $conf; }, }); __PACKAGE__->register_method({ name => 'vm_pending', path => '{vmid}/pending', method => 'GET', proxyto => 'node', description => "Get the virtual machine configuration with both current and pending values.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Audit']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), }, }, returns => { type => "array", items => { type => "object", properties => { key => { description => "Configuration option name.", type => 'string', }, value => { description => "Current value.", type => 'string', optional => 1, }, pending => { description => "Pending value.", type => 'string', optional => 1, }, delete => { description => "Indicates a pending delete request if present and not 0. " . "The value 2 indicates a force-delete request.", type => 'integer', minimum => 0, maximum => 2, optional => 1, }, }, }, }, code => sub { my ($param) = @_; my $conf = PVE::QemuConfig->load_config($param->{vmid}); my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete}); $conf->{cipassword} = '**********' if defined($conf->{cipassword}); $conf->{pending}->{cipassword} = '********** ' if defined($conf->{pending}->{cipassword}); return PVE::GuestHelpers::config_with_pending_array($conf, $pending_delete_hash); }, }); __PACKAGE__->register_method({ name => 'cloudinit_pending', path => '{vmid}/cloudinit', method => 'GET', proxyto => 'node', description => "Get the cloudinit configuration with both current and pending values.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Audit']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), }, }, returns => { type => "array", items => { type => "object", properties => { key => { description => "Configuration option name.", type => 'string', }, value => { description => "Value as it was used to generate the current cloudinit image.", type => 'string', optional => 1, }, pending => { description => "The new pending value.", type => 'string', optional => 1, }, delete => { description => "Indicates a pending delete request if present and not 0. ", type => 'integer', minimum => 0, maximum => 1, optional => 1, }, }, }, }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; my $conf = PVE::QemuConfig->load_config($vmid); my $ci = $conf->{'special-sections'}->{cloudinit}; $conf->{cipassword} = '**********' if exists $conf->{cipassword}; $ci->{cipassword} = '**********' if exists $ci->{cipassword}; my $res = []; # All the values that got added my $added = delete($ci->{added}) // ''; for my $key (PVE::Tools::split_list($added)) { push @$res, { key => $key, pending => $conf->{$key} }; } # All already existing values (+ their new value, if it exists) for my $opt (keys %$cloudinitoptions) { next if !$conf->{$opt}; next if $added =~ m/$opt/; my $item = { key => $opt, }; if (my $pending = $ci->{$opt}) { $item->{value} = $pending; $item->{pending} = $conf->{$opt}; } else { $item->{value} = $conf->{$opt}; } push @$res, $item; } # Now, we'll find the deleted ones for my $opt (keys %$ci) { next if $conf->{$opt}; push @$res, { key => $opt, delete => 1 }; } return $res; }, }); __PACKAGE__->register_method({ name => 'cloudinit_update', path => '{vmid}/cloudinit', method => 'PUT', protected => 1, proxyto => 'node', description => "Regenerate and change cloudinit config drive.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Config.Cloudinit']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $vmid = extract_param($param, 'vmid'); PVE::QemuConfig->lock_config( $vmid, sub { my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_lock($conf); my $storecfg = PVE::Storage::config(); PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid); }, ); return; }, }); # POST/PUT {vmid}/config implementation # # The original API used PUT (idempotent) an we assumed that all operations # are fast. But it turned out that almost any configuration change can # involve hot-plug actions, or disk alloc/free. Such actions can take long # time to complete and have side effects (not idempotent). # # The new implementation uses POST and forks a worker process. We added # a new option 'background_delay'. If specified we wait up to # 'background_delay' second for the worker task to complete. It returns null # if the task is finished within that time, else we return the UPID. my $update_vm_api = sub { my ($param, $sync) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $digest = extract_param($param, 'digest'); my $background_delay = extract_param($param, 'background_delay'); my $skip_cloud_init = extract_param($param, 'skip_cloud_init'); my $extraction_storage = extract_param($param, 'import-working-storage'); my @paramarr = (); # used for log message foreach my $key (sort keys %$param) { my $value = $key eq 'cipassword' ? '' : $param->{$key}; push @paramarr, "-$key", $value; } my $skiplock = extract_param($param, 'skiplock'); raise_param_exc({ skiplock => "Only root may use this option." }) if $skiplock && $authuser ne 'root@pam'; my $delete_str = extract_param($param, 'delete'); my $revert_str = extract_param($param, 'revert'); my $force = extract_param($param, 'force'); if (defined(my $ssh_keys = $param->{sshkeys})) { $ssh_keys = URI::Escape::uri_unescape($ssh_keys); PVE::Tools::validate_ssh_public_keys($ssh_keys); } $param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits}) if defined($param->{cpuunits}); # clamp value depending on cgroup version die "no options specified\n" if !$delete_str && !$revert_str && !scalar(keys %$param); my $storecfg = PVE::Storage::config(); &$resolve_cdrom_alias($param); # now try to verify all parameters my $revert = {}; foreach my $opt (PVE::Tools::split_list($revert_str)) { if (!PVE::QemuServer::option_exists($opt)) { raise_param_exc({ revert => "unknown option '$opt'" }); } raise_param_exc({ delete => "you can't use '-$opt' and " . "-revert $opt' at the same time" }) if defined($param->{$opt}); $revert->{$opt} = 1; } my @delete = (); foreach my $opt (PVE::Tools::split_list($delete_str)) { $opt = 'ide2' if $opt eq 'cdrom'; raise_param_exc({ delete => "you can't use '-$opt' and " . "-delete $opt' at the same time" }) if defined($param->{$opt}); raise_param_exc({ revert => "you can't use '-delete $opt' and " . "-revert $opt' at the same time" }) if $revert->{$opt}; if (!PVE::QemuServer::option_exists($opt)) { raise_param_exc({ delete => "unknown option '$opt'" }); } push @delete, $opt; } my $repl_conf = PVE::ReplicationConfig->new(); my $is_replicated = $repl_conf->check_for_existing_jobs($vmid, 1); my $check_replication = sub { my ($drive) = @_; return if !$is_replicated; my $volid = $drive->{file}; return if !$volid || !($drive->{replicate} // 1); return if PVE::QemuServer::drive_is_cdrom($drive); my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); die "cannot add non-managed/pass-through volume to a replicated VM\n" if !defined($storeid); return if defined($volname) && $volname eq 'cloudinit'; my $format; if ($volid =~ $PVE::QemuServer::Drive::NEW_DISK_RE) { $storeid = $2; $format = $drive->{format} || PVE::Storage::storage_default_format($storecfg, $storeid); } else { $format = (PVE::Storage::parse_volname($storecfg, $volid))[6]; } return if PVE::Storage::storage_can_replicate($storecfg, $storeid, $format); my $scfg = PVE::Storage::storage_config($storecfg, $storeid); return if $scfg->{shared}; die "cannot add non-replicatable volume to a replicated VM\n"; }; $check_drive_param->($param, $storecfg, $check_replication); foreach my $opt (keys %$param) { if ($opt =~ m/^net(\d+)$/) { # add macaddr my $net = PVE::QemuServer::Network::parse_net($param->{$opt}); $param->{$opt} = PVE::QemuServer::Network::print_net($net); } elsif ($opt eq 'vmgenid') { if ($param->{$opt} eq '1') { $param->{$opt} = PVE::QemuServer::generate_uuid(); } } elsif ($opt eq 'hookscript') { eval { PVE::GuestHelpers::check_hookscript($param->{$opt}, $storecfg); }; raise_param_exc({ $opt => $@ }) if $@; } } &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [@delete]); &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [keys %$param]); &$check_storage_access( $rpcenv, $authuser, $storecfg, $vmid, $param, undef, $extraction_storage, ); PVE::QemuServer::check_bridge_access($rpcenv, $authuser, $param); my $updatefn = sub { my $conf = PVE::QemuConfig->load_config($vmid); die "checksum mismatch (file change by other user?)\n" if $digest && $digest ne $conf->{digest}; &$check_cpu_model_access($rpcenv, $authuser, $param, $conf); # FIXME: 'suspended' lock should probabyl be a state or "weak" lock?! if (scalar(@delete) && grep { $_ eq 'vmstate' } @delete) { if (defined($conf->{lock}) && $conf->{lock} eq 'suspended') { delete $conf->{lock}; # for check lock check, not written out push @delete, 'lock'; # this is the real deal to write it out } push @delete, 'runningmachine' if $conf->{runningmachine}; push @delete, 'runningcpu' if $conf->{runningcpu}; push @delete, 'running-nets-host-mtu' if $conf->{'running-nets-host-mtu'}; } PVE::QemuConfig->check_lock($conf) if !$skiplock; foreach my $opt (keys %$revert) { if (defined($conf->{$opt})) { $param->{$opt} = $conf->{$opt}; } elsif (defined($conf->{pending}->{$opt})) { push @delete, $opt; } } if ($param->{memory} || defined($param->{balloon})) { my $memory = $param->{memory} || $conf->{pending}->{memory} || $conf->{memory}; my $maxmem = get_current_memory($memory); my $balloon = defined($param->{balloon}) ? $param->{balloon} : $conf->{pending}->{balloon} || $conf->{balloon}; die "balloon value too large (must be smaller than assigned memory)\n" if $balloon && $balloon > $maxmem; } PVE::Cluster::log_msg('info', $authuser, "update VM $vmid: " . join(' ', @paramarr)); my $worker = sub { print "update VM $vmid: " . join(' ', @paramarr) . "\n"; # write updates to pending section my $modified = {}; # record what $option we modify my @bootorder; if (my $boot = $conf->{boot}) { my $bootcfg = PVE::JSONSchema::parse_property_string('pve-qm-boot', $boot); @bootorder = PVE::Tools::split_list($bootcfg->{order}) if $bootcfg && $bootcfg->{order}; } my $bootorder_deleted = grep { $_ eq 'bootorder' } @delete; my $check_drive_perms = sub { my ($opt, $val) = @_; my $drive = PVE::QemuServer::parse_drive($opt, $val, 1); if (PVE::QemuServer::drive_is_cloudinit($drive)) { $rpcenv->check_vm_perm( $authuser, $vmid, undef, ['VM.Config.Cloudinit', 'VM.Config.CDROM'], ); } elsif (PVE::QemuServer::drive_is_cdrom($drive, 1)) { # CDROM $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.CDROM']); } else { $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']); } }; foreach my $opt (@delete) { $modified->{$opt} = 1; $conf = PVE::QemuConfig->load_config($vmid); # update/reload # value of what we want to delete, independent if pending or not my $val = $conf->{$opt} // $conf->{pending}->{$opt}; if (!defined($val)) { warn "cannot delete '$opt' - not set in current configuration!\n"; $modified->{$opt} = 0; next; } my $is_pending_val = defined($conf->{pending}->{$opt}); delete $conf->{pending}->{$opt}; # remove from bootorder if necessary if (!$bootorder_deleted && @bootorder && grep { $_ eq $opt } @bootorder) { @bootorder = grep { $_ ne $opt } @bootorder; $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder); $modified->{boot} = 1; } if ($opt =~ m/^unused/) { my $drive = PVE::QemuServer::parse_drive($opt, $val); PVE::QemuConfig->check_protection( $conf, "can't remove unused disk '$drive->{file}'", ); $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']); if (PVE::QemuServer::try_deallocate_drive( $storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, )) { delete $conf->{$opt}; PVE::QemuConfig->write_config($vmid, $conf); } } elsif ($opt eq 'vmstate') { PVE::QemuConfig->check_protection($conf, "can't remove vmstate '$val'"); if (PVE::QemuServer::try_deallocate_drive( $storecfg, $vmid, $conf, $opt, { file => $val }, $rpcenv, $authuser, 1, )) { delete $conf->{$opt}; PVE::QemuConfig->write_config($vmid, $conf); } } elsif (PVE::QemuServer::is_valid_drivename($opt)) { PVE::QemuConfig->check_protection($conf, "can't remove drive '$opt'"); $check_drive_perms->($opt, $val); PVE::QemuServer::vmconfig_register_unused_drive( $storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $val), ) if $is_pending_val; PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force); PVE::QemuConfig->write_config($vmid, $conf); } elsif ($opt =~ m/^serial\d+$/) { if ($val eq 'socket') { $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']); } elsif ($authuser ne 'root@pam') { die "only root can delete '$opt' config for real devices\n"; } PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force); PVE::QemuConfig->write_config($vmid, $conf); } elsif ($opt =~ m/^usb\d+$/) { check_usb_perm($rpcenv, $authuser, $vmid, undef, $opt, $val); PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force); PVE::QemuConfig->write_config($vmid, $conf); } elsif ($opt =~ m/^hostpci\d+$/) { check_hostpci_perm($rpcenv, $authuser, $vmid, undef, $opt, $val); PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force); PVE::QemuConfig->write_config($vmid, $conf); } elsif ($opt =~ m/^rng\d+$/) { check_rng_perm($rpcenv, $authuser, $vmid, undef, $opt, $val); PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force); PVE::QemuConfig->write_config($vmid, $conf); } elsif ($opt =~ m/^virtiofs\d$/) { check_dir_perm($rpcenv, $authuser, $vmid, undef, $opt, $val); PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force); PVE::QemuConfig->write_config($vmid, $conf); } elsif ($opt eq 'tags') { assert_tag_permissions($vmid, $val, '', $rpcenv, $authuser); delete $conf->{$opt}; PVE::QemuConfig->write_config($vmid, $conf); } elsif ($opt =~ m/^net\d+$/) { if ($conf->{$opt}) { PVE::QemuServer::check_bridge_access( $rpcenv, $authuser, { $opt => $conf->{$opt} }, ); } PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force); PVE::QemuConfig->write_config($vmid, $conf); } else { PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force); PVE::QemuConfig->write_config($vmid, $conf); } } foreach my $opt (keys %$param) { # add/change $modified->{$opt} = 1; $conf = PVE::QemuConfig->load_config($vmid); # update/reload next if defined($conf->{pending}->{$opt}) && ($param->{$opt} eq $conf->{pending}->{$opt}); # skip if nothing changed my $arch = PVE::QemuServer::Helpers::get_vm_arch($conf); if (PVE::QemuServer::is_valid_drivename($opt)) { # old drive if ($conf->{$opt}) { $check_drive_perms->($opt, $conf->{$opt}); prohibit_tpm_version_change($conf->{$opt}, $param->{$opt}) if $opt eq 'tpmstate0'; } # new drive $check_drive_perms->($opt, $param->{$opt}); PVE::QemuServer::vmconfig_register_unused_drive( $storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $conf->{pending}->{$opt}), ) if defined($conf->{pending}->{$opt}); assert_scsi_feature_compatibility($opt, $conf, $storecfg, $param->{$opt}) if $opt =~ m/^scsi\d+$/; my (undef, $created_opts) = create_disks( $rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, undef, { $opt => $param->{$opt} }, undef, undef, $extraction_storage, ); $conf->{pending}->{$_} = $created_opts->{$_} for keys $created_opts->%*; # default legacy boot order implies all cdroms anyway if (@bootorder) { # append new CD drives to bootorder to mark them bootable my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1); if ( PVE::QemuServer::drive_is_cdrom($drive, 1) && !grep(/^$opt$/, @bootorder) ) { push @bootorder, $opt; $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder); $modified->{boot} = 1; } } } elsif ($opt =~ m/^serial\d+/) { if ( (!defined($conf->{$opt}) || $conf->{$opt} eq 'socket') && $param->{$opt} eq 'socket' ) { $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']); } elsif ($authuser ne 'root@pam') { die "only root can modify '$opt' config for real devices\n"; } $conf->{pending}->{$opt} = $param->{$opt}; } elsif ($opt eq 'vga') { my $vga = PVE::QemuServer::parse_vga($param->{$opt}); PVE::QemuServer::assert_clipboard_config($vga); $conf->{pending}->{$opt} = $param->{$opt}; } elsif ($opt =~ m/^usb\d+/) { if (my $olddevice = $conf->{$opt}) { check_usb_perm($rpcenv, $authuser, $vmid, undef, $opt, $conf->{$opt}); } check_usb_perm($rpcenv, $authuser, $vmid, undef, $opt, $param->{$opt}); $conf->{pending}->{$opt} = $param->{$opt}; } elsif ($opt =~ m/^hostpci\d+$/) { if (my $oldvalue = $conf->{$opt}) { check_hostpci_perm($rpcenv, $authuser, $vmid, undef, $opt, $oldvalue); } check_hostpci_perm($rpcenv, $authuser, $vmid, undef, $opt, $param->{$opt}); $conf->{pending}->{$opt} = $param->{$opt}; } elsif ($opt =~ m/^rng\d+$/) { if (my $oldvalue = $conf->{$opt}) { check_rng_perm($rpcenv, $authuser, $vmid, undef, $opt, $oldvalue); } check_rng_perm($rpcenv, $authuser, $vmid, undef, $opt, $param->{$opt}); $conf->{pending}->{$opt} = $param->{$opt}; } elsif ($opt =~ m/^virtiofs\d$/) { if (my $oldvalue = $conf->{$opt}) { check_dir_perm($rpcenv, $authuser, $vmid, undef, $opt, $oldvalue); } check_dir_perm($rpcenv, $authuser, $vmid, undef, $opt, $param->{$opt}); $conf->{pending}->{$opt} = $param->{$opt}; } elsif ($opt eq 'tags') { assert_tag_permissions( $vmid, $conf->{$opt}, $param->{$opt}, $rpcenv, $authuser, ); $conf->{pending}->{$opt} = PVE::GuestHelpers::get_unique_tags($param->{$opt}); } elsif ($opt =~ m/^net\d+$/) { if ($conf->{$opt}) { PVE::QemuServer::check_bridge_access( $rpcenv, $authuser, { $opt => $conf->{$opt} }, ); } $conf->{pending}->{$opt} = $param->{$opt}; } elsif ($opt eq 'machine') { my $machine_conf = PVE::QemuServer::Machine::parse_machine($param->{$opt}); PVE::QemuServer::Machine::assert_valid_machine_property($machine_conf); $conf->{pending}->{$opt} = $param->{$opt}; } elsif ($opt eq 'ostype') { # Check if machine version pinning is needed when switching OS type, just like # upon creation. Skip if 'machine' is explicitly set or removed at the same time # to honor the users request. While it should be enough to look at $modified, # because 'machine' is sorted before 'ostype', be explicit just to be sure. if ( !defined($param->{machine}) && !defined($conf->{pending}->{machine}) && !$modified->{machine} # detects deletion ) { eval { my $machine_string = PVE::QemuServer::Machine::check_and_pin_machine_string( $conf->{machine}, $param->{ostype}, $arch, ); $conf->{pending}->{machine} = $machine_string if $machine_string; }; print "automatic pinning of machine version failed - $@" if $@; } $conf->{pending}->{$opt} = $param->{$opt}; } elsif ($opt eq 'cipassword') { if (!PVE::QemuServer::Helpers::windows_version($conf->{ostype})) { # Same logic as in cloud-init (but with the regex fixed...) $param->{cipassword} = PVE::Tools::encrypt_pw($param->{cipassword}) if $param->{cipassword} !~ /^\$(?:[156]|2[ay])(\$.+){2}/; } $conf->{cipassword} = $param->{cipassword}; } else { $conf->{pending}->{$opt} = $param->{$opt}; if ($opt eq 'boot') { my $new_bootcfg = PVE::JSONSchema::parse_property_string('pve-qm-boot', $param->{$opt}); if ($new_bootcfg->{order}) { my @devs = PVE::Tools::split_list($new_bootcfg->{order}); for my $dev (@devs) { my $exists = $conf->{$dev} || $conf->{pending}->{$dev} || $param->{$dev}; my $deleted = grep { $_ eq $dev } @delete; die "invalid bootorder: device '$dev' does not exist'\n" if !$exists || $deleted; } # remove legacy boot order settings if new one set $conf->{pending}->{$opt} = PVE::QemuServer::print_bootorder(\@devs); PVE::QemuConfig->add_to_pending_delete($conf, "bootdisk") if $conf->{bootdisk}; } } } PVE::QemuConfig->remove_from_pending_delete($conf, $opt); PVE::QemuConfig->write_config($vmid, $conf); } # remove pending changes when nothing changed $conf = PVE::QemuConfig->load_config($vmid); # update/reload my $changes = PVE::QemuConfig->cleanup_pending($conf); PVE::QemuConfig->write_config($vmid, $conf) if $changes; return if !scalar(keys %{ $conf->{pending} }); my $running = PVE::QemuServer::check_running($vmid); # apply pending changes $conf = PVE::QemuConfig->load_config($vmid); # update/reload my $errors = {}; if ($running) { PVE::QemuServer::vmconfig_hotplug_pending( $vmid, $conf, $storecfg, $modified, $errors, ); } else { # cloud_init must be skipped if we are in an incoming, remote live migration PVE::QemuServer::vmconfig_apply_pending( $vmid, $conf, $storecfg, $errors, $skip_cloud_init, ); } raise_param_exc($errors) if scalar(keys %$errors); return; }; if ($sync) { &$worker(); return; } else { my $upid = $rpcenv->fork_worker('qmconfig', $vmid, $authuser, $worker); if ($background_delay) { # Note: It would be better to do that in the Event based HTTPServer # to avoid blocking call to sleep. my $end_time = time() + $background_delay; my $task = PVE::Tools::upid_decode($upid); my $running = 1; while (time() < $end_time) { $running = PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}); last if !$running; sleep(1); # this gets interrupted when child process ends } if (!$running) { my $status = PVE::Tools::upid_read_status($upid); return if !PVE::Tools::upid_status_is_error($status); die "failed to update VM $vmid: $status\n"; } } return $upid; } }; return PVE::QemuConfig->lock_config($vmid, $updatefn); }; my $vm_config_perm_list = [ 'VM.Config.Disk', 'VM.Config.CDROM', 'VM.Config.CPU', 'VM.Config.Memory', 'VM.Config.Network', 'VM.Config.HWType', 'VM.Config.Options', 'VM.Config.Cloudinit', ]; __PACKAGE__->register_method({ name => 'update_vm_async', path => '{vmid}/config', method => 'POST', protected => 1, proxyto => 'node', description => "Set virtual machine options (asynchronous API).", permissions => { check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1], }, parameters => { additionalProperties => 0, properties => PVE::QemuServer::json_config_properties( { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), skiplock => get_standard_option('skiplock'), delete => { type => 'string', format => 'pve-configid-list', description => "A list of settings you want to delete.", optional => 1, }, revert => { type => 'string', format => 'pve-configid-list', description => "Revert a pending change.", optional => 1, }, force => { type => 'boolean', description => $opt_force_description, optional => 1, requires => 'delete', }, digest => { type => 'string', description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.', maxLength => 40, optional => 1, }, background_delay => { type => 'integer', description => "Time to wait for the task to finish. We return 'null' if the task finish within that time.", minimum => 1, maximum => 30, optional => 1, }, 'import-working-storage' => get_standard_option( 'pve-storage-id', { description => "A file-based storage with 'images' content-type enabled, which" . " is used as an intermediary extraction storage during import. Defaults to" . " the source storage.", optional => 1, completion => \&PVE::QemuServer::complete_storage, }, ), }, 1, # with_disk_alloc ), }, returns => { type => 'string', optional => 1, }, code => $update_vm_api, }); __PACKAGE__->register_method({ name => 'update_vm', path => '{vmid}/config', method => 'PUT', protected => 1, proxyto => 'node', description => "Set virtual machine options (synchronous API) - You should consider using the POST method instead for any actions involving hotplug or storage allocation.", permissions => { check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1], }, parameters => { additionalProperties => 0, properties => PVE::QemuServer::json_config_properties( { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }, ), skiplock => get_standard_option('skiplock'), delete => { type => 'string', format => 'pve-configid-list', description => "A list of settings you want to delete.", optional => 1, }, revert => { type => 'string', format => 'pve-configid-list', description => "Revert a pending change.", optional => 1, }, force => { type => 'boolean', description => $opt_force_description, optional => 1, requires => 'delete', }, digest => { type => 'string', description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.', maxLength => 40, optional => 1, }, }, 1, # with_disk_alloc ), }, returns => { type => 'null' }, code => sub { my ($param) = @_; &$update_vm_api($param, 1); return; }, }); __PACKAGE__->register_method({ name => 'destroy_vm', path => '{vmid}', method => 'DELETE', protected => 1, proxyto => 'node', description => "Destroy the VM and all used/owned volumes. Removes any VM specific permissions" . " and firewall rules", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Allocate']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }, ), skiplock => get_standard_option('skiplock'), purge => { type => 'boolean', description => "Remove VMID from configurations, like backup & replication jobs and HA.", optional => 1, }, 'destroy-unreferenced-disks' => { type => 'boolean', description => "If set, destroy additionally all disks not referenced in the config" . " but with a matching VMID from all enabled storages.", optional => 1, default => 0, }, }, }, returns => { type => 'string', }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $vmid = $param->{vmid}; my $skiplock = $param->{skiplock}; raise_param_exc({ skiplock => "Only root may use this option." }) if $skiplock && $authuser ne 'root@pam'; my $early_checks = sub { # test if VM exists my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_protection($conf, "can't remove VM $vmid"); my $ha_managed = PVE::HA::Config::service_is_configured("vm:$vmid"); if (!$param->{purge}) { die "unable to remove VM $vmid - used in HA resources and purge parameter not set.\n" if $ha_managed; # don't allow destroy if with replication jobs but no purge param my $repl_conf = PVE::ReplicationConfig->new(); $repl_conf->check_for_existing_jobs($vmid); } die "VM $vmid is running - destroy failed\n" if PVE::QemuServer::check_running($vmid); return $ha_managed; }; $early_checks->(); my $realcmd = sub { my $upid = shift; my $storecfg = PVE::Storage::config(); syslog('info', "destroy VM $vmid: $upid\n"); PVE::QemuConfig->lock_config( $vmid, sub { # repeat, config might have changed my $ha_managed = $early_checks->(); my $purge_unreferenced = $param->{'destroy-unreferenced-disks'}; PVE::QemuServer::destroy_vm( $storecfg, $vmid, $skiplock, { lock => 'destroyed' }, $purge_unreferenced, ); PVE::AccessControl::remove_vm_access($vmid); PVE::Firewall::remove_vmfw_conf($vmid); if ($param->{purge}) { print "purging VM $vmid from related configurations..\n"; PVE::ReplicationConfig::remove_vmid_jobs($vmid); PVE::VZDump::Plugin::remove_vmid_from_backup_jobs($vmid); if ($ha_managed) { PVE::HA::Config::delete_service_from_config( "vm:$vmid", $param->{purge}, ); print "NOTE: removed VM $vmid from HA resource configuration.\n"; } } # only now remove the zombie config, else we can have reuse race PVE::QemuConfig->destroy_config($vmid); }, ); }; return $rpcenv->fork_worker('qmdestroy', $vmid, $authuser, $realcmd); }, }); __PACKAGE__->register_method({ name => 'unlink', path => '{vmid}/unlink', method => 'PUT', protected => 1, proxyto => 'node', description => "Unlink/delete disk images.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Config.Disk']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), idlist => { type => 'string', format => 'pve-configid-list', description => "A list of disk IDs you want to delete.", }, force => { type => 'boolean', description => $opt_force_description, optional => 1, }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; $param->{delete} = extract_param($param, 'idlist'); __PACKAGE__->update_vm($param); return; }, }); my $sslcert; __PACKAGE__->register_method({ name => 'vncproxy', path => '{vmid}/vncproxy', method => 'POST', protected => 1, permissions => { check => ['perm', '/vms/{vmid}', ['VM.Console']], }, description => "Creates a TCP VNC proxy connections.", parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), websocket => { optional => 1, type => 'boolean', description => "Prepare for websocket upgrade (only required when using " . "serial terminal, otherwise upgrade is always possible).", }, # FIXME: MAJOR VERSION: Drop this, require always using explicit 'password' return value 'generate-password' => { optional => 1, type => 'boolean', default => 0, description => "Deprecated, do not use. Password is generated when required.", }, }, }, returns => { additionalProperties => 0, properties => { user => { type => 'string' }, ticket => { type => 'string' }, password => { optional => 1, description => "Password used for authentication within the VNC protocol." . " Consists of printable ASCII characters ('!' .. '~').", type => 'string', }, cert => { type => 'string' }, port => { type => 'integer' }, upid => { type => 'string' }, }, }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $vmid = $param->{vmid}; my $node = $param->{node}; my $websocket = $param->{websocket}; my $conf = PVE::QemuConfig->load_config($vmid, $node); # check if VM exists my $serial; if ($conf->{vga}) { my $vga = PVE::QemuServer::parse_vga($conf->{vga}); $serial = $vga->{type} if defined($vga->{type}) && $vga->{type} =~ m/^serial\d+$/; } my $authpath = "/vms/$vmid"; $sslcert = PVE::Tools::file_get_contents("/etc/pve/pve-root-ca.pem", 8192) if !$sslcert; my $family; my $remcmd = []; if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) { (undef, $family) = PVE::Cluster::remote_node_ip($node); my $sshinfo = PVE::SSHInfo::get_ssh_info($node); # NOTE: kvm VNC traffic is already TLS encrypted or is known insecure $remcmd = PVE::SSHInfo::ssh_info_to_command($sshinfo, defined($serial) ? '-t' : '-T'); } else { $family = PVE::Tools::get_host_address_family($node); } my $port = PVE::Tools::next_vnc_port($family); my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath, $port); my $password; if ($param->{'generate-password'} || !defined($serial) || $param->{websocket}) { $password = PVE::Ticket::generate_vnc_password(); # FIXME: MAJOR VERSION: Avoid this hack, require using explicit 'password' return value $ticket = "${password}:${ticket}"; } # else authentication happens via ticket only, not via password in VNC protocol my $timeout = 10; my $realcmd = sub { my $upid = shift; syslog('info', "starting vnc proxy $upid\n"); my $cmd; if (defined($serial)) { my $termcmd = ['/usr/sbin/qm', 'terminal', $vmid, '-iface', $serial, '-escape', '0']; $cmd = [ '/usr/bin/vncterm', '-rfbport', $port, '-timeout', $timeout, '-authpath', $authpath, '-perm', 'Sys.Console', '-verify-port', ]; if ($param->{websocket}) { $ENV{PVE_VNC_TICKET} = $password; # pass VNC protocol password to vncterm push @$cmd, '-notls', '-listen', 'localhost'; } else { $ENV{PVE_VNC_TICKET} = $ticket; # pass VNC ticket to vncterm } push @$cmd, '-c', @$remcmd, @$termcmd; run_command($cmd); } else { $ENV{LC_PVE_TICKET} = $password; # set VNC protocol password with "qm vncproxy" $cmd = [@$remcmd, "/usr/sbin/qm", 'vncproxy', $vmid]; my $sock; if ($param->{websocket}) { # listen for localhost only, no TLS my $socket_params = { ReuseAddr => 1, Listen => 1, LocalPort => $port, Proto => 'tcp', GetAddrInfoFlags => 0, LocalHost => 'localhost', }; $sock = IO::Socket::IP->new($socket_params->%*) or die "failed to create socket: $!\n"; } else { my $socket_params = { ReuseAddr => 1, Listen => 1, LocalPort => $port, Proto => 'tcp', GetAddrInfoFlags => 0, SSL_server => 1, }; my $pveproxy_cert_file = '/etc/pve/local/pveproxy-ssl.pem'; my $pveproxy_key_file = '/etc/pve/local/pveproxy-ssl.key'; if (-f $pveproxy_cert_file && -f $pveproxy_key_file) { $socket_params->{SSL_cert_file} = $pveproxy_cert_file; $socket_params->{SSL_key_file} = $pveproxy_key_file; } else { $socket_params->{SSL_cert_file} = '/etc/pve/local/pve-ssl.pem'; $socket_params->{SSL_key_file} = '/etc/pve/local/pve-ssl.key'; } $sock = IO::Socket::SSL->new($socket_params->%*) or die "failed to create SSL socket: $!\n"; } # Inside the worker we shouldn't have any previous alarms # running anyway...: alarm(0); local $SIG{ALRM} = sub { die "connection timed out\n" }; alarm $timeout; accept(my $cli, $sock) or die "connection failed: $!\n"; alarm(0); close($sock); if ( run_command( $cmd, output => '>&' . fileno($cli), input => '<&' . fileno($cli), noerr => 1, ) != 0 ) { die "Failed to run vncproxy.\n"; } } return; }; my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd, 1); PVE::Tools::wait_for_vnc_port($port); my $res = { user => $authuser, ticket => $ticket, port => $port, upid => $upid, cert => $sslcert, }; $res->{password} = $password if defined($password); return $res; }, }); __PACKAGE__->register_method({ name => 'termproxy', path => '{vmid}/termproxy', method => 'POST', protected => 1, permissions => { check => ['perm', '/vms/{vmid}', ['VM.Console']], }, description => "Creates a TCP proxy connections.", parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), serial => { optional => 1, type => 'string', enum => [qw(serial0 serial1 serial2 serial3)], description => "opens a serial terminal (defaults to display)", }, }, }, returns => { additionalProperties => 0, properties => { user => { type => 'string' }, ticket => { type => 'string' }, port => { type => 'integer' }, upid => { type => 'string' }, }, }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $vmid = $param->{vmid}; my $node = $param->{node}; my $serial = $param->{serial}; my $conf = PVE::QemuConfig->load_config($vmid, $node); # check if VM exists if (!defined($serial)) { if ($conf->{vga}) { my $vga = PVE::QemuServer::parse_vga($conf->{vga}); $serial = $vga->{type} if defined($vga->{type}) && $vga->{type} =~ m/^serial\d+$/; } } my $authpath = "/vms/$vmid"; my $family; my $remcmd = []; if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) { (undef, $family) = PVE::Cluster::remote_node_ip($node); my $sshinfo = PVE::SSHInfo::get_ssh_info($node); $remcmd = PVE::SSHInfo::ssh_info_to_command($sshinfo, '-t'); push @$remcmd, '--'; } else { $family = PVE::Tools::get_host_address_family($node); } my $port = PVE::Tools::next_vnc_port($family); my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath, $port); my $termcmd = ['/usr/sbin/qm', 'terminal', $vmid, '-escape', '0']; push @$termcmd, '-iface', $serial if $serial; my $realcmd = sub { my $upid = shift; syslog('info', "starting qemu termproxy $upid\n"); pipe(my $ticket_rd, my $ticket_wr) or die "failed to create pipe: $!\n"; my $flags = fcntl($ticket_rd, F_GETFD, 0) // die "failed to get file descriptor flags: $!\n"; fcntl($ticket_rd, F_SETFD, $flags & ~FD_CLOEXEC) // die "failed to remove CLOEXEC flag from fd: $!\n"; my $cmd = [ '/usr/bin/termproxy', $port, '--path', $authpath, '--perm', 'VM.Console', '--vncticket-endpoint', '--verify-port', '--ticket-fd', fileno($ticket_rd), '--', ]; push @$cmd, @$remcmd, @$termcmd; my $afterfork = sub { print {$ticket_wr} $ticket; close($ticket_wr); }; run_command($cmd, afterfork => $afterfork); }; my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd, 1); PVE::Tools::wait_for_vnc_port($port); return { user => $authuser, ticket => $ticket, port => $port, upid => $upid, }; }, }); __PACKAGE__->register_method({ name => 'vncwebsocket', path => '{vmid}/vncwebsocket', method => 'GET', permissions => { description => "You also need to pass a valid ticket (vncticket).", check => ['perm', '/vms/{vmid}', ['VM.Console']], }, description => "Opens a websocket for VNC traffic.", parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), vncticket => { description => "Ticket from previous call to vncproxy.", type => 'string', maxLength => 512, }, port => { description => "Port number returned by previous vncproxy call.", type => 'integer', minimum => 5900, maximum => 5999, }, }, }, returns => { type => "object", properties => { port => { type => 'string' }, }, }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $vmid = $param->{vmid}; my $node = $param->{node}; my $authpath = "/vms/$vmid"; # Note: VNC ports may be accessible from outside, so there is a password that needs to # additionally be checked inside the VNC protocol. my $port = $param->{port}; PVE::AccessControl::verify_vnc_ticket($param->{vncticket}, $authuser, $authpath, $port); my $conf = PVE::QemuConfig->load_config($vmid, $node); # VM exists ? return { port => $port }; }, }); __PACKAGE__->register_method({ name => 'spiceproxy', path => '{vmid}/spiceproxy', method => 'POST', protected => 1, proxyto => 'node', permissions => { check => ['perm', '/vms/{vmid}', ['VM.Console']], }, description => "Returns a SPICE configuration to connect to the VM.", parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), proxy => get_standard_option('spice-proxy', { optional => 1 }), }, }, returns => get_standard_option('remote-viewer-config'), code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $vmid = $param->{vmid}; my $node = $param->{node}; my $proxy = $param->{proxy}; my $conf = PVE::QemuConfig->load_config($vmid, $node); my $title = "VM $vmid"; $title .= " - " . $conf->{name} if $conf->{name}; my $port = PVE::QemuServer::spice_port($vmid); my ($ticket, undef, $remote_viewer_config) = PVE::AccessControl::remote_viewer_config($authuser, $vmid, $node, $proxy, $title, $port); mon_cmd($vmid, "set_password", protocol => 'spice', password => $ticket); mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30"); return $remote_viewer_config; }, }); __PACKAGE__->register_method({ name => 'vmcmdidx', path => '{vmid}/status', method => 'GET', proxyto => 'node', description => "Directory index", permissions => { user => 'all', }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), }, }, returns => { type => 'array', items => { type => "object", properties => { subdir => { type => 'string' }, }, }, links => [{ rel => 'child', href => "{subdir}" }], }, code => sub { my ($param) = @_; # test if VM exists my $conf = PVE::QemuConfig->load_config($param->{vmid}); my $res = [ { subdir => 'current' }, { subdir => 'start' }, { subdir => 'stop' }, { subdir => 'reset' }, { subdir => 'shutdown' }, { subdir => 'suspend' }, { subdir => 'reboot' }, ]; return $res; }, }); __PACKAGE__->register_method({ name => 'vm_status', path => '{vmid}/status/current', method => 'GET', proxyto => 'node', protected => 1, # qemu pid files are only readable by root description => "Get virtual machine status.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Audit']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), }, }, returns => { type => 'object', properties => { %$PVE::QemuServer::vmstatus_return_properties, ha => { description => "HA manager service status.", type => 'object', }, spice => { description => "QEMU VGA configuration supports spice.", type => 'boolean', optional => 1, }, agent => { description => "QEMU Guest Agent is enabled in config.", type => 'boolean', optional => 1, }, clipboard => { description => 'Enable a specific clipboard. If not set, depending on' . ' the display type the SPICE one will be added.', type => 'string', enum => ['vnc'], optional => 1, }, }, }, code => sub { my ($param) = @_; # test if VM exists my $conf = PVE::QemuConfig->load_config($param->{vmid}); my $vmstatus = PVE::QemuServer::vmstatus($param->{vmid}, 1); my $status = $vmstatus->{ $param->{vmid} }; $status->{ha} = PVE::HA::Config::get_service_status("vm:$param->{vmid}"); if ($conf->{vga}) { my $vga = PVE::QemuServer::parse_vga($conf->{vga}); my $spice = defined($vga->{type}) && $vga->{type} =~ /^virtio/; $spice ||= PVE::QemuServer::vga_conf_has_spice($conf->{vga}); $status->{spice} = 1 if $spice; $status->{clipboard} = $vga->{clipboard}; } $status->{agent} = 1 if PVE::QemuServer::Agent::get_qga_key($conf, 'enabled'); return $status; }, }); __PACKAGE__->register_method({ name => 'vm_start', path => '{vmid}/status/start', method => 'POST', protected => 1, proxyto => 'node', description => "Start virtual machine.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.PowerMgmt']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }, ), skiplock => get_standard_option('skiplock'), stateuri => get_standard_option('pve-qm-stateuri'), migratedfrom => get_standard_option('pve-node', { optional => 1 }), migration_type => { type => 'string', enum => ['secure', 'insecure'], description => "Migration traffic is encrypted using an SSH " . "tunnel by default. On secure, completely private networks " . "this can be disabled to increase performance.", optional => 1, }, migration_network => { type => 'string', format => 'CIDR', description => "CIDR of the (sub) network that is used for migration.", optional => 1, }, machine => get_standard_option('pve-qemu-machine'), 'force-cpu' => { description => "Override QEMU's -cpu argument with the given string.", type => 'string', optional => 1, }, targetstorage => get_standard_option('pve-targetstorage'), timeout => { description => "Wait maximal timeout seconds.", type => 'integer', minimum => 0, default => 'max(30, vm memory in GiB)', optional => 1, }, 'with-conntrack-state' => { type => 'boolean', optional => 1, default => 0, description => 'Whether to migrate conntrack entries for running VMs.', }, 'nets-host-mtu' => { type => 'string', pattern => 'net\d+=\d+(,net\d+=\d+)*', optional => 1, description => 'Used for migration compat. List of VirtIO network devices and their effective' . ' host_mtu setting according to the QEMU object model on the source side of' . ' the migration. A value of 0 means that the host_mtu parameter is to be' . ' avoided for the corresponding device.', }, }, }, returns => { type => 'string', }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $timeout = extract_param($param, 'timeout'); my $machine = extract_param($param, 'machine'); my $get_root_param = sub { my $value = extract_param($param, $_[0]); raise_param_exc({ "$_[0]" => "Only root may use this option." }) if $value && $authuser ne 'root@pam'; return $value; }; my $stateuri = $get_root_param->('stateuri'); my $skiplock = $get_root_param->('skiplock'); my $migratedfrom = $get_root_param->('migratedfrom'); my $migration_type = $get_root_param->('migration_type'); my $migration_network = $get_root_param->('migration_network'); my $targetstorage = $get_root_param->('targetstorage'); my $force_cpu = $get_root_param->('force-cpu'); my $with_conntrack_state = $get_root_param->('with-conntrack-state'); my $nets_host_mtu = $get_root_param->('nets-host-mtu'); my $storagemap; if ($targetstorage) { raise_param_exc( { targetstorage => "targetstorage can only by used with migratedfrom." }) if !$migratedfrom; $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') }; raise_param_exc({ targetstorage => "failed to parse storage map: $@" }) if $@; } # read spice ticket from STDIN my $spice_ticket; my $nbd_protocol_version = 0; my $replicated_volumes = {}; my $offline_volumes = {}; if ( $stateuri && ($stateuri eq 'tcp' || $stateuri eq 'unix') && $migratedfrom && ($rpcenv->{type} eq 'cli') ) { while (defined(my $line = )) { chomp $line; if ($line =~ m/^spice_ticket: (.+)$/) { $spice_ticket = $1; } elsif ($line =~ m/^nbd_protocol_version: (\d+)$/) { $nbd_protocol_version = $1; } elsif ($line =~ m/^replicated_volume: (.*)$/) { $replicated_volumes->{$1} = 1; } elsif ($line =~ m/^tpmstate0: (.*)$/) { # Deprecated, use offline_volume instead # TODO PVE 10.x drop special handling here $offline_volumes->{tpmstate0} = $1; } elsif ($line =~ m/^offline_volume: ([^:]+): (.*)$/) { $offline_volumes->{$1} = $2; } elsif (!$spice_ticket) { # fallback for old source node $spice_ticket = $line; } else { warn "unknown 'start' parameter on STDIN: '$line'\n"; } } } PVE::Cluster::check_cfs_quorum(); my $storecfg = PVE::Storage::config(); if (PVE::HA::Config::vm_is_ha_managed($vmid) && !$stateuri && $rpcenv->{type} ne 'ha') { my $hacmd = sub { my $upid = shift; print "Requesting HA start for VM $vmid\n"; my $cmd = ['ha-manager', 'set', "vm:$vmid", '--state', 'started']; run_command($cmd); return; }; return $rpcenv->fork_worker('hastart', $vmid, $authuser, $hacmd); } else { my $realcmd = sub { my $upid = shift; syslog('info', "start VM $vmid: $upid\n"); my $migrate_opts = { migratedfrom => $migratedfrom, spice_ticket => $spice_ticket, network => $migration_network, type => $migration_type, storagemap => $storagemap, nbd_proto_version => $nbd_protocol_version, replicated_volumes => $replicated_volumes, offline_volumes => $offline_volumes, with_conntrack_state => $with_conntrack_state, }; my $params = { statefile => $stateuri, skiplock => $skiplock, forcemachine => $machine, timeout => $timeout, forcecpu => $force_cpu, 'nets-host-mtu' => $nets_host_mtu, }; PVE::QemuServer::vm_start($storecfg, $vmid, $params, $migrate_opts); return; }; return $rpcenv->fork_worker('qmstart', $vmid, $authuser, $realcmd); } }, }); __PACKAGE__->register_method({ name => 'vm_stop', path => '{vmid}/status/stop', method => 'POST', protected => 1, proxyto => 'node', description => "Stop virtual machine. The qemu process will exit immediately. This" . " is akin to pulling the power plug of a running computer and may damage the VM data.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.PowerMgmt']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), skiplock => get_standard_option('skiplock'), migratedfrom => get_standard_option('pve-node', { optional => 1 }), timeout => { description => "Wait maximal timeout seconds.", type => 'integer', minimum => 0, optional => 1, }, keepActive => { description => "Do not deactivate storage volumes.", type => 'boolean', optional => 1, default => 0, }, 'overrule-shutdown' => { description => "Try to abort active 'qmshutdown' tasks before stopping.", optional => 1, type => 'boolean', default => 0, }, }, }, returns => { type => 'string', }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $skiplock = extract_param($param, 'skiplock'); raise_param_exc({ skiplock => "Only root may use this option." }) if $skiplock && $authuser ne 'root@pam'; my $keepActive = extract_param($param, 'keepActive'); raise_param_exc({ keepActive => "Only root may use this option." }) if $keepActive && $authuser ne 'root@pam'; my $migratedfrom = extract_param($param, 'migratedfrom'); raise_param_exc({ migratedfrom => "Only root may use this option." }) if $migratedfrom && $authuser ne 'root@pam'; my $overrule_shutdown = extract_param($param, 'overrule-shutdown'); my $storecfg = PVE::Storage::config(); if ( PVE::HA::Config::vm_is_ha_managed($vmid) && ($rpcenv->{type} ne 'ha') && !defined($migratedfrom) ) { raise_param_exc({ 'overrule-shutdown' => "Not applicable for HA resources." }) if $overrule_shutdown; my $hacmd = sub { my $upid = shift; print "Requesting HA stop for VM $vmid\n"; my $cmd = ['ha-manager', 'crm-command', 'stop', "vm:$vmid", '0']; run_command($cmd); return; }; return $rpcenv->fork_worker('hastop', $vmid, $authuser, $hacmd); } else { my $realcmd = sub { my $upid = shift; syslog('info', "stop VM $vmid: $upid\n"); if ($overrule_shutdown) { my $overruled_tasks = PVE::GuestHelpers::abort_guest_tasks($rpcenv, 'qmshutdown', $vmid); my $overruled_tasks_list = join(", ", $overruled_tasks->@*); print "overruled qmshutdown tasks: $overruled_tasks_list\n" if @$overruled_tasks; } PVE::QemuServer::vm_stop( $storecfg, $vmid, $skiplock, 0, $param->{timeout}, 0, 1, $keepActive, $migratedfrom, ); return; }; return $rpcenv->fork_worker('qmstop', $vmid, $authuser, $realcmd); } }, }); __PACKAGE__->register_method({ name => 'vm_reset', path => '{vmid}/status/reset', method => 'POST', protected => 1, proxyto => 'node', description => "Reset virtual machine.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.PowerMgmt']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), skiplock => get_standard_option('skiplock'), }, }, returns => { type => 'string', }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $skiplock = extract_param($param, 'skiplock'); raise_param_exc({ skiplock => "Only root may use this option." }) if $skiplock && $authuser ne 'root@pam'; die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid); my $realcmd = sub { my $upid = shift; PVE::QemuServer::vm_reset($vmid, $skiplock); return; }; return $rpcenv->fork_worker('qmreset', $vmid, $authuser, $realcmd); }, }); __PACKAGE__->register_method({ name => 'vm_shutdown', path => '{vmid}/status/shutdown', method => 'POST', protected => 1, proxyto => 'node', description => "Shutdown virtual machine. This is similar to pressing the power button on a" . " physical machine. This will send an ACPI event for the guest OS, which should then" . " proceed to a clean shutdown.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.PowerMgmt']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), skiplock => get_standard_option('skiplock'), timeout => { description => "Wait maximal timeout seconds.", type => 'integer', minimum => 0, optional => 1, }, forceStop => { description => "Make sure the VM stops.", type => 'boolean', optional => 1, default => 0, }, keepActive => { description => "Do not deactivate storage volumes.", type => 'boolean', optional => 1, default => 0, }, }, }, returns => { type => 'string', }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $skiplock = extract_param($param, 'skiplock'); raise_param_exc({ skiplock => "Only root may use this option." }) if $skiplock && $authuser ne 'root@pam'; my $keepActive = extract_param($param, 'keepActive'); raise_param_exc({ keepActive => "Only root may use this option." }) if $keepActive && $authuser ne 'root@pam'; my $storecfg = PVE::Storage::config(); my $shutdown = 1; # sending a graceful shutdown command to paused VMs runs into timeouts, and even worse, when # the VM gets resumed later, it still gets the request delivered and powers off if (PVE::QemuServer::vm_is_paused($vmid, 1)) { if ($param->{forceStop}) { warn "VM is paused - stop instead of shutdown\n"; $shutdown = 0; } else { die "VM is paused - cannot shutdown\n"; } } if (PVE::HA::Config::vm_is_ha_managed($vmid) && $rpcenv->{type} ne 'ha') { my $timeout = $param->{timeout} // 60; my $hacmd = sub { my $upid = shift; print "Requesting HA stop for VM $vmid\n"; my $cmd = ['ha-manager', 'crm-command', 'stop', "vm:$vmid", "$timeout"]; run_command($cmd); return; }; return $rpcenv->fork_worker('hastop', $vmid, $authuser, $hacmd); } else { my $realcmd = sub { my $upid = shift; syslog('info', "shutdown VM $vmid: $upid\n"); PVE::QemuServer::vm_stop( $storecfg, $vmid, $skiplock, 0, $param->{timeout}, $shutdown, $param->{forceStop}, $keepActive, ); return; }; return $rpcenv->fork_worker('qmshutdown', $vmid, $authuser, $realcmd); } }, }); __PACKAGE__->register_method({ name => 'vm_reboot', path => '{vmid}/status/reboot', method => 'POST', protected => 1, proxyto => 'node', description => "Reboot the VM by shutting it down, and starting it again. Applies pending changes.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.PowerMgmt']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), timeout => { description => "Wait maximal timeout seconds for the shutdown.", type => 'integer', minimum => 0, optional => 1, }, }, }, returns => { type => 'string', }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); die "VM is paused - cannot shutdown\n" if PVE::QemuServer::vm_is_paused($vmid, 1); die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid); my $realcmd = sub { my $upid = shift; syslog('info', "requesting reboot of VM $vmid: $upid\n"); PVE::QemuServer::vm_reboot($vmid, $param->{timeout}); return; }; return $rpcenv->fork_worker('qmreboot', $vmid, $authuser, $realcmd); }, }); __PACKAGE__->register_method({ name => 'vm_suspend', path => '{vmid}/status/suspend', method => 'POST', protected => 1, proxyto => 'node', description => "Suspend virtual machine.", permissions => { description => "You need 'VM.PowerMgmt' on /vms/{vmid}, and if you have set 'todisk'," . " you need also 'VM.Config.Disk' on /vms/{vmid} and 'Datastore.AllocateSpace'" . " on the storage for the vmstate.", check => ['perm', '/vms/{vmid}', ['VM.PowerMgmt']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), skiplock => get_standard_option('skiplock'), todisk => { type => 'boolean', default => 0, optional => 1, description => 'If set, suspends the VM to disk. Will be resumed on next VM start.', }, statestorage => get_standard_option( 'pve-storage-id', { description => "The storage for the VM state", requires => 'todisk', optional => 1, completion => \&PVE::Storage::complete_storage_enabled, }, ), }, }, returns => { type => 'string', }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $todisk = extract_param($param, 'todisk') // 0; my $statestorage = extract_param($param, 'statestorage'); my $skiplock = extract_param($param, 'skiplock'); raise_param_exc({ skiplock => "Only root may use this option." }) if $skiplock && $authuser ne 'root@pam'; die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid); die "Cannot suspend HA managed VM to disk\n" if $todisk && PVE::HA::Config::vm_is_ha_managed($vmid); # early check for storage permission, for better user feedback if ($todisk) { $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']); my $conf = PVE::QemuConfig->load_config($vmid); # cannot save the state of a non-virtualized PCIe device, so resume cannot really work for my $key (keys %$conf) { next if $key !~ /^hostpci\d+/; die "cannot suspend VM to disk due to passed-through PCI device(s), which lack the" . " possibility to save/restore their internal state\n"; } if (!$statestorage) { # get statestorage from config if none is given my $storecfg = PVE::Storage::config(); $statestorage = PVE::QemuConfig::find_vmstate_storage($conf, $storecfg); } $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']); } my $realcmd = sub { my $upid = shift; syslog('info', "suspend VM $vmid: $upid\n"); PVE::QemuServer::RunState::vm_suspend($vmid, $skiplock, $todisk, $statestorage); return; }; my $taskname = $todisk ? 'qmsuspend' : 'qmpause'; return $rpcenv->fork_worker($taskname, $vmid, $authuser, $realcmd); }, }); __PACKAGE__->register_method({ name => 'vm_resume', path => '{vmid}/status/resume', method => 'POST', protected => 1, proxyto => 'node', description => "Resume virtual machine.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.PowerMgmt']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), skiplock => get_standard_option('skiplock'), nocheck => { type => 'boolean', optional => 1 }, }, }, returns => { type => 'string', }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $skiplock = extract_param($param, 'skiplock'); raise_param_exc({ skiplock => "Only root may use this option." }) if $skiplock && $authuser ne 'root@pam'; # nocheck is used as part of migration when config file might be still # be on source node my $nocheck = extract_param($param, 'nocheck'); raise_param_exc({ nocheck => "Only root may use this option." }) if $nocheck && $authuser ne 'root@pam'; my $to_disk_suspended; eval { PVE::QemuConfig->lock_config( $vmid, sub { my $conf = PVE::QemuConfig->load_config($vmid); $to_disk_suspended = PVE::QemuConfig->has_lock($conf, 'suspended'); }, ); }; die "VM $vmid not running\n" if !$to_disk_suspended && !PVE::QemuServer::check_running($vmid, $nocheck); my $realcmd = sub { my $upid = shift; syslog('info', "resume VM $vmid: $upid\n"); if (!$to_disk_suspended) { PVE::QemuServer::RunState::vm_resume($vmid, $skiplock, $nocheck); } else { my $storecfg = PVE::Storage::config(); PVE::QemuServer::vm_start($storecfg, $vmid, { skiplock => $skiplock }); } return; }; return $rpcenv->fork_worker('qmresume', $vmid, $authuser, $realcmd); }, }); __PACKAGE__->register_method({ name => 'vm_sendkey', path => '{vmid}/sendkey', method => 'PUT', protected => 1, proxyto => 'node', description => "Send key event to virtual machine.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Console']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), skiplock => get_standard_option('skiplock'), key => { description => "The key (qemu monitor encoding).", type => 'string', }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $skiplock = extract_param($param, 'skiplock'); raise_param_exc({ skiplock => "Only root may use this option." }) if $skiplock && $authuser ne 'root@pam'; PVE::QemuServer::vm_sendkey($vmid, $skiplock, $param->{key}); return; }, }); __PACKAGE__->register_method({ name => 'vm_feature', path => '{vmid}/feature', method => 'GET', proxyto => 'node', protected => 1, description => "Check if feature for virtual machine is available.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Audit']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), feature => { description => "Feature to check.", type => 'string', enum => ['snapshot', 'clone', 'copy'], }, snapname => get_standard_option('pve-snapshot-name', { optional => 1, }), }, }, returns => { type => "object", properties => { hasFeature => { type => 'boolean' }, nodes => { type => 'array', items => { type => 'string' }, }, }, }, code => sub { my ($param) = @_; my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $snapname = extract_param($param, 'snapname'); my $feature = extract_param($param, 'feature'); my $running = PVE::QemuServer::check_running($vmid); my $conf = PVE::QemuConfig->load_config($vmid); if ($snapname) { my $snap = $conf->{snapshots}->{$snapname}; die "snapshot '$snapname' does not exist\n" if !defined($snap); $conf = $snap; } my $storecfg = PVE::Storage::config(); my $nodelist = PVE::QemuServer::shared_nodes($conf, $storecfg); my $hasFeature = PVE::QemuConfig->has_feature($feature, $conf, $storecfg, $snapname, $running); return { hasFeature => $hasFeature, nodes => [keys %$nodelist], }; }, }); __PACKAGE__->register_method({ name => 'clone_vm', path => '{vmid}/clone', method => 'POST', protected => 1, proxyto => 'node', description => "Create a copy of virtual machine/template.", permissions => { description => "You need 'VM.Clone' permissions on /vms/{vmid}, and 'VM.Allocate' permissions " . "on /vms/{newid} (or on the VM pool /pool/{pool}). You also need " . "'Datastore.AllocateSpace' on any used storage and 'SDN.Use' on any used bridge/vnet", check => [ 'and', ['perm', '/vms/{vmid}', ['VM.Clone']], [ 'or', ['perm', '/vms/{newid}', ['VM.Allocate']], ['perm', '/pool/{pool}', ['VM.Allocate'], require_param => 'pool'], ], ], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), newid => get_standard_option( 'pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid, description => 'VMID for the clone.', }, ), name => { optional => 1, type => 'string', format => 'dns-name', description => "Set a name for the new VM.", }, description => { optional => 1, type => 'string', description => "Description for the new VM.", }, pool => { optional => 1, type => 'string', format => 'pve-poolid', description => "Add the new VM to the specified pool.", }, snapname => get_standard_option('pve-snapshot-name', { optional => 1, }), storage => get_standard_option( 'pve-storage-id', { description => "Target storage for full clone.", optional => 1, }, ), 'format' => { description => "Target format for file storage. Only valid for full clone.", type => 'string', optional => 1, enum => ['raw', 'qcow2', 'vmdk'], }, full => { optional => 1, type => 'boolean', description => "Create a full copy of all disks. This is always done when " . "you clone a normal VM. For VM templates, we try to create a linked clone by default.", }, target => get_standard_option( 'pve-node', { description => "Target node. Only allowed if the original VM is on shared storage.", optional => 1, }, ), bwlimit => { description => "Override I/O bandwidth limit (in KiB/s).", optional => 1, type => 'integer', minimum => '0', default => 'clone limit from datacenter or storage config', }, }, }, returns => { type => 'string', }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $newid = extract_param($param, 'newid'); my $pool = extract_param($param, 'pool'); my $snapname = extract_param($param, 'snapname'); my $storage = extract_param($param, 'storage'); my $format = extract_param($param, 'format'); my $target = extract_param($param, 'target'); my $localnode = PVE::INotify::nodename(); if ($target && ($target eq $localnode || $target eq 'localhost')) { undef $target; } my $running = PVE::QemuServer::check_running($vmid) || 0; my $load_and_check = sub { $rpcenv->check_pool_exist($pool) if defined($pool); PVE::Cluster::check_node_exists($target) if $target; my $storecfg = PVE::Storage::config(); if ($storage) { # check if storage is enabled on local node and supports vm images my $scfg = PVE::Storage::storage_check_enabled($storecfg, $storage); raise_param_exc({ storage => "storage '$storage' does not support vm images" }) if !$scfg->{content}->{images}; if ($target) { # check if storage is available on target node PVE::Storage::storage_check_enabled($storecfg, $storage, $target); # clone only works if target storage is shared die "can't clone to non-shared storage '$storage'\n" if !$scfg->{shared}; } } PVE::Cluster::check_cfs_quorum(); my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_lock($conf); my $verify_running = PVE::QemuServer::check_running($vmid) || 0; die "unexpected state change\n" if $verify_running != $running; die "snapshot '$snapname' does not exist\n" if $snapname && !defined($conf->{snapshots}->{$snapname}); my $full = $param->{full} // !PVE::QemuConfig->is_template($conf); die "parameter 'storage' not allowed for linked clones\n" if defined($storage) && !$full; die "parameter 'format' not allowed for linked clones\n" if defined($format) && !$full; my $oldconf = $snapname ? $conf->{snapshots}->{$snapname} : $conf; my $sharedvm = &$check_storage_access_clone($rpcenv, $authuser, $storecfg, $oldconf, $storage); PVE::QemuServer::check_mapping_access($rpcenv, $authuser, $oldconf); PVE::QemuServer::check_bridge_access($rpcenv, $authuser, $oldconf); die "can't clone VM to node '$target' (VM uses local storage)\n" if $target && !$sharedvm; my $conffile = PVE::QemuConfig->config_file($newid); die "unable to create VM $newid: config file already exists\n" if -f $conffile; my $newconf = { lock => 'clone' }; my $drives = {}; my $fullclone = {}; my $vollist = []; for my $opt (sort keys %$oldconf) { my $value = $oldconf->{$opt}; # do not copy snapshot related info next if $opt eq 'snapshots' || $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'snapstate' || $opt eq 'runningcpu' || $opt eq 'runningmachine' || $opt eq 'running-nets-host-mtu'; # no need to copy unused images, because VMID(owner) changes anyways next if $opt =~ m/^unused\d+$/; die "cannot clone TPM state while VM is running\n" if $full && $running && !$snapname && $opt eq 'tpmstate0'; # always change MAC! address if ($opt =~ m/^net(\d+)$/) { my $net = PVE::QemuServer::Network::parse_net($value); my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg'); $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}); $newconf->{$opt} = PVE::QemuServer::Network::print_net($net); } elsif (PVE::QemuServer::is_valid_drivename($opt)) { my $drive = PVE::QemuServer::parse_drive($opt, $value); die "unable to parse drive options for '$opt'\n" if !$drive; if (PVE::QemuServer::drive_is_cdrom($drive, 1)) { $newconf->{$opt} = $value; # simply copy configuration } else { my $volid = $drive->{file}; my $msg = "clone feature is not supported for"; $msg .= " a snapshot of" if $snapname; $msg .= " '$volid' ($opt)"; if ( $full || PVE::QemuServer::drive_is_cloudinit($drive) || $opt eq 'tpmstate0' ) { die "Full $msg\n" if !PVE::Storage::volume_has_feature( $storecfg, 'copy', $volid, $snapname, $running, ); $fullclone->{$opt} = 1; } else { # not full means clone instead of copy die "Linked $msg\n" if !PVE::Storage::volume_has_feature( $storecfg, 'clone', $volid, $snapname, $running, ); } $drives->{$opt} = $drive; next if PVE::QemuServer::drive_is_cloudinit($drive); push @$vollist, $volid; } } else { # copy everything else $newconf->{$opt} = $value; } } return ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone); }; my $clonefn = sub { my ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone) = $load_and_check->(); print("creating a clone of VM $vmid with ID $newid\n"); my $storecfg = PVE::Storage::config(); # auto generate a new uuid my $smbios1 = PVE::QemuServer::parse_smbios1($newconf->{smbios1} || ''); $smbios1->{uuid} = PVE::QemuServer::generate_uuid(); $newconf->{smbios1} = PVE::QemuServer::print_smbios1($smbios1); # auto generate a new vmgenid only if the option was set for template if ($newconf->{vmgenid}) { $newconf->{vmgenid} = PVE::QemuServer::generate_uuid(); } delete $newconf->{template}; if ($param->{name}) { $newconf->{name} = $param->{name}; } else { $newconf->{name} = "Copy-of-VM-" . ($oldconf->{name} // $vmid); } if ($param->{description}) { $newconf->{description} = $param->{description}; } # create empty/temp config - this fails if VM already exists on other node # FIXME use PVE::QemuConfig->create_and_lock_config and adapt code PVE::Tools::file_set_contents($conffile, "# qmclone temporary file\nlock: clone\n"); PVE::Firewall::clone_vmfw_conf($vmid, $newid); my $newvollist = []; my $jobs = {}; eval { local $SIG{INT} = local $SIG{TERM} = local $SIG{QUIT} = local $SIG{HUP} = sub { die "interrupted by signal\n"; }; PVE::Storage::activate_volumes($storecfg, $vollist, $snapname); my $bwlimit = extract_param($param, 'bwlimit'); my $total_jobs = scalar(keys %{$drives}); my $i = 1; foreach my $opt (sort keys %$drives) { my $drive = $drives->{$opt}; my $skipcomplete = ($total_jobs != $i); # finish after last drive my $completion = $skipcomplete ? 'skip' : 'complete'; my $src_sid = PVE::Storage::parse_volume_id($drive->{file}); my $storage_list = [$src_sid]; push @$storage_list, $storage if defined($storage); my $clonelimit = PVE::Storage::get_bandwidth_limit('clone', $storage_list, $bwlimit); my $source_info = { vmid => $vmid, running => $running, drivename => $opt, drive => $drive, snapname => $snapname, }; my $dest_info = { vmid => $newid, drivename => $opt, storage => $storage, format => $format, }; $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($oldconf) if $opt eq 'efidisk0'; my $fs_freeze = PVE::QemuServer::Agent::should_fs_freeze($oldconf); my $newdrive = PVE::QemuServer::clone_disk( $storecfg, $source_info, $dest_info, $fullclone->{$opt}, $newvollist, $jobs, $completion, $fs_freeze, $clonelimit, ); $newconf->{$opt} = PVE::QemuServer::print_drive($newdrive); PVE::QemuConfig->write_config($newid, $newconf); $i++; } delete $newconf->{lock}; # do not write pending changes if (my @changes = keys %{ $newconf->{pending} }) { my $pending = join(',', @changes); warn "found pending changes for '$pending', discarding for clone\n"; delete $newconf->{pending}; } PVE::QemuConfig->write_config($newid, $newconf); PVE::QemuServer::Network::create_ifaces_ipams_ips($newconf, $newid); if ($target) { if (!$running) { # always deactivate volumes - avoids that LVM LVs are active on several nodes eval { PVE::Storage::deactivate_volumes($storecfg, $vollist, $snapname); }; # but only warn when that fails (e.g., parallel clones keeping them active) log_warn($@) if $@; } PVE::Storage::deactivate_volumes($storecfg, $newvollist); my $newconffile = PVE::QemuConfig->config_file($newid, $target); die "Failed to move config to node '$target' - rename failed: $!\n" if !rename($conffile, $newconffile); } PVE::AccessControl::add_vm_to_pool($newid, $pool) if $pool; }; if (my $err = $@) { eval { PVE::QemuServer::BlockJob::qemu_blockjobs_cancel(vm_qmp_peer($vmid), $jobs); }; sleep 1; # some storage like rbd need to wait before release volume - really? foreach my $volid (@$newvollist) { eval { PVE::Storage::vdisk_free($storecfg, $volid); }; warn $@ if $@; } PVE::Firewall::remove_vmfw_conf($newid); unlink $conffile; # avoid races -> last thing before die die "clone failed: $err"; } return; }; # Acquire exclusive lock lock for $newid my $lock_target_vm = sub { return PVE::QemuConfig->lock_config_full($newid, 1, $clonefn); }; my $lock_source_vm = sub { # exclusive lock if VM is running - else shared lock is enough; if ($running) { return PVE::QemuConfig->lock_config_full($vmid, 1, $lock_target_vm); } else { return PVE::QemuConfig->lock_config_shared($vmid, 1, $lock_target_vm); } }; $load_and_check->(); # early checks before forking/locking return $rpcenv->fork_worker('qmclone', $vmid, $authuser, $lock_source_vm); }, }); __PACKAGE__->register_method({ name => 'move_vm_disk', path => '{vmid}/move_disk', method => 'POST', protected => 1, proxyto => 'node', description => "Move volume to different storage or to a different VM.", permissions => { description => "You need 'VM.Config.Disk' permissions on /vms/{vmid}, " . "and 'Datastore.AllocateSpace' permissions on the storage. To move " . "a disk to another VM, you need the permissions on the target VM as well.", check => ['perm', '/vms/{vmid}', ['VM.Config.Disk']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), 'target-vmid' => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid, optional => 1, }, ), disk => { type => 'string', description => "The disk you want to move.", enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()], }, storage => get_standard_option( 'pve-storage-id', { description => "Target storage.", completion => \&PVE::QemuServer::complete_storage, optional => 1, }, ), 'format' => { type => 'string', description => "Target Format.", enum => ['raw', 'qcow2', 'vmdk'], optional => 1, }, delete => { type => 'boolean', description => "Delete the original disk after successful copy. By default the" . " original disk is kept as unused disk.", optional => 1, default => 0, }, digest => { type => 'string', description => 'Prevent changes if current configuration file has different SHA1' . ' digest. This can be used to prevent concurrent modifications.', maxLength => 40, optional => 1, }, bwlimit => { description => "Override I/O bandwidth limit (in KiB/s).", optional => 1, type => 'integer', minimum => '0', default => 'move limit from datacenter or storage config', }, 'target-disk' => { type => 'string', description => "The config key the disk will be moved to on the target VM" . " (for example, ide0 or scsi1). Default is the source disk key.", enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()], optional => 1, }, 'target-digest' => { type => 'string', description => 'Prevent changes if the current config file of the target VM has a' . ' different SHA1 digest. This can be used to detect concurrent modifications.', maxLength => 40, optional => 1, }, }, }, returns => { type => 'string', description => "the task ID.", }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $target_vmid = extract_param($param, 'target-vmid'); my $digest = extract_param($param, 'digest'); my $target_digest = extract_param($param, 'target-digest'); my $disk = extract_param($param, 'disk'); my $target_disk = extract_param($param, 'target-disk') // $disk; my $storeid = extract_param($param, 'storage'); my $format = extract_param($param, 'format'); my $storecfg = PVE::Storage::config(); my $load_and_check_move = sub { my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_lock($conf); PVE::Tools::assert_if_modified($digest, $conf->{digest}); die "disk '$disk' does not exist\n" if !$conf->{$disk}; my $drive = PVE::QemuServer::parse_drive($disk, $conf->{$disk}); die "disk '$disk' has no associated volume\n" if !$drive->{file}; die "you can't move a cdrom\n" if PVE::QemuServer::drive_is_cdrom($drive, 1); my $old_volid = $drive->{file}; my ($oldstoreid, $oldvolname) = PVE::Storage::parse_volume_id($old_volid); my $oldfmt = (PVE::Storage::parse_volname($storecfg, $old_volid))[6]; die "you can't move to the same storage with same format\n" if $oldstoreid eq $storeid && (!$format || !$oldfmt || $oldfmt eq $format); my $scfg = PVE::Storage::storage_check_enabled($storecfg, $storeid); raise_param_exc({ storage => "storage '$storeid' does not support vm images" }) if !$scfg->{content}->{images}; # this only checks snapshots because $disk is passed! my $snapshotted = PVE::QemuServer::Drive::is_volume_in_use( $storecfg, $conf, $disk, $old_volid, ); die "you can't move a disk with snapshots and delete the source\n" if $snapshotted && $param->{delete}; return ($conf, $drive, $oldstoreid, $snapshotted); }; my $move_updatefn = sub { my ($conf, $drive, $oldstoreid, $snapshotted) = $load_and_check_move->(); my $old_volid = $drive->{file}; PVE::Cluster::log_msg( 'info', $authuser, "move disk VM $vmid: move --disk $disk --storage $storeid", ); print("copying volume '$old_volid' from current storage '$oldstoreid' to target" . " storage '$storeid'\n"); my $running = PVE::QemuServer::check_running($vmid); PVE::Storage::activate_volumes($storecfg, [$drive->{file}]); my $newvollist = []; eval { local $SIG{INT} = local $SIG{TERM} = local $SIG{QUIT} = local $SIG{HUP} = sub { die "interrupted by signal\n"; }; warn "moving disk with snapshots, snapshots will not be moved!\n" if $snapshotted; my $bwlimit = extract_param($param, 'bwlimit'); my $movelimit = PVE::Storage::get_bandwidth_limit( 'move', [$oldstoreid, $storeid], $bwlimit, ); my $source_info = { vmid => $vmid, running => $running, drivename => $disk, drive => $drive, snapname => undef, }; my $dest_info = { vmid => $vmid, drivename => $disk, storage => $storeid, format => $format, }; $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf) if $disk eq 'efidisk0'; my $newdrive = PVE::QemuServer::clone_disk( $storecfg, $source_info, $dest_info, 1, $newvollist, undef, undef, undef, $movelimit, ); $conf->{$disk} = PVE::QemuServer::print_drive($newdrive); if (!$param->{delete}) { my $unused_key = PVE::QemuConfig->add_unused_volume($conf, $old_volid); print("adding source volume '$old_volid' as '$unused_key' to config\n"); } # convert moved disk to base if part of template PVE::QemuServer::template_create($vmid, $conf, $disk) if PVE::QemuConfig->is_template($conf); PVE::QemuConfig->write_config($vmid, $conf); my $do_trim = PVE::QemuServer::Agent::get_qga_key($conf, 'fstrim_cloned_disks'); if ($running && $do_trim && PVE::QemuServer::Agent::qga_check_running($vmid)) { eval { mon_cmd($vmid, "guest-fstrim") }; } eval { # try to deactivate volumes - avoid lvm LVs to be active on several nodes PVE::Storage::deactivate_volumes($storecfg, [$newdrive->{file}]) if !$running; }; warn $@ if $@; }; if (my $err = $@) { foreach my $volid (@$newvollist) { eval { PVE::Storage::vdisk_free($storecfg, $volid) }; warn $@ if $@; } die "storage migration failed: $err"; } if ($param->{delete}) { print("removing source volume '$old_volid' after successful copy\n"); eval { PVE::Storage::deactivate_volumes($storecfg, [$old_volid]); PVE::Storage::vdisk_free($storecfg, $old_volid); }; warn $@ if $@; } }; my $load_and_check_reassign_configs = sub { my $vmlist = PVE::Cluster::get_vmlist()->{ids}; die "could not find VM ${vmid}\n" if !exists($vmlist->{$vmid}); die "could not find target VM ${target_vmid}\n" if !exists($vmlist->{$target_vmid}); my $source_node = $vmlist->{$vmid}->{node}; my $target_node = $vmlist->{$target_vmid}->{node}; die "Both VMs need to be on the same node ($source_node != $target_node)\n" if $source_node ne $target_node; my $source_conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_lock($source_conf); my $target_conf = PVE::QemuConfig->load_config($target_vmid); PVE::QemuConfig->check_lock($target_conf); die "Can't move disks from or to template VMs\n" if ($source_conf->{template} || $target_conf->{template}); if ($digest) { eval { PVE::Tools::assert_if_modified($digest, $source_conf->{digest}) }; die "VM ${vmid}: $@" if $@; } if ($target_digest) { eval { PVE::Tools::assert_if_modified($target_digest, $target_conf->{digest}) }; die "VM ${target_vmid}: $@" if $@; } die "Disk '${disk}' for VM '$vmid' does not exist\n" if !defined($source_conf->{$disk}); die "Target disk key '${target_disk}' is already in use for VM '$target_vmid'\n" if $target_conf->{$target_disk}; my $drive = PVE::QemuServer::parse_drive( $disk, $source_conf->{$disk}, ); die "failed to parse source disk - $@\n" if !$drive; my $source_volid = $drive->{file}; die "disk '${disk}' has no associated volume\n" if !$source_volid; die "CD drive contents can't be moved to another VM\n" if PVE::QemuServer::drive_is_cdrom($drive, 1); my $storeid = PVE::Storage::parse_volume_id($source_volid, 1); die "Volume '$source_volid' not managed by PVE\n" if !defined($storeid); die "Can't move disk used by a snapshot to another VM\n" if PVE::QemuServer::Drive::is_volume_in_use( $storecfg, $source_conf, $disk, $source_volid, ); die "Storage does not support moving of this disk to another VM\n" if (!PVE::Storage::volume_has_feature($storecfg, 'rename', $source_volid)); die "Cannot move disk to another VM while the source VM is running - detach first\n" if PVE::QemuServer::check_running($vmid) && $disk !~ m/^unused\d+$/; # now re-parse using target disk slot format if ($target_disk =~ /^unused\d+$/) { $drive = PVE::QemuServer::parse_drive( $target_disk, $source_volid, ); } else { $drive = PVE::QemuServer::parse_drive( $target_disk, $source_conf->{$disk}, ); } die "failed to parse source disk for target disk format - $@\n" if !$drive; my $repl_conf = PVE::ReplicationConfig->new(); if ($repl_conf->check_for_existing_jobs($target_vmid, 1)) { my $format = (PVE::Storage::parse_volname($storecfg, $source_volid))[6]; die "Cannot move disk to a replicated VM. Storage does not support replication!\n" if !PVE::Storage::storage_can_replicate($storecfg, $storeid, $format); } return ($source_conf, $target_conf, $drive); }; my $logfunc = sub { my ($msg) = @_; print STDERR "$msg\n"; }; my $disk_reassignfn = sub { return PVE::QemuConfig->lock_config( $vmid, sub { return PVE::QemuConfig->lock_config( $target_vmid, sub { my ($source_conf, $target_conf, $drive) = &$load_and_check_reassign_configs(); my $source_volid = $drive->{file}; print("reassign disk '$disk' from VM '$vmid' as '$target_disk' to" . " VM '$target_vmid'\n"); my ($storeid, $source_volname) = PVE::Storage::parse_volume_id($source_volid); my $fmt = (PVE::Storage::parse_volname($storecfg, $source_volid))[6]; my $new_volid = PVE::Storage::rename_volume( $storecfg, $source_volid, $target_vmid, ); $drive->{file} = $new_volid; my $boot_order = PVE::QemuServer::device_bootorder($source_conf); if (defined(delete $boot_order->{$disk})) { print "removing disk '$disk' from boot order config\n"; my $boot_devs = [ sort { $boot_order->{$a} <=> $boot_order->{$b} } keys %$boot_order ]; $source_conf->{boot} = PVE::QemuServer::print_bootorder($boot_devs); } delete $source_conf->{$disk}; print "removing disk '${disk}' from VM '${vmid}' config\n"; PVE::QemuConfig->write_config($vmid, $source_conf); my $drive_string = PVE::QemuServer::print_drive($drive); if ($target_disk =~ /^unused\d+$/) { $target_conf->{$target_disk} = $drive_string; PVE::QemuConfig->write_config($target_vmid, $target_conf); } else { &$update_vm_api( { node => $node, vmid => $target_vmid, digest => $target_digest, $target_disk => $drive_string, }, 1, ); } # remove possible replication snapshots if ( PVE::Storage::volume_has_feature( $storecfg, 'replicate', $source_volid, ), ) { eval { PVE::Replication::prepare( $storecfg, [$new_volid], undef, 1, undef, $logfunc, ); }; if (my $err = $@) { print "Failed to remove replication snapshots on moved disk " . "'$target_disk'. Manual cleanup could be necessary.\n"; } } }, ); }, ); }; if ($target_vmid && $storeid) { my $msg = "either set 'storage' or 'target-vmid', but not both"; raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg }); } elsif ($target_vmid) { $rpcenv->check_vm_perm($authuser, $target_vmid, undef, ['VM.Config.Disk']) if $authuser ne 'root@pam'; raise_param_exc( { 'target-vmid' => "must be different than source VMID to reassign disk" }) if $vmid eq $target_vmid; my (undef, undef, $drive) = &$load_and_check_reassign_configs(); my $storage = PVE::Storage::parse_volume_id($drive->{file}); $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']); return $rpcenv->fork_worker( 'qmmove', "${vmid}-${disk}>${target_vmid}-${target_disk}", $authuser, $disk_reassignfn, ); } elsif ($storeid) { $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']); $load_and_check_move->(); # early checks before forking/locking my $realcmd = sub { PVE::QemuConfig->lock_config($vmid, $move_updatefn); }; return $rpcenv->fork_worker('qmmove', $vmid, $authuser, $realcmd); } else { my $msg = "both 'storage' and 'target-vmid' missing, either needs to be set"; raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg }); } }, }); my $check_vm_disks_local = sub { my ($storecfg, $vmconf, $vmid) = @_; my $local_disks = {}; # add some more information to the disks e.g. cdrom PVE::QemuServer::foreach_volid( $vmconf, sub { my ($volid, $attr) = @_; my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); if ($storeid) { my $scfg = PVE::Storage::storage_config($storecfg, $storeid); return if $scfg->{shared}; } # The shared attr here is just a special case where the vdisk # is marked as shared manually return if $attr->{shared}; return if $attr->{cdrom} and $volid eq "none"; if (exists $local_disks->{$volid}) { @{ $local_disks->{$volid} }{ keys %$attr } = values %$attr; } else { $local_disks->{$volid} = $attr; # ensure volid is present in case it's needed $local_disks->{$volid}->{volid} = $volid; } }, ); return $local_disks; }; __PACKAGE__->register_method({ name => 'migrate_vm_precondition', path => '{vmid}/migrate', method => 'GET', protected => 1, proxyto => 'node', description => "Get preconditions for migration.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Migrate']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), target => get_standard_option( 'pve-node', { description => "Target node.", completion => \&PVE::Cluster::complete_migration_target, optional => 1, }, ), }, }, returns => { # TODO 9.x: rework the api call to return more sensible structures # e.g. a simple list of nodes with their blockers and/or notices to show type => "object", properties => { running => { type => 'boolean', description => "Determines if the VM is running.", }, allowed_nodes => { type => 'array', items => { type => 'string', description => "An allowed node", }, optional => 1, description => "List of nodes allowed for migration.", }, not_allowed_nodes => { type => 'object', optional => 1, properties => { unavailable_storages => { type => 'array', optional => 1, description => 'A list of not available storages.', items => { type => 'string', description => 'A storage', }, }, 'blocking-ha-resources' => { description => "HA resources, which are blocking the" . " VM from being migrated to the node.", type => 'array', optional => 1, items => { description => "A blocking HA resource", type => 'object', properties => { sid => { type => 'string', description => "The blocking HA resource id", }, cause => { type => 'string', description => "The reason why the HA" . " resource is blocking the migration.", enum => ['resource-affinity'], }, }, }, }, }, description => "List of not allowed nodes with additional information.", }, local_disks => { type => 'array', items => { type => 'object', properties => { size => { type => 'integer', description => 'The size of the disk in bytes.', }, volid => { type => 'string', description => 'The volid of the disk.', }, cdrom => { type => 'boolean', description => 'True if the disk is a cdrom.', }, is_unused => { type => 'boolean', description => 'True if the disk is unused.', }, }, }, description => "List local disks including CD-Rom, unused and not referenced disks", }, local_resources => { type => 'array', items => { type => 'string', description => "A local resource", }, description => "List local resources (e.g. pci, usb) that block migration.", }, # FIXME: remove with 9.0 'mapped-resources' => { type => 'array', items => { type => 'string', description => "A mapped resource", }, description => "List of mapped resources e.g. pci, usb. Deprecated, use 'mapped-resource-info' instead.", }, 'mapped-resource-info' => { type => 'object', description => "Object of mapped resources with additional information such if they're live migratable.", }, 'has-dbus-vmstate' => { type => 'boolean', description => 'Whether the VM host supports migrating additional VM state, ' . 'such as conntrack entries.', }, 'dependent-ha-resources' => { description => "HA resources, which will be migrated to the same target node as" . " the VM, because these are in positive affinity with the VM.", type => 'array', optional => 1, items => { type => 'string', description => "The ':' resource IDs of a HA resource with a" . " positive affinity rule to this VM.", }, }, }, }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); PVE::Cluster::check_cfs_quorum(); my $res = {}; my $vmid = extract_param($param, 'vmid'); my $target = extract_param($param, 'target'); my $localnode = PVE::INotify::nodename(); # test if VM exists my $vmconf = PVE::QemuConfig->load_config($vmid); my $storecfg = PVE::Storage::config(); # try to detect errors early PVE::QemuConfig->check_lock($vmconf); $res->{running} = PVE::QemuServer::check_running($vmid) ? 1 : 0; my ($local_resources, $mapped_resources, $missing_mappings_by_node) = PVE::QemuMigrate::Helpers::check_local_resources($vmconf, $res->{running}, 1); my $vga = PVE::QemuServer::parse_vga($vmconf->{vga}); if ($res->{running} && $vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') { my $machine_version = PVE::QemuServer::Machine::get_current_qemu_machine($vmid); push $local_resources->@*, "clipboard=vnc" if !PVE::QemuServer::Machine::is_machine_version_at_least($machine_version, 10, 1); } $res->{allowed_nodes} = []; $res->{not_allowed_nodes} = {}; my $storage_nodehash = PVE::QemuServer::check_local_storage_availability($vmconf, $storecfg); my $blocking_ha_resources_by_node = {}; if (PVE::HA::Config::vm_is_ha_managed($vmid)) { (my $dependent_ha_resource, $blocking_ha_resources_by_node) = PVE::HA::Config::get_resource_motion_info("vm:$vmid"); $res->{'dependent-ha-resources'} = $dependent_ha_resource // []; } my $nodelist = PVE::Cluster::get_nodelist(); for my $node ($nodelist->@*) { next if $node eq $localnode; # extract missing storage info if (my $storage_info = $storage_nodehash->{$node}) { $res->{not_allowed_nodes}->{$node} = $storage_info; } # extract missing mappings info my $missing_mappings = $missing_mappings_by_node->{$node}; if (scalar($missing_mappings->@*)) { $res->{not_allowed_nodes}->{$node}->{'unavailable-resources'} = $missing_mappings; } # extracting blocking resources for current node if (my $blocking_ha_resources = $blocking_ha_resources_by_node->{$node}) { $res->{not_allowed_nodes}->{$node}->{'blocking-ha-resources'} = $blocking_ha_resources; } # if nothing came up, add it to the allowed nodes if (scalar($res->{not_allowed_nodes}->{$node}->%*) == 0) { push $res->{allowed_nodes}->@*, $node; } } my $local_disks = &$check_vm_disks_local($storecfg, $vmconf, $vmid); $res->{local_disks} = [values %$local_disks]; $res->{local_resources} = $local_resources; $res->{'mapped-resources'} = [sort keys $mapped_resources->%*]; $res->{'mapped-resource-info'} = $mapped_resources; $res->{'has-dbus-vmstate'} = 1; return $res; }, }); __PACKAGE__->register_method({ name => 'migrate_vm', path => '{vmid}/migrate', method => 'POST', protected => 1, proxyto => 'node', description => "Migrate virtual machine. Creates a new migration task.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Migrate']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), target => get_standard_option( 'pve-node', { description => "Target node.", completion => \&PVE::Cluster::complete_migration_target, }, ), online => { type => 'boolean', description => "Use online/live migration if VM is running. Ignored if VM is stopped.", optional => 1, }, force => { type => 'boolean', description => "Allow to migrate VMs which use local devices. Only root may use this option.", optional => 1, }, migration_type => { type => 'string', enum => ['secure', 'insecure'], description => "Migration traffic is encrypted using an SSH tunnel by default. On secure, completely private networks this can be disabled to increase performance.", optional => 1, }, migration_network => { type => 'string', format => 'CIDR', description => "CIDR of the (sub) network that is used for migration.", optional => 1, }, "with-local-disks" => { type => 'boolean', description => "Enable live storage migration for local disk", optional => 1, }, targetstorage => get_standard_option( 'pve-targetstorage', { completion => \&PVE::QemuServer::complete_migration_storage, }, ), bwlimit => { description => "Override I/O bandwidth limit (in KiB/s).", optional => 1, type => 'integer', minimum => '0', default => 'migrate limit from datacenter or storage config', }, 'with-conntrack-state' => { type => 'boolean', optional => 1, default => 0, description => 'Whether to migrate conntrack entries for running VMs.', }, }, }, returns => { type => 'string', description => "the task ID.", }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $target = extract_param($param, 'target'); my $localnode = PVE::INotify::nodename(); raise_param_exc({ target => "target is local node." }) if $target eq $localnode; PVE::Cluster::check_cfs_quorum(); PVE::Cluster::check_node_exists($target); my $targetip = PVE::Cluster::remote_node_ip($target); my $vmid = extract_param($param, 'vmid'); raise_param_exc({ force => "Only root may use this option." }) if $param->{force} && $authuser ne 'root@pam'; raise_param_exc({ migration_type => "Only root may use this option." }) if $param->{migration_type} && $authuser ne 'root@pam'; # allow root only until better network permissions are available raise_param_exc({ migration_network => "Only root may use this option." }) if $param->{migration_network} && $authuser ne 'root@pam'; # test if VM exists my $conf = PVE::QemuConfig->load_config($vmid); # try to detect errors early PVE::QemuConfig->check_lock($conf); if (PVE::QemuServer::check_running($vmid)) { die "can't migrate running VM without --online\n" if !$param->{online}; my $repl_conf = PVE::ReplicationConfig->new(); my $is_replicated = $repl_conf->check_for_existing_jobs($vmid, 1); my $is_replicated_to_target = defined($repl_conf->find_local_replication_job($vmid, $target)); if (!$param->{force} && $is_replicated && !$is_replicated_to_target) { die "Cannot live-migrate replicated VM to node '$target' - not a replication " . "target. Use 'force' to override.\n"; } } else { warn "VM isn't running. Doing offline migration instead.\n" if $param->{online}; $param->{online} = 0; $param->{'with-conntrack-state'} = 0; } my $storecfg = PVE::Storage::config(); if (my $targetstorage = $param->{targetstorage}) { my $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') }; raise_param_exc({ targetstorage => "failed to parse storage map: $@" }) if $@; $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']) if !defined($storagemap->{identity}); foreach my $target_sid (values %{ $storagemap->{entries} }) { $check_storage_access_migrate->( $rpcenv, $authuser, $storecfg, $target_sid, $target, ); } $check_storage_access_migrate->( $rpcenv, $authuser, $storecfg, $storagemap->{default}, $target, ) if $storagemap->{default}; PVE::QemuServer::check_storage_availability($storecfg, $conf, $target) if $storagemap->{identity}; $param->{storagemap} = $storagemap; } else { PVE::QemuServer::check_storage_availability($storecfg, $conf, $target); } if (PVE::HA::Config::vm_is_ha_managed($vmid) && $rpcenv->{type} ne 'ha') { my $hacmd = sub { my $upid = shift; print "Requesting HA migration for VM $vmid to node $target\n"; my $cmd = ['ha-manager', 'migrate', "vm:$vmid", $target]; run_command($cmd); return; }; return $rpcenv->fork_worker('hamigrate', $vmid, $authuser, $hacmd); } else { my $realcmd = sub { PVE::QemuMigrate->migrate($target, $targetip, $vmid, $param); }; my $worker = sub { return PVE::GuestHelpers::guest_migration_lock($vmid, 10, $realcmd); }; return $rpcenv->fork_worker('qmigrate', $vmid, $authuser, $worker); } }, }); __PACKAGE__->register_method({ name => 'remote_migrate_vm', path => '{vmid}/remote_migrate', method => 'POST', protected => 1, proxyto => 'node', description => "Migrate virtual machine to a remote cluster. Creates a new migration task. EXPERIMENTAL feature!", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Migrate']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), 'target-vmid' => get_standard_option('pve-vmid', { optional => 1 }), 'target-endpoint' => get_standard_option('proxmox-remote', { description => "Remote target endpoint", }), online => { type => 'boolean', description => "Use online/live migration if VM is running. Ignored if VM is stopped.", optional => 1, }, delete => { type => 'boolean', description => "Delete the original VM and related data after successful migration. By default the original VM is kept on the source cluster in a stopped state.", optional => 1, default => 0, }, 'target-storage' => get_standard_option( 'pve-targetstorage', { completion => \&PVE::QemuServer::complete_migration_storage, optional => 0, }, ), 'target-bridge' => { type => 'string', description => "Mapping from source to target bridges. Providing only a single bridge ID maps all source bridges to that bridge. Providing the special value '1' will map each source bridge to itself.", format => 'bridge-pair-list', }, bwlimit => { description => "Override I/O bandwidth limit (in KiB/s).", optional => 1, type => 'integer', minimum => '0', default => 'migrate limit from datacenter or storage config', }, }, }, returns => { type => 'string', description => "the task ID.", }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $source_vmid = extract_param($param, 'vmid'); my $target_endpoint = extract_param($param, 'target-endpoint'); my $target_vmid = extract_param($param, 'target-vmid') // $source_vmid; my $delete = extract_param($param, 'delete') // 0; PVE::Cluster::check_cfs_quorum(); # test if VM exists my $conf = PVE::QemuConfig->load_config($source_vmid); PVE::QemuConfig->check_lock($conf); raise_param_exc({ vmid => "cannot remote-migrate VM that is configured for HA" }) if PVE::HA::Config::service_is_configured("vm:$source_vmid"); my $remote = PVE::JSONSchema::parse_property_string('proxmox-remote', $target_endpoint); # TODO: move this as helper somewhere appropriate? my $conn_args = { protocol => 'https', host => $remote->{host}, port => $remote->{port} // 8006, apitoken => $remote->{apitoken}, }; my $fp; if ($fp = $remote->{fingerprint}) { $conn_args->{cached_fingerprints} = { uc($fp) => 1 }; } print "Establishing API connection with remote at '$remote->{host}'\n"; my $api_client = PVE::APIClient::LWP->new(%$conn_args); if (!defined($fp)) { my $cert_info = $api_client->get("/nodes/localhost/certificates/info"); foreach my $cert (@$cert_info) { my $filename = $cert->{filename}; next if $filename ne 'pveproxy-ssl.pem' && $filename ne 'pve-ssl.pem'; $fp = $cert->{fingerprint} if !$fp || $filename eq 'pveproxy-ssl.pem'; } $conn_args->{cached_fingerprints} = { uc($fp) => 1 } if defined($fp); } my $repl_conf = PVE::ReplicationConfig->new(); my $is_replicated = $repl_conf->check_for_existing_jobs($source_vmid, 1); die "cannot remote-migrate replicated VM\n" if $is_replicated; if (PVE::QemuServer::check_running($source_vmid)) { die "can't migrate running VM without --online\n" if !$param->{online}; } else { warn "VM isn't running. Doing offline migration instead.\n" if $param->{online}; $param->{online} = 0; } my $storecfg = PVE::Storage::config(); my $target_storage = extract_param($param, 'target-storage'); my $storagemap = eval { PVE::JSONSchema::parse_idmap($target_storage, 'pve-storage-id') }; raise_param_exc({ 'target-storage' => "failed to parse storage map: $@" }) if $@; my $target_bridge = extract_param($param, 'target-bridge'); my $bridgemap = eval { PVE::JSONSchema::parse_idmap($target_bridge, 'pve-bridge-id') }; raise_param_exc({ 'target-bridge' => "failed to parse bridge map: $@" }) if $@; die "remote migration requires explicit storage mapping!\n" if $storagemap->{identity}; $param->{storagemap} = $storagemap; $param->{bridgemap} = $bridgemap; $param->{remote} = { conn => $conn_args, # re-use fingerprint for tunnel client => $api_client, vmid => $target_vmid, }; $param->{migration_type} = 'websocket'; $param->{'with-local-disks'} = 1; $param->{delete} = $delete if $delete; my $cluster_status = $api_client->get("/cluster/status"); my $target_node; foreach my $entry (@$cluster_status) { next if $entry->{type} ne 'node'; if ($entry->{local}) { $target_node = $entry->{name}; last; } } die "couldn't determine endpoint's node name\n" if !defined($target_node); my $realcmd = sub { PVE::QemuMigrate->migrate($target_node, $remote->{host}, $source_vmid, $param); }; my $worker = sub { return PVE::GuestHelpers::guest_migration_lock($source_vmid, 10, $realcmd); }; return $rpcenv->fork_worker('qmigrate', $source_vmid, $authuser, $worker); }, }); __PACKAGE__->register_method({ name => 'monitor', path => '{vmid}/monitor', method => 'POST', protected => 1, proxyto => 'node', description => "Execute QEMU monitor commands.", permissions => { description => PVE::API2::Qemu::HMPPerms::generate_description(), check => ['perm', '/vms/{vmid}', ['Sys.Audit', 'Sys.Modify'], any => 1], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), command => { type => 'string', description => "The monitor command.", }, }, }, returns => { type => 'string' }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $command = $param->{command} or die "no command specified\n"; die "unexpected command '$command'\n" if $command !~ m/^\s*(\S+)/; my $command_name = $1; my $required_perm = $PVE::API2::Qemu::HMPPerms::hmp_command_perms->{$command_name}; if (!$required_perm) { my $msg = "command '$command_name' non-existent or not assigned a required permission" . " yet, limiting to root user\n"; die $msg if $authuser ne 'root@pam'; warn $msg; $required_perm = 'root'; } if ($required_perm eq 'root') { die "root-only command '$command_name'\n" if $authuser ne 'root@pam'; } elsif ($required_perm eq 'Sys.Modify') { $rpcenv->check_full($authuser, "/", ['Sys.Modify']); } elsif ($required_perm eq 'none') { # nothing to check } else { die "unexpected required permission '$required_perm' for command '$command_name'\n"; } my $vmid = $param->{vmid}; my $conf = PVE::QemuConfig->load_config($vmid); # check if VM exists my $res = ''; eval { $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, $param->{command}, 25); }; $res = "ERROR: $@" if $@; return $res; }, }); __PACKAGE__->register_method({ name => 'resize_vm', path => '{vmid}/resize', method => 'PUT', protected => 1, proxyto => 'node', description => "Extend volume size.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Config.Disk']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), skiplock => get_standard_option('skiplock'), disk => { type => 'string', description => "The disk you want to resize.", enum => [PVE::QemuServer::Drive::valid_drive_names()], }, size => { type => 'string', pattern => '\+?\d+(\.\d+)?[KMGT]?', description => "The new size. With the `+` sign the value is added to the actual size of the volume and without it, the value is taken as an absolute one. Shrinking disk size is not supported.", }, digest => { type => 'string', description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.', maxLength => 40, optional => 1, }, }, }, returns => { type => 'string', description => "the task ID.", }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $digest = extract_param($param, 'digest'); my $disk = extract_param($param, 'disk'); my $sizestr = extract_param($param, 'size'); my $skiplock = extract_param($param, 'skiplock'); raise_param_exc({ skiplock => "Only root may use this option." }) if $skiplock && $authuser ne 'root@pam'; my $storecfg = PVE::Storage::config(); my $updatefn = sub { my $conf = PVE::QemuConfig->load_config($vmid); die "checksum mismatch (file change by other user?)\n" if $digest && $digest ne $conf->{digest}; PVE::QemuConfig->check_lock($conf) if !$skiplock; die "disk '$disk' does not exist\n" if !$conf->{$disk}; my $drive = PVE::QemuServer::parse_drive($disk, $conf->{$disk}); my (undef, undef, undef, undef, undef, undef, $format) = PVE::Storage::parse_volname($storecfg, $drive->{file}); my $volid = $drive->{file}; die "disk '$disk' has no associated volume\n" if !$volid; die "you can't resize a cdrom\n" if PVE::QemuServer::drive_is_cdrom($drive); my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']); PVE::Storage::activate_volumes($storecfg, [$volid]); my $size = PVE::Storage::volume_size_info($storecfg, $volid, 5); die "Could not determine current size of volume '$volid'\n" if !defined($size); die "internal error" if $sizestr !~ m/^(\+)?(\d+(\.\d+)?)([KMGT])?$/; my ($ext, $newsize, $unit) = ($1, $2, $4); if ($unit) { if ($unit eq 'K') { $newsize = $newsize * 1024; } elsif ($unit eq 'M') { $newsize = $newsize * 1024 * 1024; } elsif ($unit eq 'G') { $newsize = $newsize * 1024 * 1024 * 1024; } elsif ($unit eq 'T') { $newsize = $newsize * 1024 * 1024 * 1024 * 1024; } } $newsize += $size if $ext; $newsize = int($newsize); die "shrinking disks is not supported\n" if $newsize < $size; return if $size == $newsize; PVE::Cluster::log_msg( 'info', $authuser, "update VM $vmid: resize --disk $disk --size $sizestr", ); PVE::QemuServer::Blockdev::resize( $vmid, "drive-$disk", $storecfg, $volid, $newsize, ); $drive->{size} = $newsize; $conf->{$disk} = PVE::QemuServer::print_drive($drive); PVE::QemuConfig->write_config($vmid, $conf); }; my $worker = sub { PVE::QemuConfig->lock_config($vmid, $updatefn); }; return $rpcenv->fork_worker('resize', $vmid, $authuser, $worker); }, }); __PACKAGE__->register_method({ name => 'snapshot_list', path => '{vmid}/snapshot', method => 'GET', description => "List all snapshots.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Audit']], }, proxyto => 'node', protected => 1, # qemu pid files are only readable by root parameters => { additionalProperties => 0, properties => { vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), node => get_standard_option('pve-node'), }, }, returns => { type => 'array', items => { type => "object", properties => { name => { description => "Snapshot identifier. Value 'current' identifies the current VM.", type => 'string', }, vmstate => { description => "Snapshot includes RAM.", type => 'boolean', optional => 1, }, description => { description => "Snapshot description.", type => 'string', }, snaptime => { description => "Snapshot creation time", type => 'integer', renderer => 'timestamp', optional => 1, }, parent => { description => "Parent snapshot identifier.", type => 'string', optional => 1, }, }, }, links => [{ rel => 'child', href => "{name}" }], }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; my $conf = PVE::QemuConfig->load_config($vmid); my $snaphash = $conf->{snapshots} || {}; my $res = []; foreach my $name (keys %$snaphash) { my $d = $snaphash->{$name}; my $item = { name => $name, snaptime => $d->{snaptime} || 0, vmstate => $d->{vmstate} ? 1 : 0, description => $d->{description} || '', }; $item->{parent} = $d->{parent} if $d->{parent}; $item->{snapstate} = $d->{snapstate} if $d->{snapstate}; push @$res, $item; } my $running = PVE::QemuServer::check_running($vmid, 1) ? 1 : 0; my $current = { name => 'current', digest => $conf->{digest}, running => $running, description => "You are here!", }; $current->{parent} = $conf->{parent} if $conf->{parent}; push @$res, $current; return $res; }, }); __PACKAGE__->register_method({ name => 'snapshot', path => '{vmid}/snapshot', method => 'POST', protected => 1, proxyto => 'node', description => "Snapshot a VM.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Snapshot']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), snapname => get_standard_option('pve-snapshot-name'), vmstate => { optional => 1, type => 'boolean', description => "Save the vmstate", }, description => { optional => 1, type => 'string', description => "A textual description or comment.", }, }, }, returns => { type => 'string', description => "the task ID.", }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $snapname = extract_param($param, 'snapname'); die "unable to use snapshot name 'current' (reserved name)\n" if $snapname eq 'current'; die "unable to use snapshot name 'pending' (reserved name)\n" if lc($snapname) eq 'pending'; my $vmconf = PVE::QemuConfig->load_config($vmid); PVE::QemuMigrate::Helpers::check_non_migratable_resources($vmconf, $param->{vmstate}, 0); my $realcmd = sub { PVE::Cluster::log_msg('info', $authuser, "snapshot VM $vmid: $snapname"); PVE::QemuConfig->snapshot_create( $vmid, $snapname, $param->{vmstate}, $param->{description}, ); }; return $rpcenv->fork_worker('qmsnapshot', $vmid, $authuser, $realcmd); }, }); __PACKAGE__->register_method({ name => 'snapshot_cmd_idx', path => '{vmid}/snapshot/{snapname}', description => '', method => 'GET', permissions => { user => 'all', }, parameters => { additionalProperties => 0, properties => { vmid => get_standard_option('pve-vmid'), node => get_standard_option('pve-node'), snapname => get_standard_option('pve-snapshot-name'), }, }, returns => { type => 'array', items => { type => "object", properties => {}, }, links => [{ rel => 'child', href => "{cmd}" }], }, code => sub { my ($param) = @_; my $res = []; push @$res, { cmd => 'rollback' }; push @$res, { cmd => 'config' }; return $res; }, }); __PACKAGE__->register_method({ name => 'update_snapshot_config', path => '{vmid}/snapshot/{snapname}/config', method => 'PUT', protected => 1, proxyto => 'node', description => "Update snapshot metadata.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Snapshot']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), snapname => get_standard_option('pve-snapshot-name'), description => { optional => 1, type => 'string', description => "A textual description or comment.", }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $vmid = extract_param($param, 'vmid'); my $snapname = extract_param($param, 'snapname'); return if !defined($param->{description}); my $updatefn = sub { my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_lock($conf); my $snap = $conf->{snapshots}->{$snapname}; die "snapshot '$snapname' does not exist\n" if !defined($snap); $snap->{description} = $param->{description} if defined($param->{description}); PVE::QemuConfig->write_config($vmid, $conf); }; PVE::QemuConfig->lock_config($vmid, $updatefn); return; }, }); __PACKAGE__->register_method({ name => 'get_snapshot_config', path => '{vmid}/snapshot/{snapname}/config', method => 'GET', proxyto => 'node', description => "Get snapshot configuration", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Snapshot', 'VM.Snapshot.Rollback', 'VM.Audit'], any => 1], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), snapname => get_standard_option('pve-snapshot-name'), }, }, returns => { type => "object" }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $vmid = extract_param($param, 'vmid'); my $snapname = extract_param($param, 'snapname'); my $conf = PVE::QemuConfig->load_config($vmid); my $snap = $conf->{snapshots}->{$snapname}; die "snapshot '$snapname' does not exist\n" if !defined($snap); return $snap; }, }); __PACKAGE__->register_method({ name => 'rollback', path => '{vmid}/snapshot/{snapname}/rollback', method => 'POST', protected => 1, proxyto => 'node', description => "Rollback VM state to specified snapshot.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Snapshot', 'VM.Snapshot.Rollback'], any => 1], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), snapname => get_standard_option('pve-snapshot-name'), start => { type => 'boolean', description => "Whether the VM should get started after rolling back successfully." . " (Note: VMs will be automatically started if the snapshot includes RAM.)", optional => 1, default => 0, }, }, }, returns => { type => 'string', description => "the task ID.", }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $snapname = extract_param($param, 'snapname'); my $realcmd = sub { PVE::Cluster::log_msg('info', $authuser, "rollback snapshot VM $vmid: $snapname"); PVE::QemuConfig->snapshot_rollback($vmid, $snapname); if ($param->{start} && !PVE::QemuServer::Helpers::vm_running_locally($vmid)) { PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node }); } }; my $worker = sub { # hold migration lock, this makes sure that nobody create replication snapshots return PVE::GuestHelpers::guest_migration_lock($vmid, 10, $realcmd); }; return $rpcenv->fork_worker('qmrollback', $vmid, $authuser, $worker); }, }); __PACKAGE__->register_method({ name => 'delsnapshot', path => '{vmid}/snapshot/{snapname}', method => 'DELETE', protected => 1, proxyto => 'node', description => "Delete a VM snapshot.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Snapshot']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), snapname => get_standard_option('pve-snapshot-name'), force => { optional => 1, type => 'boolean', description => "For removal from config file, even if removing disk snapshots fails.", }, }, }, returns => { type => 'string', description => "the task ID.", }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $snapname = extract_param($param, 'snapname'); my $lock_obtained; my $do_delete = sub { $lock_obtained = 1; PVE::QemuConfig->lock_config( $vmid, sub { my $conf = PVE::QemuConfig->load_config($vmid); assert_tpm_snapshot_delete_possible( $vmid, $conf, $conf->{snapshots}->{$snapname}, $snapname, ); }, ); PVE::Cluster::log_msg('info', $authuser, "delete snapshot VM $vmid: $snapname"); PVE::QemuConfig->snapshot_delete($vmid, $snapname, $param->{force}); }; my $realcmd = sub { if ($param->{force}) { $do_delete->(); } else { eval { PVE::GuestHelpers::guest_migration_lock($vmid, 10, $do_delete); }; if (my $err = $@) { die $err if $lock_obtained; die "Failed to obtain guest migration lock - replication running?\n"; } } }; return $rpcenv->fork_worker('qmdelsnapshot', $vmid, $authuser, $realcmd); }, }); __PACKAGE__->register_method({ name => 'template', path => '{vmid}/template', method => 'POST', protected => 1, proxyto => 'node', description => "Create a Template.", permissions => { description => "You need 'VM.Allocate' permissions on /vms/{vmid}", check => ['perm', '/vms/{vmid}', ['VM.Allocate']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }, ), disk => { optional => 1, type => 'string', description => "If you want to convert only 1 disk to base image.", enum => [PVE::QemuServer::Drive::valid_drive_names()], }, }, }, returns => { type => 'string', description => "the task ID.", }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $disk = extract_param($param, 'disk'); my $load_and_check = sub { my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_lock($conf); die "unable to create template, because VM contains snapshots\n" if $conf->{snapshots} && scalar(keys %{ $conf->{snapshots} }); die "you can't convert a template to a template\n" if PVE::QemuConfig->is_template($conf) && !$disk; die "you can't convert a VM to template if VM is running\n" if PVE::QemuServer::check_running($vmid); return $conf; }; $load_and_check->(); my $realcmd = sub { PVE::QemuConfig->lock_config( $vmid, sub { my $conf = $load_and_check->(); $conf->{template} = 1; PVE::QemuConfig->write_config($vmid, $conf); PVE::QemuServer::template_create($vmid, $conf, $disk); }, ); }; return $rpcenv->fork_worker('qmtemplate', $vmid, $authuser, $realcmd); }, }); __PACKAGE__->register_method({ name => 'cloudinit_generated_config_dump', path => '{vmid}/cloudinit/dump', method => 'GET', proxyto => 'node', description => "Get automatically generated cloudinit config.", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Audit']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), type => { description => 'Config type.', type => 'string', enum => ['user', 'network', 'meta'], }, }, }, returns => { type => 'string', }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $conf = PVE::QemuConfig->load_config($param->{vmid}); my $mask_password = !$rpcenv->check($authuser, '/vms/{vmid}', ['VM.Config.Cloudinit'], 1); return PVE::QemuServer::Cloudinit::dump_cloudinit_config( $conf, $param->{vmid}, $param->{type}, $mask_password, ); }, }); __PACKAGE__->register_method({ name => 'mtunnel', path => '{vmid}/mtunnel', method => 'POST', protected => 1, description => 'Migration tunnel endpoint - only for internal use by VM migration.', permissions => { check => [ 'and', ['perm', '/vms/{vmid}', ['VM.Allocate']], ['perm', '/', ['Sys.Incoming']], ], description => "You need 'VM.Allocate' permissions on '/vms/{vmid}' and Sys.Incoming" . " on '/'. Further permission checks happen during the actual migration.", }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), storages => { type => 'string', format => 'pve-storage-id-list', optional => 1, description => 'List of storages to check permission and availability. Will be checked again for all actually used storages during migration.', }, bridges => { type => 'string', format => 'pve-bridge-id-list', optional => 1, description => 'List of network bridges to check availability. Will be checked again for actually used bridges during migration.', }, }, }, returns => { additionalProperties => 0, properties => { upid => { type => 'string' }, ticket => { type => 'string' }, socket => { type => 'string' }, }, }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $storages = extract_param($param, 'storages'); my $bridges = extract_param($param, 'bridges'); my $nodename = PVE::INotify::nodename(); raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" }) if $node ne 'localhost' && $node ne $nodename; $node = $nodename; my $storecfg = PVE::Storage::config(); foreach my $storeid (PVE::Tools::split_list($storages)) { $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storeid, $node); } foreach my $bridge (PVE::Tools::split_list($bridges)) { PVE::Network::read_bridge_mtu($bridge); } PVE::Cluster::check_cfs_quorum(); my $lock = 'create'; eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, $lock); }; raise_param_exc({ vmid => "unable to create empty VM config - $@" }) if $@; my $realcmd = sub { my $state = { storecfg => PVE::Storage::config(), lock => $lock, vmid => $vmid, }; my $run_locked = sub { my ($code, $params) = @_; return PVE::QemuConfig->lock_config( $state->{vmid}, sub { my $conf = PVE::QemuConfig->load_config($state->{vmid}); $state->{conf} = $conf; die "Encountered wrong lock - aborting mtunnel command handling.\n" if $state->{lock} && !PVE::QemuConfig->has_lock($conf, $state->{lock}); return $code->($params); }, ); }; my $cmd_desc = { config => { conf => { type => 'string', description => 'Full VM config, adapted for target cluster/node', }, 'firewall-config' => { type => 'string', description => 'VM firewall config', optional => 1, }, }, disk => { format => PVE::JSONSchema::get_standard_option('pve-qm-image-format'), storage => { type => 'string', format => 'pve-storage-id', }, drive => { type => 'object', description => 'parsed drive information without volid and format', }, }, start => { start_params => { type => 'object', description => 'params passed to vm_start_nolock', }, migrate_opts => { type => 'object', description => 'migrate_opts passed to vm_start_nolock', }, }, ticket => { path => { type => 'string', description => 'socket path for which the ticket should be valid. must be known to current mtunnel instance.', }, }, quit => { cleanup => { type => 'boolean', description => 'remove VM config and disks, aborting migration', default => 0, }, }, 'disk-import' => $PVE::StorageTunnel::cmd_schema->{'disk-import'}, 'query-disk-import' => $PVE::StorageTunnel::cmd_schema->{'query-disk-import'}, bwlimit => $PVE::StorageTunnel::cmd_schema->{bwlimit}, }; my $cmd_handlers = { 'version' => sub { # compared against other end's version # bump/reset for breaking changes # bump/bump for opt-in changes return { api => $PVE::QemuMigrate::WS_TUNNEL_VERSION, age => 0, }; }, 'config' => sub { my ($params) = @_; # parse and write out VM FW config if given if (my $fw_conf = $params->{'firewall-config'}) { my ($path, $fh) = PVE::Tools::tempfile_contents($fw_conf, 700); my $empty_conf = { rules => [], options => {}, aliases => {}, ipset => {}, ipset_comments => {}, }; my $cluster_fw_conf = PVE::Firewall::load_clusterfw_conf(); # TODO: add flag for strict parsing? # TODO: add import sub that does all this given raw content? my $vmfw_conf = PVE::Firewall::generic_fw_config_parser( $path, $cluster_fw_conf, $empty_conf, 'vm', ); $vmfw_conf->{vmid} = $state->{vmid}; PVE::Firewall::save_vmfw_conf($state->{vmid}, $vmfw_conf); $state->{cleanup}->{fw} = 1; } my $conf_fn = "incoming/qemu-server/$state->{vmid}.conf"; my $new_conf = PVE::QemuServer::parse_vm_config($conf_fn, $params->{conf}, 1); delete $new_conf->{lock}; delete $new_conf->{digest}; # TODO handle properly? delete $new_conf->{snapshots}; delete $new_conf->{parent}; delete $new_conf->{pending}; # not handled by update_vm_api my $vmgenid = delete $new_conf->{vmgenid}; my $meta = delete $new_conf->{meta}; my $special_sections = delete $new_conf->{'special-sections'} // {}; # fleecing state is specific to source side delete $special_sections->{fleecing}; $new_conf->{skip_cloud_init} = 1; # re-use image from source side # TODO PVE 10 - remove backwards-compat handling? my $cloudinit = delete $new_conf->{cloudinit}; if ($cloudinit) { if ($special_sections->{cloudinit}) { warn "config has duplicate special 'cloudinit' sections - skipping" . " legacy variant\n"; } else { $special_sections->{cloudinit} = $cloudinit; } } $new_conf->{vmid} = $state->{vmid}; $new_conf->{node} = $node; PVE::QemuConfig->remove_lock($state->{vmid}, 'create'); eval { $update_vm_api->($new_conf, 1); }; if (my $err = $@) { # revert to locked previous config my $conf = PVE::QemuConfig->load_config($state->{vmid}); $conf->{lock} = 'create'; PVE::QemuConfig->write_config($state->{vmid}, $conf); die $err; } my $conf = PVE::QemuConfig->load_config($state->{vmid}); $conf->{lock} = 'migrate'; $conf->{vmgenid} = $vmgenid if defined($vmgenid); $conf->{meta} = $meta if defined($meta); $conf->{'special-sections'} = $special_sections; PVE::QemuConfig->write_config($state->{vmid}, $conf); $state->{lock} = 'migrate'; return; }, 'bwlimit' => sub { my ($params) = @_; return PVE::StorageTunnel::handle_bwlimit($params); }, 'disk' => sub { my ($params) = @_; my $format = $params->{format}; my $storeid = $params->{storage}; my $drive = $params->{drive}; $check_storage_access_migrate->( $rpcenv, $authuser, $state->{storecfg}, $storeid, $node, ); my $storagemap = { default => $storeid, }; my $source_volumes = { 'disk' => [ undef, $storeid, $drive, 0, $format, ], }; my $res = PVE::QemuServer::vm_migrate_alloc_nbd_disks( $state->{storecfg}, $state->{vmid}, $source_volumes, $storagemap, ); if (defined($res->{disk})) { $state->{cleanup}->{volumes}->{ $res->{disk}->{volid} } = 1; return $res->{disk}; } else { die "failed to allocate NBD disk..\n"; } }, 'disk-import' => sub { my ($params) = @_; $check_storage_access_migrate->( $rpcenv, $authuser, $state->{storecfg}, $params->{storage}, $node, ); $params->{unix} = "/run/qemu-server/$state->{vmid}.storage"; return PVE::StorageTunnel::handle_disk_import($state, $params); }, 'query-disk-import' => sub { my ($params) = @_; return PVE::StorageTunnel::handle_query_disk_import($state, $params); }, 'start' => sub { my ($params) = @_; my $info = PVE::QemuServer::vm_start_nolock( $state->{storecfg}, $state->{vmid}, $state->{conf}, $params->{start_params}, $params->{migrate_opts}, ); if ($info->{migrate}->{proto} ne 'unix') { PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1); die "migration over non-UNIX sockets not possible\n"; } my $socket = $info->{migrate}->{addr}; chown $state->{socket_uid}, -1, $socket; $state->{sockets}->{$socket} = 1; my $unix_sockets = $info->{migrate}->{unix_sockets}; foreach my $socket (@$unix_sockets) { chown $state->{socket_uid}, -1, $socket; $state->{sockets}->{$socket} = 1; } return $info; }, 'fstrim' => sub { if (PVE::QemuServer::Agent::qga_check_running($state->{vmid})) { eval { mon_cmd($state->{vmid}, "guest-fstrim") }; warn "fstrim failed: $@\n" if $@; } return; }, 'stop' => sub { PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1); return; }, 'nbdstop' => sub { PVE::QemuServer::QMPHelpers::nbd_stop($state->{vmid}); return; }, 'resume' => sub { if (PVE::QemuServer::Helpers::vm_running_locally($state->{vmid})) { PVE::QemuServer::RunState::vm_resume($state->{vmid}, 1, 1); } else { die "VM $state->{vmid} not running\n"; } return; }, 'unlock' => sub { PVE::QemuConfig->remove_lock($state->{vmid}, $state->{lock}); delete $state->{lock}; return; }, 'ticket' => sub { my ($params) = @_; my $path = $params->{path}; die "Not allowed to generate ticket for unknown socket '$path'\n" if !defined($state->{sockets}->{$path}); return { ticket => PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$path"), }; }, 'quit' => sub { my ($params) = @_; if ($params->{cleanup}) { if ($state->{cleanup}->{fw}) { PVE::Firewall::remove_vmfw_conf($state->{vmid}); } for my $volid (keys $state->{cleanup}->{volumes}->%*) { print "freeing volume '$volid' as part of cleanup\n"; eval { PVE::Storage::vdisk_free($state->{storecfg}, $volid) }; warn $@ if $@; } PVE::QemuServer::DBusVMState::qemu_del_dbus_vmstate($state->{vmid}); PVE::QemuServer::destroy_vm($state->{storecfg}, $state->{vmid}, 1); } print "switching to exit-mode, waiting for client to disconnect\n"; $state->{exit} = 1; return; }, }; $run_locked->(sub { my $socket_addr = "/run/qemu-server/$state->{vmid}.mtunnel"; unlink $socket_addr; $state->{socket} = IO::Socket::UNIX->new( Type => SOCK_STREAM(), Local => $socket_addr, Listen => 1, ); $state->{socket_uid} = getpwnam('www-data') or die "Failed to resolve user 'www-data' to numeric UID\n"; chown $state->{socket_uid}, -1, $socket_addr; }); print "mtunnel started\n"; my $conn = eval { PVE::Tools::run_with_timeout(300, sub { $state->{socket}->accept() }); }; if ($@) { warn "Failed to accept tunnel connection - $@\n"; warn "Removing tunnel socket..\n"; unlink $state->{socket}; warn "Removing temporary VM config..\n"; $run_locked->(sub { PVE::QemuServer::destroy_vm($state->{storecfg}, $state->{vmid}, 1); }); die "Exiting mtunnel\n"; } $state->{conn} = $conn; my $reply_err = sub { my ($msg) = @_; my $reply = JSON::encode_json({ success => JSON::false, msg => $msg, }); $conn->print("$reply\n"); $conn->flush(); }; my $reply_ok = sub { my ($res) = @_; $res->{success} = JSON::true; my $reply = JSON::encode_json($res); $conn->print("$reply\n"); $conn->flush(); }; while (my $line = <$conn>) { chomp $line; # untaint, we validate below if needed ($line) = $line =~ /^(.*)$/; my $parsed = eval { JSON::decode_json($line) }; if ($@) { $reply_err->("failed to parse command - $@"); next; } my $cmd = delete $parsed->{cmd}; if (!defined($cmd)) { $reply_err->("'cmd' missing"); } elsif ($state->{exit}) { $reply_err->("tunnel is in exit-mode, processing '$cmd' cmd not possible"); next; } elsif (my $handler = $cmd_handlers->{$cmd}) { print "received command '$cmd'\n"; eval { if (my $props = $cmd_desc->{$cmd}) { my $schema = { type => 'object', properties => $props, }; PVE::JSONSchema::validate($parsed, $schema); } else { $parsed = {}; } my $res = $run_locked->($handler, $parsed); $reply_ok->($res); }; $reply_err->("failed to handle '$cmd' command - $@") if $@; } else { $reply_err->("unknown command '$cmd' given"); } } if ($state->{exit}) { print "mtunnel exited\n"; } else { die "mtunnel exited unexpectedly\n"; } }; my $socket_addr = "/run/qemu-server/$vmid.mtunnel"; my $ticket = PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$socket_addr"); my $upid = $rpcenv->fork_worker('qmtunnel', $vmid, $authuser, $realcmd); return { ticket => $ticket, upid => $upid, socket => $socket_addr, }; }, }); __PACKAGE__->register_method({ name => 'mtunnelwebsocket', path => '{vmid}/mtunnelwebsocket', method => 'GET', permissions => { description => "You need to pass a ticket valid for the selected socket. Tickets can be created via the mtunnel API call, which will check permissions accordingly.", user => 'all', # check inside }, description => 'Migration tunnel endpoint for websocket upgrade - only for internal use by VM migration.', parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), socket => { type => "string", description => "unix socket to forward to", }, ticket => { type => "string", description => "ticket return by initial 'mtunnel' API call, or retrieved via 'ticket' tunnel command", }, }, }, returns => { type => "object", properties => { port => { type => 'string', optional => 1 }, socket => { type => 'string', optional => 1 }, }, }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $nodename = PVE::INotify::nodename(); my $node = extract_param($param, 'node'); raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" }) if $node ne 'localhost' && $node ne $nodename; my $vmid = $param->{vmid}; # check VM exists PVE::QemuConfig->load_config($vmid); my $socket = $param->{socket}; PVE::AccessControl::verify_tunnel_ticket($param->{ticket}, $authuser, "/socket/$socket"); return { socket => $socket }; }, }); __PACKAGE__->register_method({ name => 'dbus_vmstate', path => '{vmid}/dbus-vmstate', method => 'POST', proxyto => 'node', description => 'Control the dbus-vmstate helper for a given running VM.', permissions => { check => ['perm', '/vms/{vmid}', ['VM.Migrate']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), action => { type => 'string', enum => [qw(start stop)], description => 'Action to perform on the DBus VMState helper.', optional => 0, }, }, }, returns => { type => 'null', }, code => sub { my ($param) = @_; my ($node, $vmid, $action) = $param->@{qw(node vmid action)}; my $nodename = PVE::INotify::nodename(); if ($node ne 'localhost' && $node ne $nodename) { raise_param_exc( { node => "node needs to be 'localhost' or local hostname '$nodename'" }); } if (!PVE::QemuServer::Helpers::vm_running_locally($vmid)) { raise_param_exc({ node => "VM $vmid not running locally on node '$nodename'" }); } if ($action eq 'start') { syslog('info', "starting dbus-vmstate helper for VM $vmid\n"); PVE::QemuServer::DBusVMState::qemu_add_dbus_vmstate($vmid); } elsif ($action eq 'stop') { syslog('info', "stopping dbus-vmstate helper for VM $vmid\n"); PVE::QemuServer::DBusVMState::qemu_del_dbus_vmstate($vmid); } else { die "unknown action $action\n"; } }, }); 1; ================================================ FILE: src/PVE/CLI/Makefile ================================================ DESTDIR= PREFIX=/usr PERLDIR=$(PREFIX)/share/perl5 SOURCES=qm.pm qmrestore.pm .PHONY: install install: $(SOURCES) install -d -m 0755 $(DESTDIR)$(PERLDIR)/PVE/CLI for i in $(SOURCES); do install -D -m 0644 $$i $(DESTDIR)$(PERLDIR)/PVE/CLI/$$i; done ================================================ FILE: src/PVE/CLI/qm.pm ================================================ package PVE::CLI::qm; use strict; use warnings; # Note: disable '+' prefix for Getopt::Long (for resize command) use Getopt::Long qw(:config no_getopt_compat); use Fcntl ':flock'; use File::Path; use IO::Select; use IO::Socket::UNIX; use JSON; use POSIX qw(strftime); use Term::ReadLine; use URI::Escape; use PVE::APIClient::LWP; use PVE::Cluster; use PVE::Exception qw(raise_param_exc); use PVE::GuestHelpers; use PVE::GuestImport::OVF; use PVE::INotify; use PVE::JSONSchema qw(get_standard_option); use PVE::Network; use PVE::RPCEnvironment; use PVE::SafeSyslog; use PVE::Tools qw(extract_param file_get_contents); use PVE::API2::Qemu::Agent; use PVE::API2::Qemu; use PVE::QemuConfig; use PVE::QemuServer::Drive qw(is_valid_drivename parse_drive print_drive); use PVE::QemuServer::Helpers; use PVE::QemuServer::Agent; use PVE::QemuServer::ImportDisk; use PVE::QemuServer::Monitor qw(mon_cmd); use PVE::QemuServer::OVMF; use PVE::QemuServer::QMPHelpers; use PVE::QemuServer::RunState; use PVE::QemuServer::DBusVMState; use PVE::QemuServer; use PVE::CLIHandler; use base qw(PVE::CLIHandler); my $upid_exit = sub { my $upid = shift; my $status = PVE::Tools::upid_read_status($upid); exit(PVE::Tools::upid_status_is_error($status) ? -1 : 0); }; my $nodename = PVE::INotify::nodename(); my %node = (node => $nodename); sub setup_environment { PVE::RPCEnvironment->setup_default_cli_env(); } sub run_vnc_proxy { my ($path) = @_; my $c; while (++$c < 10 && !-e $path) { sleep(1); } my $s = IO::Socket::UNIX->new(Peer => $path, Timeout => 120); die "unable to connect to socket '$path' - $!" if !$s; my $select = IO::Select->new(); $select->add(\*STDIN); $select->add($s); my $timeout = 60 * 15; # 15 minutes my @handles; while ( $select->count && scalar(@handles = $select->can_read($timeout)) ) { foreach my $h (@handles) { my $buf; my $n = $h->sysread($buf, 4096); if ($h == \*STDIN) { if ($n) { syswrite($s, $buf); } else { exit(0); } } elsif ($h == $s) { if ($n) { syswrite(\*STDOUT, $buf); } else { exit(0); } } } } exit(0); } sub print_recursive_hash { my ($prefix, $hash, $key) = @_; if (ref($hash) eq 'HASH') { if (defined($key)) { print "$prefix$key:\n"; } for my $itemkey (sort keys %$hash) { print_recursive_hash("\t$prefix", $hash->{$itemkey}, $itemkey); } } elsif (ref($hash) eq 'ARRAY') { if (defined($key)) { print "$prefix$key:\n"; } for my $item (@$hash) { print_recursive_hash("\t$prefix", $item); } } elsif ((!ref($hash) && defined($hash)) || ref($hash) eq 'JSON::PP::Boolean') { if (defined($key)) { print "$prefix$key: $hash\n"; } else { print "$prefix$hash\n"; } } } __PACKAGE__->register_method({ name => 'showcmd', path => 'showcmd', method => 'GET', description => "Show command line which is used to start the VM (debug info).", parameters => { additionalProperties => 0, properties => { vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), pretty => { description => "Puts each option on a new line to enhance human readability", type => 'boolean', optional => 1, default => 0, }, snapshot => get_standard_option( 'pve-snapshot-name', { description => "Fetch config values from given snapshot.", optional => 1, completion => sub { my ($cmd, $pname, $cur, $args) = @_; PVE::QemuConfig->snapshot_list($args->[0]); }, }, ), }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $storecfg = PVE::Storage::config(); my $cmdline = PVE::QemuServer::vm_commandline($storecfg, $param->{vmid}, $param->{snapshot}); $cmdline =~ s/ -/ \\\n -/g if $param->{pretty}; print "$cmdline\n"; return; }, }); __PACKAGE__->register_method({ name => 'remote_migrate_vm', path => 'remote_migrate_vm', method => 'POST', description => "Migrate virtual machine to a remote cluster. Creates a new migration task. EXPERIMENTAL feature!", permissions => { check => ['perm', '/vms/{vmid}', ['VM.Migrate']], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), 'target-vmid' => get_standard_option('pve-vmid', { optional => 1 }), 'target-endpoint' => get_standard_option('proxmox-remote', { description => "Remote target endpoint", }), online => { type => 'boolean', description => "Use online/live migration if VM is running. Ignored if VM is stopped.", optional => 1, }, delete => { type => 'boolean', description => "Delete the original VM and related data after successful migration. By default the original VM is kept on the source cluster in a stopped state.", optional => 1, default => 0, }, 'target-storage' => get_standard_option( 'pve-targetstorage', { completion => \&PVE::QemuServer::complete_migration_storage, optional => 0, }, ), 'target-bridge' => { type => 'string', description => "Mapping from source to target bridges. Providing only a single bridge ID maps all source bridges to that bridge. Providing the special value '1' will map each source bridge to itself.", format => 'bridge-pair-list', }, bwlimit => { description => "Override I/O bandwidth limit (in KiB/s).", optional => 1, type => 'integer', minimum => '0', default => 'migrate limit from datacenter or storage config', }, }, }, returns => { type => 'string', description => "the task ID.", }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $source_vmid = $param->{vmid}; my $target_endpoint = $param->{'target-endpoint'}; my $target_vmid = $param->{'target-vmid'} // $source_vmid; my $remote = PVE::JSONSchema::parse_property_string('proxmox-remote', $target_endpoint); # TODO: move this as helper somewhere appropriate? my $conn_args = { protocol => 'https', host => $remote->{host}, port => $remote->{port} // 8006, apitoken => $remote->{apitoken}, }; $conn_args->{cached_fingerprints} = { uc($remote->{fingerprint}) => 1 } if defined($remote->{fingerprint}); my $api_client = PVE::APIClient::LWP->new(%$conn_args); my $resources = $api_client->get("/cluster/resources", { type => 'vm' }); if (grep { defined($_->{vmid}) && $_->{vmid} eq $target_vmid } @$resources) { raise_param_exc( { target_vmid => "Guest with ID '$target_vmid' already exists on remote cluster", }, ); } my $storages = $api_client->get("/nodes/localhost/storage", { enabled => 1 }); my $storecfg = PVE::Storage::config(); my $target_storage = $param->{'target-storage'}; my $storagemap = eval { PVE::JSONSchema::parse_idmap($target_storage, 'pve-storage-id') }; raise_param_exc({ 'target-storage' => "failed to parse storage map: $@" }) if $@; my $check_remote_storage = sub { my ($storage) = @_; my $found = [grep { $_->{storage} eq $storage } @$storages]; die "remote: storage '$storage' does not exist (or missing permission)!\n" if !@$found; $found = @$found[0]; my $content_types = [PVE::Tools::split_list($found->{content})]; die "remote: storage '$storage' cannot store images\n" if !grep { $_ eq 'images' } @$content_types; }; foreach my $target_sid (values %{ $storagemap->{entries} }) { $check_remote_storage->($target_sid); } $check_remote_storage->($storagemap->{default}) if $storagemap->{default}; return PVE::API2::Qemu->remote_migrate_vm($param); }, }); __PACKAGE__->register_method({ name => 'status', path => 'status', method => 'GET', description => "Show VM status.", parameters => { additionalProperties => 0, properties => { vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), verbose => { description => "Verbose output format", type => 'boolean', optional => 1, }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; # test if VM exists my $conf = PVE::QemuConfig->load_config($param->{vmid}); my $vmstatus = PVE::QemuServer::vmstatus($param->{vmid}, 1); my $stat = $vmstatus->{ $param->{vmid} }; if ($param->{verbose}) { foreach my $k (sort (keys %$stat)) { next if $k eq 'cpu' || $k eq 'relcpu'; # always 0 my $v = $stat->{$k}; print_recursive_hash("", $v, $k); } } else { my $status = $stat->{qmpstatus} || 'unknown'; print "status: $status\n"; } return; }, }); __PACKAGE__->register_method({ name => 'vncproxy', path => 'vncproxy', method => 'PUT', description => "Proxy VM VNC traffic to stdin/stdout", parameters => { additionalProperties => 0, properties => { vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; PVE::QemuConfig::assert_config_exists_on_node($vmid); my $vnc_socket = PVE::QemuServer::Helpers::vnc_socket($vmid); if (my $ticket = $ENV{LC_PVE_TICKET}) { # NOTE: ssh on debian only pass LC_* variables mon_cmd($vmid, "set_password", protocol => 'vnc', password => $ticket); mon_cmd($vmid, "expire_password", protocol => 'vnc', time => "+30"); } else { die "LC_PVE_TICKET not set, VNC proxy without password is forbidden\n"; } run_vnc_proxy($vnc_socket); return; }, }); __PACKAGE__->register_method({ name => 'unlock', path => 'unlock', method => 'PUT', description => "Unlock the VM.", parameters => { additionalProperties => 0, properties => { vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; PVE::QemuConfig->lock_config( $vmid, sub { my $conf = PVE::QemuConfig->load_config($vmid); delete $conf->{lock}; delete $conf->{pending}->{lock} if $conf->{pending}; # just to be sure PVE::QemuConfig->write_config($vmid, $conf); }, ); return; }, }); __PACKAGE__->register_method({ name => 'nbdstop', path => 'nbdstop', method => 'PUT', description => "Stop embedded nbd server.", parameters => { additionalProperties => 0, properties => { vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; eval { PVE::QemuServer::QMPHelpers::nbd_stop($vmid) }; warn $@ if $@; return; }, }); __PACKAGE__->register_method({ name => 'mtunnel', path => 'mtunnel', method => 'POST', description => "Used by qmigrate - do not use manually.", parameters => { additionalProperties => 0, properties => {}, }, returns => { type => 'null' }, code => sub { my ($param) = @_; if (!PVE::Cluster::check_cfs_quorum(1)) { print "no quorum\n"; return; } my $tunnel_write = sub { my $text = shift; chomp $text; print "$text\n"; *STDOUT->flush(); }; $tunnel_write->("tunnel online"); $tunnel_write->("ver 1"); while (my $line = ) { chomp $line; if ($line =~ /^quit$/) { $tunnel_write->("OK"); last; } elsif ($line =~ /^resume (\d+)$/) { my $vmid = $1; # check_running and vm_resume with nocheck, since local node # might not have processed config move/rename yet if (PVE::QemuServer::check_running($vmid, 1)) { eval { PVE::QemuServer::RunState::vm_resume($vmid, 1, 1); }; if ($@) { $tunnel_write->("ERR: resume failed - $@"); } else { $tunnel_write->("OK"); } } else { $tunnel_write->("ERR: resume failed - VM $vmid not running"); } } } return; }, }); __PACKAGE__->register_method({ name => 'wait', path => 'wait', method => 'GET', description => "Wait until the VM is stopped.", parameters => { additionalProperties => 0, properties => { vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), timeout => { description => "Timeout in seconds. Default is to wait forever.", type => 'integer', minimum => 1, optional => 1, }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; my $timeout = $param->{timeout}; my $pid = PVE::QemuServer::check_running($vmid); return if !$pid; print "waiting until VM $vmid stops (PID $pid)\n"; my $count = 0; while ((!$timeout || ($count < $timeout)) && PVE::QemuServer::check_running($vmid)) { $count++; sleep 1; } die "wait failed - got timeout\n" if PVE::QemuServer::check_running($vmid); return; }, }); __PACKAGE__->register_method({ name => 'monitor', path => 'monitor', method => 'POST', description => "Enter QEMU Monitor interface.", parameters => { additionalProperties => 0, properties => { vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; my $conf = PVE::QemuConfig->load_config($vmid); # check if VM exists print "Entering QEMU Monitor for VM $vmid - type 'help' for help\n"; my $term = Term::ReadLine->new('qm'); while (defined(my $input = $term->readline('qm> '))) { chomp $input; next if $input =~ m/^\s*$/; last if $input =~ m/^\s*q(uit)?\s*$/; eval { print PVE::QemuServer::Monitor::hmp_cmd($vmid, $input, 30) }; print "ERROR: $@" if $@; } return; }, }); __PACKAGE__->register_method({ name => 'rescan', path => 'rescan', method => 'POST', description => "Rescan all storages and update disk sizes and unused disk images.", parameters => { additionalProperties => 0, properties => { vmid => get_standard_option( 'pve-vmid', { optional => 1, completion => \&PVE::QemuServer::complete_vmid, }, ), dryrun => { type => 'boolean', optional => 1, default => 0, description => 'Do not actually write changes out to VM config(s).', }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $dryrun = $param->{dryrun}; print "NOTE: running in dry-run mode, won't write changes out!\n" if $dryrun; PVE::QemuServer::rescan($param->{vmid}, 0, $dryrun); return; }, }); __PACKAGE__->register_method({ name => 'importdisk', path => 'importdisk', method => 'POST', description => "Import an external disk image as an unused disk in a VM. The image format has to be supported by qemu-img(1).", parameters => { additionalProperties => 0, properties => { vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), source => { description => 'Path to the disk image to import', type => 'string', optional => 0, }, storage => get_standard_option( 'pve-storage-id', { description => 'Target storage ID', completion => \&PVE::QemuServer::complete_storage, optional => 0, }, ), format => { type => 'string', description => 'Target format', enum => ['raw', 'qcow2', 'vmdk'], optional => 1, }, 'target-disk' => { type => 'string', description => 'The disk name where the volume will be imported to (e.g. scsi1).', enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()], optional => 1, }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $vmid = extract_param($param, 'vmid'); my $source = extract_param($param, 'source'); my $storeid = extract_param($param, 'storage'); my $format = extract_param($param, 'format'); my $target_disk = extract_param($param, 'target-disk'); # do_import does not allow invalid drive names (e.g. unused0) $target_disk = undef if $target_disk && !is_valid_drivename($target_disk); my $vm_conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_lock($vm_conf); die "$source: non-existent or non-regular file\n" if (!-f $source); my $storecfg = PVE::Storage::config(); PVE::Storage::storage_check_enabled($storecfg, $storeid); my $target_storage_config = PVE::Storage::storage_config($storecfg, $storeid); die "storage $storeid does not support vm images\n" if !$target_storage_config->{content}->{images}; print "importing disk '$source' to VM $vmid ...\n"; my $size = PVE::Storage::file_size_info($source, undef, 'auto-detect'); my ($drive_id, $volid) = PVE::QemuServer::ImportDisk::do_import( $source, $size, $vmid, $storeid, { drive_name => $target_disk, format => $format, }, ); $vm_conf = PVE::QemuConfig->load_config($vmid); # change imported _used_ disk to a base volume in case the VM is a template PVE::QemuServer::template_create($vmid, $vm_conf, $drive_id) if is_valid_drivename($drive_id) && PVE::QemuConfig->is_template($vm_conf); print "$drive_id: successfully imported disk '$vm_conf->{$drive_id}'\n"; return; }, }); __PACKAGE__->register_method({ name => 'enroll-efi-keys', path => 'enroll-efi-keys', method => 'POST', description => "Enroll important updated certificates to the EFI disk with pre-enrolled-keys. Currently," . " these are UEFI 2023 certificates from Microsoft. Must be called while the VM is shut" . " down.", parameters => { additionalProperties => 0, properties => { vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $vmid = extract_param($param, 'vmid'); my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_lock($conf); die "VM $vmid is running\n" if PVE::QemuServer::Helpers::vm_running_locally($vmid); die "VM $vmid is a template\n" if PVE::QemuConfig->is_template($conf); die "VM $vmid has no EFI disk configured\n" if !$conf->{efidisk0}; my $storecfg = PVE::Storage::config(); my $efidisk = parse_drive('efidisk0', $conf->{efidisk0}); my $updated = PVE::QemuServer::OVMF::ensure_ms_2023_cert_enrolled($storecfg, $vmid, $efidisk); if (!$updated) { print "skipping - no pre-enrolled keys or already got ms-cert=2023k marker\n"; return; } PVE::QemuConfig->lock_config( $vmid, sub { my $locked_conf = PVE::QemuConfig->load_config($vmid); eval { PVE::Tools::assert_if_modified($conf->{digest}, $locked_conf->{digest}) }; die "VM ${vmid}: $@" if $@; $locked_conf->{efidisk0} = print_drive($updated); PVE::QemuConfig->write_config($vmid, $locked_conf); print "successfully updated efidisk\n"; }, ); return; }, }); __PACKAGE__->register_method({ name => 'terminal', path => 'terminal', method => 'POST', description => "Open a terminal using a serial device (The VM need to have a serial device configured, for example 'serial0: socket')", parameters => { additionalProperties => 0, properties => { vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), iface => { description => "Select the serial device. By default we simply use the first suitable device.", type => 'string', optional => 1, enum => [qw(serial0 serial1 serial2 serial3)], }, escape => { description => "Escape character.", type => 'string', optional => 1, default => '^O', }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; my $escape = $param->{escape} // '^O'; if ($escape =~ /^\^([\x40-\x7a])$/) { $escape = ord($1) & 0x1F; } elsif ($escape =~ /^0x[0-9a-f]+$/i) { $escape = hex($escape); } elsif ($escape =~ /^[0-9]+$/) { $escape = int($escape); } else { die "invalid escape character definition: $escape\n"; } my $escapemsg = ''; if ($escape) { $escapemsg = sprintf(' (press Ctrl+%c to exit)', $escape + 0x40); $escape = sprintf(',escape=0x%x', $escape); } else { $escape = ''; } my $conf = PVE::QemuConfig->load_config($vmid); # check if VM exists my $iface = $param->{iface}; if ($iface) { die "serial interface '$iface' is not configured\n" if !$conf->{$iface}; die "wrong serial type on interface '$iface'\n" if $conf->{$iface} ne 'socket'; } else { foreach my $opt (qw(serial0 serial1 serial2 serial3)) { if ($conf->{$opt} && ($conf->{$opt} eq 'socket')) { $iface = $opt; last; } } die "unable to find a serial interface\n" if !$iface; } die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid); my $socket = "/var/run/qemu-server/${vmid}.$iface"; my $cmd = "socat UNIX-CONNECT:$socket STDIO,raw,echo=0$escape"; print "starting serial terminal on interface ${iface}${escapemsg}\n"; system($cmd); return; }, }); __PACKAGE__->register_method({ name => 'importovf', path => 'importovf', description => "Create a new VM using parameters read from an OVF manifest", parameters => { additionalProperties => 0, properties => { vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }, ), manifest => { type => 'string', description => 'path to the ovf file', }, storage => get_standard_option( 'pve-storage-id', { description => 'Target storage ID', completion => \&PVE::QemuServer::complete_storage, optional => 0, }, ), format => { type => 'string', description => 'Target format', enum => ['raw', 'qcow2', 'vmdk'], optional => 1, }, dryrun => { type => 'boolean', description => 'Print a parsed representation of the extracted OVF parameters, but do not create a VM', optional => 1, }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $vmid = PVE::Tools::extract_param($param, 'vmid'); my $ovf_file = PVE::Tools::extract_param($param, 'manifest'); my $storeid = PVE::Tools::extract_param($param, 'storage'); my $format = PVE::Tools::extract_param($param, 'format'); my $dryrun = PVE::Tools::extract_param($param, 'dryrun'); die "$ovf_file: non-existent or non-regular file\n" if (!-f $ovf_file); my $storecfg = PVE::Storage::config(); PVE::Storage::storage_check_enabled($storecfg, $storeid); my $parsed = PVE::GuestImport::OVF::parse_ovf($ovf_file); if ($dryrun) { print to_json($parsed, { pretty => 1, canonical => 1 }); return; } eval { PVE::QemuConfig->create_and_lock_config($vmid) }; die "Reserving empty config for OVF import to VM $vmid failed: $@" if $@; my $conf = PVE::QemuConfig->load_config($vmid); die "Internal error: Expected 'create' lock in config of VM $vmid!" if !PVE::QemuConfig->has_lock($conf, "create"); $conf->{name} = $parsed->{qm}->{name} if defined($parsed->{qm}->{name}); $conf->{memory} = $parsed->{qm}->{memory} if defined($parsed->{qm}->{memory}); $conf->{cores} = $parsed->{qm}->{cores} if defined($parsed->{qm}->{cores}); my $imported_disks = []; eval { # order matters, as do_import() will load_config() internally $conf->{vmgenid} = PVE::QemuServer::generate_uuid(); $conf->{smbios1} = PVE::QemuServer::generate_smbios1_uuid(); PVE::QemuConfig->write_config($vmid, $conf); foreach my $disk (@{ $parsed->{disks} }) { my ($file, $drive) = ($disk->{backing_file}, $disk->{disk_address}); my $size = PVE::Storage::file_size_info($file, undef, 'auto-detect'); my ($name, $volid) = PVE::QemuServer::ImportDisk::do_import( $file, $size, $vmid, $storeid, { drive_name => $drive, format => $format, skiplock => 1, }, ); # for cleanup on (later) error push @$imported_disks, $volid; } # reload after disks entries have been created $conf = PVE::QemuConfig->load_config($vmid); my $devs = PVE::QemuServer::get_default_bootdevices($conf); $conf->{boot} = PVE::QemuServer::print_bootorder($devs); PVE::QemuConfig->write_config($vmid, $conf); }; if (my $err = $@) { my $skiplock = 1; warn "error during import, cleaning up created resources...\n"; for my $volid (@$imported_disks) { eval { PVE::Storage::vdisk_free($storecfg, $volid) }; warn "cleanup of $volid failed: $@\n" if $@; } eval { PVE::QemuServer::destroy_vm($storecfg, $vmid, $skiplock) }; warn "Could not destroy VM $vmid: $@" if "$@"; die "import failed - $err"; } PVE::QemuConfig->remove_lock($vmid, "create"); return; }, }); __PACKAGE__->register_method({ name => 'exec', path => 'exec', method => 'POST', protected => 1, description => "Executes the given command via the guest agent", parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), synchronous => { type => 'boolean', optional => 1, default => 1, description => "If set to off, returns the pid immediately instead of waiting for the command to finish or the timeout.", }, 'timeout' => { type => 'integer', description => "The maximum time to wait synchronously for the command to finish. If reached, the pid gets returned. Set to 0 to deactivate", minimum => 0, optional => 1, default => 30, }, 'pass-stdin' => { type => 'boolean', description => "When set, read STDIN until EOF and forward to guest agent via 'input-data' (usually treated as STDIN to process launched by guest agent). Allows maximal 1 MiB.", optional => 1, default => 0, }, 'extra-args' => get_standard_option('extra-args'), }, }, returns => { type => 'object', }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; my $sync = $param->{synchronous} // 1; my $pass_stdin = $param->{'pass-stdin'}; if (defined($param->{timeout}) && !$sync) { raise_param_exc({ synchronous => "needs to be set for 'timeout'" }); } my $input_data = undef; if ($pass_stdin) { $input_data = ''; while (my $line = ) { $input_data .= $line; if (length($input_data) > 1024 * 1024) { # not sure how QEMU handles large amounts of data being # passed into the QMP socket, so limit to be safe die "'input-data' (STDIN) is limited to 1 MiB, aborting\n"; } } } my $args = $param->{'extra-args'}; $args = undef if !$args || !@$args; my $conf = PVE::QemuConfig->load_config($vmid); my $res = PVE::QemuServer::Agent::qemu_exec($vmid, $conf, $input_data, $args); if ($sync) { my $pid = $res->{pid}; my $timeout = $param->{timeout} // 30; my $starttime = time(); while ($timeout == 0 || (time() - $starttime) < $timeout) { my $out = PVE::QemuServer::Agent::qemu_exec_status($vmid, $conf, $pid); if ($out->{exited}) { $res = $out; last; } sleep 1; } if (!$res->{exited}) { warn "timeout reached, returning pid\n"; } } return { result => $res }; }, }); __PACKAGE__->register_method({ name => 'cleanup', path => 'cleanup', method => 'POST', protected => 1, description => "Cleans up resources like tap devices, vgpus, etc. Called after a vm shuts down, crashes, etc.", parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }, ), 'clean-shutdown' => { type => 'boolean', description => "Indicates if qemu shutdown cleanly.", }, 'guest-requested' => { type => 'boolean', description => "Indicates if the shutdown was requested by the guest or via qmp.", }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $vmid = $param->{vmid}; my $clean = $param->{'clean-shutdown'}; my $guest = $param->{'guest-requested'}; my $restart = 0; # return if we do not have the config anymore return if !-f PVE::QemuConfig->config_file($vmid); my $storecfg = PVE::Storage::config(); warn "Starting cleanup for $vmid\n"; # mdev cleanup can take a while, so wait up to 60 seconds PVE::QemuConfig->lock_config_full( $vmid, 60, sub { my $conf = PVE::QemuConfig->load_config($vmid); my $pid = PVE::QemuServer::check_running($vmid); die "vm still running\n" if $pid; # Rollback already does cleanup when preparing and afterwards temporarily drops the # lock on the configuration file to rollback the volumes. Deactivating volumes here # again while that is happening would be problematic. die "skipping cleanup - 'rollback' lock is present\n" if $conf->{lock} && $conf->{lock} eq 'rollback'; if (!$clean) { # we have to cleanup the tap devices after a crash foreach my $opt (keys %$conf) { next if $opt !~ m/^net(\d+)$/; my $interface = $1; PVE::Network::tap_unplug("tap${vmid}i${interface}"); } } if (!$clean || $guest) { # vm was shutdown from inside the guest or crashed, doing api cleanup PVE::QemuServer::vm_stop_cleanup($storecfg, $vmid, $conf, 0, 0, 1); } PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-stop'); $restart = eval { PVE::QemuServer::clear_reboot_request($vmid) }; warn $@ if $@; }, ); warn "Finished cleanup for $vmid\n"; if ($restart) { warn "Restarting VM $vmid\n"; PVE::API2::Qemu->vm_start({ vmid => $vmid, %node, }); } return; }, }); __PACKAGE__->register_method({ name => 'vm_import', path => 'vm-import', description => "Import a foreign virtual guest from a supported import source, such as an ESXi storage.", parameters => { additionalProperties => 0, properties => PVE::QemuServer::json_config_properties( { vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }, ), 'source' => { type => 'string', description => 'The import source volume id.', }, storage => get_standard_option( 'pve-storage-id', { description => "Default storage.", completion => \&PVE::QemuServer::complete_storage, }, ), 'live-import' => { type => 'boolean', optional => 1, default => 0, description => "Immediately start the VM and copy the data in the background.", }, 'dryrun' => { type => 'boolean', optional => 1, default => 0, description => "Show the create command and exit without doing anything.", }, delete => { type => 'string', format => 'pve-configid-list', description => "A list of settings you want to delete.", optional => 1, }, format => { type => 'string', description => 'Target format', enum => ['raw', 'qcow2', 'vmdk'], optional => 1, }, }, 1, # with_disk_alloc ), }, returns => { type => 'null' }, code => sub { my ($param) = @_; my ($vmid, $source, $storage, $format, $live_import, $dryrun, $delete) = delete $param->@{qw(vmid source storage format live-import dryrun delete)}; if (defined($format)) { $format = ",format=$format"; } else { $format = ''; } my $storecfg = PVE::Storage::config(); my $metadata = PVE::Storage::get_import_metadata($storecfg, $source); my $create_args = $metadata->{'create-args'}; if (my $netdevs = $metadata->{net}) { for my $net (keys $netdevs->%*) { my $value = $netdevs->{$net}; $create_args->{$net} = join(',', map { $_ . '=' . $value->{$_} } sort keys %$value); } } if (my $disks = $metadata->{disks}) { if (delete $disks->{efidisk0}) { $create_args->{efidisk0} = "$storage:1$format,efitype=4m"; } for my $disk (keys $disks->%*) { my $value = $disks->{$disk}->{volid}; $create_args->{$disk} = "$storage:0${format},import-from=$value"; } } $create_args->{'live-restore'} = 1 if $live_import; $create_args->{$_} = $param->{$_} for keys $param->%*; delete $create_args->{$_} for PVE::Tools::split_list($delete); if ($dryrun) { print("# dry-run – the resulting create command for the import would be:\n"); print("qm create $vmid \\\n "); print(join( " \\\n ", map { "--$_ $create_args->{$_}" } sort keys $create_args->%*)); print("\n"); return; } PVE::API2::Qemu->create_vm({ %node, vmid => $vmid, %$create_args, }); return; }, }); my $print_agent_result = sub { my ($data) = @_; my $result = $data->{result} // $data; return if !defined($result); my $class = ref($result); if (!$class) { chomp $result; return if $result =~ m/^\s*$/; print "$result\n"; return; } if (($class eq 'HASH') && !scalar(keys %$result)) { # empty hash return; } print to_json($result, { pretty => 1, canonical => 1, utf8 => 1 }); }; sub param_mapping { my ($name) = @_; my $ssh_key_map = [ 'sshkeys', sub { return URI::Escape::uri_escape(file_get_contents($_[0])); }, ]; my $cipassword_map = PVE::CLIHandler::get_standard_mapping('pve-password', { name => 'cipassword' }); my $password_map = PVE::CLIHandler::get_standard_mapping('pve-password'); my $mapping = { 'update_vm' => [$ssh_key_map, $cipassword_map], 'create_vm' => [$ssh_key_map, $cipassword_map], 'set-user-password' => [$password_map], }; return $mapping->{$name}; } our $cmddef = { list => [ "PVE::API2::Qemu", 'vmlist', [], {%node}, sub { my $vmlist = shift; exit 0 if (!scalar(@$vmlist)); printf "%10s %-20s %-10s %-10s %12s %-10s\n", qw(VMID NAME STATUS MEM(MB) BOOTDISK(GB) PID); foreach my $rec (sort { $a->{vmid} <=> $b->{vmid} } @$vmlist) { printf "%10s %-20s %-10s %-10s %12.2f %-10s\n", $rec->{vmid}, $rec->{name}, $rec->{qmpstatus} || $rec->{status}, ($rec->{maxmem} || 0) / (1024 * 1024), ($rec->{maxdisk} || 0) / (1024 * 1024 * 1024), $rec->{pid} || 0; } }, ], create => ["PVE::API2::Qemu", 'create_vm', ['vmid'], {%node}, $upid_exit], destroy => ["PVE::API2::Qemu", 'destroy_vm', ['vmid'], {%node}, $upid_exit], clone => ["PVE::API2::Qemu", 'clone_vm', ['vmid', 'newid'], {%node}, $upid_exit], migrate => ["PVE::API2::Qemu", 'migrate_vm', ['vmid', 'target'], {%node}, $upid_exit], 'remote-migrate' => [ __PACKAGE__, 'remote_migrate_vm', ['vmid', 'target-vmid', 'target-endpoint'], {%node}, $upid_exit, ], set => ["PVE::API2::Qemu", 'update_vm', ['vmid'], {%node}], config => [ "PVE::API2::Qemu", 'vm_config', ['vmid'], {%node}, sub { my $config = shift; foreach my $k (sort (keys %$config)) { next if $k eq 'digest'; my $v = $config->{$k}; if ($k eq 'description') { $v = PVE::Tools::encode_text($v); } print "$k: $v\n"; } }, ], pending => ["PVE::API2::Qemu", 'vm_pending', ['vmid'], {%node}, \&PVE::GuestHelpers::format_pending], showcmd => [__PACKAGE__, 'showcmd', ['vmid']], status => [__PACKAGE__, 'status', ['vmid']], # FIXME: for 8.0 move to command group snapshot { create, list, destroy, rollback } snapshot => ["PVE::API2::Qemu", 'snapshot', ['vmid', 'snapname'], {%node}, $upid_exit], delsnapshot => ["PVE::API2::Qemu", 'delsnapshot', ['vmid', 'snapname'], {%node}, $upid_exit], listsnapshot => [ "PVE::API2::Qemu", 'snapshot_list', ['vmid'], {%node}, \&PVE::GuestHelpers::print_snapshot_tree, ], rollback => ["PVE::API2::Qemu", 'rollback', ['vmid', 'snapname'], {%node}, $upid_exit], template => ["PVE::API2::Qemu", 'template', ['vmid'], {%node}], # FIXME: should be in a power command group? start => ["PVE::API2::Qemu", 'vm_start', ['vmid'], {%node}, $upid_exit], stop => ["PVE::API2::Qemu", 'vm_stop', ['vmid'], {%node}, $upid_exit], reset => ["PVE::API2::Qemu", 'vm_reset', ['vmid'], {%node}, $upid_exit], shutdown => ["PVE::API2::Qemu", 'vm_shutdown', ['vmid'], {%node}, $upid_exit], reboot => ["PVE::API2::Qemu", 'vm_reboot', ['vmid'], {%node}, $upid_exit], suspend => ["PVE::API2::Qemu", 'vm_suspend', ['vmid'], {%node}, $upid_exit], resume => ["PVE::API2::Qemu", 'vm_resume', ['vmid'], {%node}, $upid_exit], sendkey => ["PVE::API2::Qemu", 'vm_sendkey', ['vmid', 'key'], {%node}], vncproxy => [__PACKAGE__, 'vncproxy', ['vmid']], wait => [__PACKAGE__, 'wait', ['vmid']], unlock => [__PACKAGE__, 'unlock', ['vmid']], # TODO: evaluate dropping below aliases for 8.0, if no usage is left importdisk => { alias => 'disk import' }, 'move-disk' => { alias => 'disk move' }, move_disk => { alias => 'disk move' }, rescan => { alias => 'disk rescan' }, resize => { alias => 'disk resize' }, unlink => { alias => 'disk unlink' }, disk => { import => [__PACKAGE__, 'importdisk', ['vmid', 'source', 'storage']], 'move' => ["PVE::API2::Qemu", 'move_vm_disk', ['vmid', 'disk', 'storage'], {%node}, $upid_exit], rescan => [__PACKAGE__, 'rescan', []], resize => ["PVE::API2::Qemu", 'resize_vm', ['vmid', 'disk', 'size'], {%node}], unlink => ["PVE::API2::Qemu", 'unlink', ['vmid'], {%node}], }, 'enroll-efi-keys' => [__PACKAGE__, 'enroll-efi-keys', ['vmid']], monitor => [__PACKAGE__, 'monitor', ['vmid']], agent => { alias => 'guest cmd' }, # FIXME: remove with PVE 8.0 guest => { cmd => ["PVE::API2::Qemu::Agent", 'agent', ['vmid', 'command'], {%node}, $print_agent_result], passwd => ["PVE::API2::Qemu::Agent", 'set-user-password', ['vmid', 'username'], {%node}], exec => [__PACKAGE__, 'exec', ['vmid', 'extra-args'], {%node}, $print_agent_result], 'exec-status' => [ "PVE::API2::Qemu::Agent", 'exec-status', ['vmid', 'pid'], {%node}, $print_agent_result, ], }, mtunnel => [__PACKAGE__, 'mtunnel', []], nbdstop => [__PACKAGE__, 'nbdstop', ['vmid']], terminal => [__PACKAGE__, 'terminal', ['vmid']], importovf => [__PACKAGE__, 'importovf', ['vmid', 'manifest', 'storage']], cleanup => [__PACKAGE__, 'cleanup', ['vmid', 'clean-shutdown', 'guest-requested'], {%node}], cloudinit => { dump => [ "PVE::API2::Qemu", 'cloudinit_generated_config_dump', ['vmid', 'type'], {%node}, sub { print "$_[0]\n"; }, ], pending => [ "PVE::API2::Qemu", 'cloudinit_pending', ['vmid'], {%node}, \&PVE::GuestHelpers::format_pending, ], update => ["PVE::API2::Qemu", 'cloudinit_update', ['vmid'], { node => $nodename }], }, import => [__PACKAGE__, 'vm_import', ['vmid', 'source']], }; 1; ================================================ FILE: src/PVE/CLI/qmrestore.pm ================================================ package PVE::CLI::qmrestore; use strict; use warnings; use PVE::SafeSyslog; use PVE::Tools qw(extract_param); use PVE::INotify; use PVE::RPCEnvironment; use PVE::CLIHandler; use PVE::JSONSchema qw(get_standard_option); use PVE::Cluster; use PVE::QemuServer; use PVE::API2::Qemu; use base qw(PVE::CLIHandler); sub setup_environment { PVE::RPCEnvironment->setup_default_cli_env(); } __PACKAGE__->register_method({ name => 'qmrestore', path => 'qmrestore', method => 'POST', description => "Restore QemuServer vzdump backups.", parameters => { additionalProperties => 0, properties => { vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }, ), archive => { description => "The backup file. You can pass '-' to read from standard input.", type => 'string', maxLength => 255, completion => \&PVE::QemuServer::complete_backup_archives, }, storage => get_standard_option( 'pve-storage-id', { description => "Default storage.", optional => 1, completion => \&PVE::QemuServer::complete_storage, }, ), force => { optional => 1, type => 'boolean', description => "Allow to overwrite existing VM.", }, unique => { optional => 1, type => 'boolean', description => "Assign a unique random ethernet address.", }, pool => { optional => 1, type => 'string', format => 'pve-poolid', description => "Add the VM to the specified pool.", }, bwlimit => { description => "Override I/O bandwidth limit (in KiB/s).", optional => 1, type => 'number', minimum => '0', }, 'live-restore' => { optional => 1, type => 'boolean', description => "Start the VM immediately from the backup and restore in background. PBS only.", }, start => { optional => 1, type => 'boolean', default => 0, description => "Start VM after it was restored successfully.", }, 'ha-managed' => { optional => 1, type => 'boolean', default => 0, description => "Add the VM as a HA resource after it was restored.", }, }, }, returns => { type => 'string', }, code => sub { my ($param) = @_; $param->{node} = PVE::INotify::nodename(); return PVE::API2::Qemu->create_vm($param); }, }); our $cmddef = [ __PACKAGE__, 'qmrestore', ['archive', 'vmid'], undef, sub { my $upid = shift; my $status = PVE::Tools::upid_read_status($upid); exit(PVE::Tools::upid_status_is_error($status) ? -1 : 0); }, ]; 1; ================================================ FILE: src/PVE/Makefile ================================================ DESTDIR= PREFIX=/usr PERLDIR=$(PREFIX)/share/perl5 PERLSOURCE = \ QemuServer.pm \ QemuMigrate.pm \ QMPClient.pm \ QemuConfig.pm .PHONY: install install: install -d $(DESTDIR)$(PERLDIR)/PVE install -m 0644 $(PERLSOURCE) $(DESTDIR)$(PERLDIR)/PVE/ $(MAKE) -C VZDump install $(MAKE) -C API2 install $(MAKE) -C CLI install $(MAKE) -C QemuConfig install $(MAKE) -C QemuMigrate install $(MAKE) -C QemuServer install ================================================ FILE: src/PVE/QMPClient.pm ================================================ package PVE::QMPClient; use strict; use warnings; use IO::Multiplex; use IO::Socket::UNIX; use JSON; use POSIX qw(EINTR EAGAIN); use Scalar::Util qw(weaken); use Time::HiRes qw(usleep gettimeofday tv_interval); use PVE::IPCC; use PVE::QemuServer::Helpers; # QEMU Monitor Protocol (QMP) client. # # This implementation uses IO::Multiplex (libio-multiplex-perl) and # allows you to issue qmp and qga commands to different VMs in parallel. # Note: qemu can only handle 1 connection, so we close connections asap sub new { my ($class, $eventcb) = @_; my $mux = IO::Multiplex->new(); my $self = bless { mux => $mux, queue_lookup => {}, # $fh => $queue_info queue_info => {}, }, $class; $self->{eventcb} = $eventcb if $eventcb; $mux->set_callback_object($self); # make sure perl doesn't believe this is a circular reference as we # delete mux in DESTROY weaken($mux->{_object}); return $self; } # Note: List of special QGA command. Those commands can close the connection # without sending a response. my $qga_allow_close_cmds = { 'guest-shutdown' => 1, 'guest-suspend-ram' => 1, 'guest-suspend-disk' => 1, 'guest-suspend-hybrid' => 1, }; my $push_cmd_to_queue = sub { my ($self, $peer, $cmd) = @_; my $execute = $cmd->{execute} || die "no command name specified"; my $sname = PVE::QemuServer::Helpers::qmp_socket($peer); $self->{queue_info}->{$sname} = { peer => $peer, sname => $sname, cmds => [] } if !$self->{queue_info}->{$sname}; push @{ $self->{queue_info}->{$sname}->{cmds} }, $cmd; return $self->{queue_info}->{$sname}; }; # add a single command to the queue for later execution # with queue_execute() sub queue_cmd { my ($self, $peer, $callback, $execute, %params) = @_; my $cmd = {}; $cmd->{execute} = $execute; $cmd->{arguments} = \%params; $cmd->{callback} = $callback; &$push_cmd_to_queue($self, $peer, $cmd); return; } # execute a single command sub cmd { my ($self, $peer, $cmd, $timeout, $noerr) = @_; my $result; my $callback = sub { my ($id, $resp) = @_; $result = $resp->{'return'}; $result = { error => $resp->{'error'} } if !defined($result) && $resp->{'error'}; }; die "no command specified" if !($cmd && $cmd->{execute}); $cmd->{callback} = $callback; $cmd->{arguments} = {} if !defined($cmd->{arguments}); my $queue_info = &$push_cmd_to_queue($self, $peer, $cmd); if (!$timeout) { # hack: monitor sometime blocks if ($cmd->{execute} eq 'query-migrate') { $timeout = 60 * 60; # 1 hour } elsif ($cmd->{execute} =~ m/^(eject|change)/) { $timeout = 60; # note: cdrom mount command is slow } elsif ($cmd->{execute} eq 'guest-fsfreeze-freeze') { # consider using the guest_fsfreeze() helper in Agent.pm # # freeze syncs all guest FS, if we kill it it stays in an unfreezable # locked state with high probability, so use an generous timeout $timeout = 60 * 60; # 1 hour } elsif ($cmd->{execute} eq 'guest-fsfreeze-thaw') { # While it should return instantly or never (dead locked) for Linux guests, # the variance for Windows guests can be big. And there might be hook scripts # that are executed upon thaw, so use 3 minutes to be on the safe side. $timeout = 3 * 60; } elsif ( $cmd->{execute} eq 'blockdev-add' || $cmd->{execute} eq 'blockdev-insert-medium' || $cmd->{execute} eq 'block-export-add' || $cmd->{execute} eq 'device_add' || $cmd->{execute} eq 'device_del' || $cmd->{execute} eq 'netdev_add' || $cmd->{execute} eq 'netdev_del' || $cmd->{execute} eq 'object-add' || $cmd->{execute} eq 'object-del' ) { $timeout = 60; } elsif ( $cmd->{execute} eq 'backup-cancel' || $cmd->{execute} eq 'block-commit' || $cmd->{execute} eq 'block-export-del' || $cmd->{execute} eq 'block-stream' || $cmd->{execute} eq 'blockdev-del' || $cmd->{execute} eq 'blockdev-mirror' || $cmd->{execute} eq 'blockdev-remove-medium' || $cmd->{execute} eq 'blockdev-reopen' || $cmd->{execute} eq 'block-job-cancel' || $cmd->{execute} eq 'job-complete' || $cmd->{execute} eq 'drive-mirror' || $cmd->{execute} eq 'guest-fstrim' || $cmd->{execute} eq 'guest-shutdown' || $cmd->{execute} eq 'query-backup' || $cmd->{execute} eq 'query-block-jobs' || $cmd->{execute} eq 'query-savevm' || $cmd->{execute} eq 'savevm-end' || $cmd->{execute} eq 'savevm-start' ) { $timeout = 10 * 60; # 10 mins } elsif ( $cmd->{execute} eq 'blockdev-snapshot-delete-internal-sync' || $cmd->{execute} eq 'blockdev-snapshot-internal-sync' ) { $timeout = 60 * 60; # 1 hour } else { # NOTE: if you came here as user and want to change this, try using IO-Threads first # which move out quite some processing of the main thread, leaving more time for QMP $timeout = 5; # default } } $self->queue_execute($timeout, 2); if (defined($queue_info->{error})) { die "$peer->{name} $peer->{type} command '$cmd->{execute}' failed - $queue_info->{error}" if !$noerr; $result = { error => $queue_info->{error} }; $result->{'error-is-timeout'} = 1 if $queue_info->{'error-is-timeout'}; } return $result; } my $cmdid_seq = 0; my $cmdid_seq_qga = 0; my $next_cmdid = sub { my ($qga) = @_; if ($qga) { $cmdid_seq_qga++; return "$$" . "0" . $cmdid_seq_qga; } else { $cmdid_seq++; return "$$:$cmdid_seq"; } }; my $lookup_queue_info = sub { my ($self, $fh, $noerr) = @_; my $queue_info = $self->{queue_lookup}->{$fh}; if (!$queue_info) { warn "internal error - unable to lookup queue info" if !$noerr; return; } return $queue_info; }; my $close_connection = sub { my ($self, $queue_info) = @_; if (my $fh = delete $queue_info->{fh}) { delete $self->{queue_lookup}->{$fh}; $self->{mux}->close($fh); } }; my $open_connection = sub { my ($self, $queue_info, $timeout) = @_; die "duplicate call to open" if defined($queue_info->{fh}); my $peer = $queue_info->{peer}; my ($peer_name, $sotype) = $peer->@{qw(name type)}; my $sname = PVE::QemuServer::Helpers::qmp_socket($peer); $timeout = 1 if !$timeout; my $fh; my $starttime = [gettimeofday]; my $count = 0; for (;;) { $count++; $fh = IO::Socket::UNIX->new(Peer => $sname, Blocking => 0, Timeout => 1); last if $fh; if ($! != EINTR && $! != EAGAIN) { die "unable to connect to $peer_name $sotype socket - $!\n"; } my $elapsed = tv_interval($starttime, [gettimeofday]); if ($elapsed >= $timeout) { die "unable to connect to $peer_name $sotype socket - timeout after $count retries\n"; } usleep(100000); } $queue_info->{fh} = $fh; $self->{queue_lookup}->{$fh} = $queue_info; $self->{mux}->add($fh); $self->{mux}->set_timeout($fh, $timeout); return $fh; }; my $check_queue = sub { my ($self) = @_; my $running = 0; foreach my $sname (keys %{ $self->{queue_info} }) { my $queue_info = $self->{queue_info}->{$sname}; my $fh = $queue_info->{fh}; next if !$fh; my $qga = $queue_info->{peer}->{type} eq 'qga'; if ($queue_info->{error}) { &$close_connection($self, $queue_info); next; } if ($queue_info->{current}) { # command running, waiting for response $running++; next; } if (!scalar(@{ $queue_info->{cmds} })) { # no more commands &$close_connection($self, $queue_info); next; } eval { my $cmd = $queue_info->{current} = shift @{ $queue_info->{cmds} }; $cmd->{id} = &$next_cmdid($qga); my $fd = -1; if ($cmd->{execute} eq 'add-fd' || $cmd->{execute} eq 'getfd') { $fd = $cmd->{arguments}->{fd}; delete $cmd->{arguments}->{fd}; } my $qmpcmd; if ($qga) { $qmpcmd = to_json({ execute => 'guest-sync-delimited', arguments => { id => int($cmd->{id}) }, }) . "\n" . to_json({ execute => $cmd->{execute}, arguments => $cmd->{arguments} }) . "\n"; } else { $qmpcmd = to_json({ execute => $cmd->{execute}, arguments => $cmd->{arguments}, id => $cmd->{id}, }); } if ($fd >= 0) { my $ret = PVE::IPCC::sendfd(fileno($fh), $fd, $qmpcmd); die "sendfd failed" if $ret < 0; } else { $self->{mux}->write($fh, $qmpcmd); } }; if (my $err = $@) { $queue_info->{error} = $err; } else { $running++; } } $self->{mux}->endloop() if !$running; return $running; }; # execute all queued command sub queue_execute { my ($self, $timeout, $noerr) = @_; $timeout = 3 if !$timeout; # open all necessary connections foreach my $sname (keys %{ $self->{queue_info} }) { my $queue_info = $self->{queue_info}->{$sname}; next if !scalar(@{ $queue_info->{cmds} }); # no commands $queue_info->{error} = undef; $queue_info->{current} = undef; eval { &$open_connection($self, $queue_info, $timeout); if ($queue_info->{peer}->{type} ne 'qga') { my $cap_cmd = { execute => 'qmp_capabilities', arguments => {} }; unshift @{ $queue_info->{cmds} }, $cap_cmd; } }; if (my $err = $@) { $queue_info->{error} = $err; } } my $running; for (;;) { $running = &$check_queue($self); last if !$running; $self->{mux}->loop; } # make sure we close everything my $errors = ''; foreach my $sname (keys %{ $self->{queue_info} }) { my $queue_info = $self->{queue_info}->{$sname}; &$close_connection($self, $queue_info); if ($queue_info->{error}) { if ($noerr) { warn $queue_info->{error} if $noerr < 2; } else { $errors .= $queue_info->{error}; } } } $self->{queue_info} = $self->{queue_lookup} = {}; die $errors if $errors; } sub mux_close { my ($self, $mux, $fh) = @_; my $queue_info = &$lookup_queue_info($self, $fh, 1); return if !$queue_info; $queue_info->{error} = "client closed connection\n" if !$queue_info->{error}; } # mux_input is called when input is available on one of the descriptors. sub mux_input { my ($self, $mux, $fh, $input) = @_; my $queue_info = &$lookup_queue_info($self, $fh); return if !$queue_info; my $sname = $queue_info->{sname}; my ($id, $peer_name) = $queue_info->{peer}->@{qw(id name)}; my $qga = $queue_info->{peer}->{type} eq 'qga'; my $curcmd = $queue_info->{current}; die "unable to lookup current command for $peer_name ($sname)\n" if !$curcmd; my $raw; if ($qga) { return if $$input !~ s/^.*\xff([^\n]+}\r?\n[^\n]+})\r?\n(.*)$/$2/so; $raw = $1; } else { return if $$input !~ s/^(.*})\r?\n(.*)$/$2/so; $raw = $1; } eval { my @jsons = split("\n", $raw); if ($qga) { die "response is not complete" if @jsons != 2; my $obj = from_json($jsons[0]); my $cmdid = $obj->{'return'}; die "received response without command id\n" if !$cmdid; # skip results from previous commands return if $cmdid < $curcmd->{id}; if ($curcmd->{id} ne $cmdid) { die "got wrong command id '$cmdid' (expected $curcmd->{id})\n"; } delete $queue_info->{current}; $obj = from_json($jsons[1]); if (my $callback = $curcmd->{callback}) { &$callback($id, $obj); } return; } foreach my $json (@jsons) { my $obj = from_json($json); next if defined($obj->{QMP}); # skip monitor greeting if (exists($obj->{error}->{desc})) { my $desc = $obj->{error}->{desc}; chomp $desc; die "$desc\n" if $desc !~ m/Connection can not be completed immediately/; next; } if (defined($obj->{event})) { if (my $eventcb = $self->{eventcb}) { &$eventcb($obj); } next; } my $cmdid = $obj->{id}; die "received response without command id\n" if !$cmdid; if ($curcmd->{id} ne $cmdid) { die "got wrong command id '$cmdid' (expected $curcmd->{id})\n"; } delete $queue_info->{current}; if (my $callback = $curcmd->{callback}) { &$callback($id, $obj); } } }; if (my $err = $@) { $queue_info->{error} = $err; } &$check_queue($self); } sub mux_timeout { my ($self, $mux, $fh) = @_; if (my $queue_info = &$lookup_queue_info($self, $fh)) { $queue_info->{error} = "got timeout\n"; $queue_info->{'error-is-timeout'} = 1; $self->{mux}->inbuffer($fh, ''); # clear to avoid warnings } &$check_queue($self); } sub mux_eof { my ($self, $mux, $fh, $input) = @_; my $queue_info = &$lookup_queue_info($self, $fh); return if !$queue_info; my $sname = $queue_info->{sname}; my ($id, $peer_name) = $queue_info->{peer}->@{qw(id name)}; my $qga = $queue_info->{peer}->{type} eq 'qga'; my $curcmd = $queue_info->{current}; die "unable to lookup current command for $peer_name ($sname)\n" if !$curcmd; if ($qga && $qga_allow_close_cmds->{ $curcmd->{execute} }) { return if $$input !~ s/^.*\xff([^\n]+})\r?\n(.*)$/$2/so; my $raw = $1; eval { my $obj = from_json($raw); my $cmdid = $obj->{'return'}; die "received response without command id\n" if !$cmdid; delete $queue_info->{current}; if (my $callback = $curcmd->{callback}) { &$callback($id, undef); } }; if (my $err = $@) { $queue_info->{error} = $err; } &$close_connection($self, $queue_info); if (scalar(@{ $queue_info->{cmds} }) && !$queue_info->{error}) { $queue_info->{error} = "Got EOF but command queue is not empty.\n"; } } } sub DESTROY { my ($self) = @_; foreach my $sname (keys %{ $self->{queue_info} }) { my $queue_info = $self->{queue_info}->{$sname}; $close_connection->($self, $queue_info); } } 1; ================================================ FILE: src/PVE/QemuConfig/Makefile ================================================ DESTDIR= PREFIX=/usr PERLDIR=$(PREFIX)/share/perl5 SOURCES=NoWrite.pm .PHONY: install install: $(SOURCES) for i in $(SOURCES); do install -D -m 0644 $$i $(DESTDIR)$(PERLDIR)/PVE/QemuConfig/$$i; done ================================================ FILE: src/PVE/QemuConfig/NoWrite.pm ================================================ package PVE::QemuConfig::NoWrite; use strict; use warnings; use PVE::RESTEnvironment qw(log_warn); use base qw(PVE::QemuConfig); sub mark_config { my ($class, $conf) = @_; bless($conf, $class); } sub write_config { my ($class, $vmid, $conf) = @_; die("refusing to write temporary configuration\n"); } 1; ================================================ FILE: src/PVE/QemuConfig.pm ================================================ package PVE::QemuConfig; use strict; use warnings; use Scalar::Util qw(blessed); use PVE::AbstractConfig; use PVE::INotify; use PVE::JSONSchema; use PVE::QemuMigrate::Helpers; use PVE::QemuServer::Agent; use PVE::QemuServer::Blockdev; use PVE::QemuServer::CPUConfig; use PVE::QemuServer::Drive; use PVE::QemuServer::Helpers; use PVE::QemuServer::Monitor qw(mon_cmd); use PVE::QemuServer; use PVE::QemuServer::Machine; use PVE::QemuServer::Memory qw(get_current_memory); use PVE::RESTEnvironment qw(log_warn); use PVE::Storage; use PVE::Tools; use PVE::Format qw(render_bytes render_duration); use base qw(PVE::AbstractConfig); my $nodename = PVE::INotify::nodename(); mkdir "/etc/pve/nodes/$nodename"; mkdir "/etc/pve/nodes/$nodename/qemu-server"; my $lock_dir = "/var/lock/qemu-server"; mkdir $lock_dir; sub assert_config_exists_on_node { my ($vmid, $node) = @_; $node //= $nodename; my $filename = __PACKAGE__->config_file($vmid, $node); my $exists = -f $filename; my $type = guest_type(); die "unable to find configuration file for $type $vmid on node '$node'\n" if !$exists; } # BEGIN implemented abstract methods from PVE::AbstractConfig sub guest_type { return "VM"; } sub __config_max_unused_disks { my ($class) = @_; return $PVE::QemuServer::Drive::MAX_UNUSED_DISKS; } sub config_file_lock { my ($class, $vmid) = @_; return "$lock_dir/lock-$vmid.conf"; } sub cfs_config_path { my ($class, $vmid, $node) = @_; $node = $nodename if !$node; return "nodes/$node/qemu-server/$vmid.conf"; } sub has_feature { my ($class, $feature, $conf, $storecfg, $snapname, $running, $backup_only) = @_; my $err; $class->foreach_volume( $conf, sub { my ($ds, $drive) = @_; return if PVE::QemuServer::Drive::drive_is_cdrom($drive); return if $backup_only && defined($drive->{backup}) && !$drive->{backup}; my $volid = $drive->{file}; $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running); }, ); return $err ? 0 : 1; } sub valid_volume_keys { my ($class, $reverse) = @_; my @keys = PVE::QemuServer::Drive::valid_drive_names(); return $reverse ? reverse @keys : @keys; } # FIXME: adapt parse_drive to use $noerr for better error messages sub parse_volume { my ($class, $key, $volume_string, $noerr) = @_; my $volume; if ($key eq 'vmstate') { eval { PVE::JSONSchema::check_format('pve-volume-id', $volume_string) }; if (my $err = $@) { return if $noerr; die $err; } $volume = { 'file' => $volume_string }; } else { $volume = PVE::QemuServer::Drive::parse_drive($key, $volume_string); } die "unable to parse volume\n" if !defined($volume) && !$noerr; return $volume; } sub print_volume { my ($class, $key, $volume) = @_; return PVE::QemuServer::Drive::print_drive($volume); } sub volid_key { my ($class) = @_; return 'file'; } sub get_replicatable_volumes { my ($class, $storecfg, $vmid, $conf, $cleanup, $noerr) = @_; my $volhash = {}; my $test_volid = sub { my ($volid, $attr) = @_; return if $attr->{cdrom}; return if !$cleanup && !$attr->{replicate}; if ($volid =~ m|^/|) { return if !$attr->{replicate}; return if $cleanup || $noerr; die "unable to replicate local file/device '$volid'\n"; } my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, $noerr); return if !$storeid; my $scfg = PVE::Storage::storage_config($storecfg, $storeid); return if $scfg->{shared}; my ($path, $owner, $vtype) = PVE::Storage::path($storecfg, $volid); return if !$owner || ($owner != $vmid); if ($vtype ne 'images') { return if $cleanup || $noerr; die "unable to replicate volume '$volid', type '$vtype'\n"; } if (!PVE::Storage::volume_has_feature($storecfg, 'replicate', $volid)) { return if $cleanup || $noerr; die "missing replicate feature on volume '$volid'\n"; } $volhash->{$volid} = 1; }; PVE::QemuServer::foreach_volid($conf, $test_volid); return $volhash; } sub get_backup_volumes { my ($class, $conf) = @_; my $return_volumes = []; my $test_volume = sub { my ($key, $drive) = @_; return if PVE::QemuServer::Drive::drive_is_cdrom($drive); my $included = $drive->{backup} // 1; my $reason = "backup="; $reason .= defined($drive->{backup}) ? 'no' : 'yes'; if ($key =~ m/^efidisk/ && (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf')) { $included = 0; $reason = "efidisk but no OVMF BIOS"; } push @$return_volumes, { key => $key, included => $included, reason => $reason, volume_config => $drive, }; }; PVE::QemuConfig->foreach_volume($conf, $test_volume); return $return_volumes; } sub __snapshot_assert_no_blockers { my ($class, $vmconf, $save_vmstate) = @_; PVE::QemuMigrate::Helpers::check_non_migratable_resources($vmconf, $save_vmstate, 0); } sub __snapshot_save_vmstate { my ($class, $vmid, $conf, $snapname, $storecfg, $statestorage, $suspend) = @_; # use given storage or search for one from the config my $target = $statestorage; if (!$target) { $target = find_vmstate_storage($conf, $storecfg); } my $mem_size = get_current_memory($conf->{memory}); my $driver_state_size = 500; # assume 500MB is enough to safe all driver state; # our savevm-start does live-save of the memory until the space left in the # volume is just enough for the remaining memory content + internal state # then it stops the vm and copies the rest so we reserve twice the # memory content + state to minimize vm downtime my $size = $mem_size * 2 + $driver_state_size; my $scfg = PVE::Storage::storage_config($storecfg, $target); my $name = "vm-$vmid-state-$snapname"; $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage my $statefile = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size * 1024); my $runningmachine = PVE::QemuServer::Machine::get_current_qemu_machine($vmid); # get current QEMU -cpu argument to ensure consistency of custom CPU models my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid) or die "cannot obtain PID for VM $vmid!\n"; my $runningcpu = PVE::QemuServer::CPUConfig::get_cpu_from_running_vm($pid); my $nets_host_mtu = PVE::QemuServer::Network::get_nets_host_mtu($vmid, $conf); if (!$suspend) { $conf = $conf->{snapshots}->{$snapname}; } $conf->{vmstate} = $statefile; $conf->{runningmachine} = $runningmachine; $conf->{runningcpu} = $runningcpu; $conf->{'running-nets-host-mtu'} = $nets_host_mtu; return $statefile; } sub __snapshot_activate_storages { my ($class, $conf, $include_vmstate) = @_; my $storecfg = PVE::Storage::config(); my $opts = $include_vmstate ? { 'extra_keys' => ['vmstate'] } : {}; my $storage_hash = {}; $class->foreach_volume_full( $conf, $opts, sub { my ($key, $drive) = @_; return if PVE::QemuServer::Drive::drive_is_cdrom($drive); my ($storeid) = PVE::Storage::parse_volume_id($drive->{file}); $storage_hash->{$storeid} = 1; }, ); PVE::Storage::activate_storage_list($storecfg, [sort keys $storage_hash->%*]); } sub __snapshot_check_running { my ($class, $vmid) = @_; return PVE::QemuServer::Helpers::vm_running_locally($vmid); } sub __snapshot_check_freeze_needed { my ($class, $vmid, $config, $save_vmstate) = @_; my $running = $class->__snapshot_check_running($vmid); if (!$save_vmstate) { return ( $running, $running && PVE::QemuServer::Agent::should_fs_freeze($config) && PVE::QemuServer::Agent::qga_check_running($vmid), ); } else { return ($running, 0); } } sub __snapshot_freeze { my ($class, $vmid, $unfreeze) = @_; if ($unfreeze) { eval { PVE::QemuServer::Agent::guest_fsthaw($vmid); }; warn "guest-fsfreeze-thaw problems - $@" if $@; } else { eval { PVE::QemuServer::Agent::guest_fsfreeze($vmid); }; warn $@ if $@; } } sub __snapshot_create_vol_snapshots_hook { my ($class, $vmid, $snap, $running, $hook) = @_; if ($running) { my $storecfg = PVE::Storage::config(); if ($hook eq "before") { if ($snap->{vmstate}) { my $path = PVE::Storage::path($storecfg, $snap->{vmstate}); PVE::Storage::activate_volumes($storecfg, [$snap->{vmstate}]); my $state_storage_id = PVE::Storage::parse_volume_id($snap->{vmstate}); PVE::QemuMigrate::Helpers::set_migration_caps($vmid, 1); mon_cmd($vmid, "savevm-start", statefile => $path); print "saving VM state and RAM using storage '$state_storage_id'\n"; my $render_state = sub { my ($stat) = @_; my $b = render_bytes($stat->{bytes}); my $t = render_duration($stat->{'total-time'} / 1000); return ($b, $t); }; my $round = 0; for (;;) { $round++; my $stat = mon_cmd($vmid, "query-savevm"); if (!$stat->{status}) { die "savevm not active\n"; } elsif ($stat->{status} eq 'active') { if ($round < 60 || $round % 10 == 0) { my ($b, $t) = $render_state->($stat); print "$b in $t\n"; } print "reducing reporting rate to every 10s\n" if $round == 60; sleep(1); next; } elsif ($stat->{status} eq 'completed') { my ($b, $t) = $render_state->($stat); print "completed saving the VM state in $t, saved $b\n"; last; } elsif ($stat->{status} eq 'failed') { my $err = $stat->{error} || 'unknown error'; die "unable to save VM state and RAM - $err\n"; } else { die "query-savevm returned unexpected status '$stat->{status}'\n"; } } } else { mon_cmd($vmid, "savevm-start"); } } elsif ($hook eq "after") { eval { mon_cmd($vmid, "savevm-end"); PVE::Storage::deactivate_volumes($storecfg, [$snap->{vmstate}]) if $snap->{vmstate}; }; warn $@ if $@; } elsif ($hook eq "after-freeze") { # savevm-end is async, we need to wait for (;;) { my $stat = mon_cmd($vmid, "query-savevm"); if (!$stat->{bytes}) { last; } else { print "savevm not yet finished\n"; sleep(1); next; } } } } } sub __snapshot_create_vol_snapshot { my ($class, $vmid, $ds, $drive, $snapname) = @_; return if PVE::QemuServer::Drive::drive_is_cdrom($drive); my $volid = $drive->{file}; my $device = "drive-$ds"; my $storecfg = PVE::Storage::config(); print "snapshotting '$device' ($drive->{file})\n"; PVE::QemuServer::qemu_volume_snapshot($vmid, $device, $storecfg, $drive, $snapname); } sub __snapshot_delete_remove_drive { my ($class, $snap, $remove_drive) = @_; if ($remove_drive eq 'vmstate') { delete $snap->{$remove_drive}; } else { my $drive = PVE::QemuServer::Drive::parse_drive($remove_drive, $snap->{$remove_drive}); return if PVE::QemuServer::Drive::drive_is_cdrom($drive); my $volid = $drive->{file}; delete $snap->{$remove_drive}; $class->add_unused_volume($snap, $volid); } } sub __snapshot_delete_vmstate_file { my ($class, $snap, $force) = @_; my $storecfg = PVE::Storage::config(); eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); }; if (my $err = $@) { die $err if !$force; warn $err; } } sub __snapshot_delete_vol_snapshot { my ($class, $vmid, $ds, $drive, $snapname, $unused) = @_; return if PVE::QemuServer::Drive::drive_is_cdrom($drive); my $storecfg = PVE::Storage::config(); my $volid = $drive->{file}; PVE::QemuServer::qemu_volume_snapshot_delete($vmid, $storecfg, $drive, $snapname); push @$unused, $volid; } sub __snapshot_rollback_hook { my ($class, $vmid, $conf, $snap, $prepare, $data) = @_; if ($prepare) { # we save the machine of the current config $data->{oldmachine} = $conf->{machine}; } else { # if we have a 'runningmachine' entry in the snapshot we use that # for the forcemachine parameter, else we use the old logic if (defined($conf->{runningmachine})) { $data->{forcemachine} = $conf->{runningmachine}; delete $conf->{runningmachine}; # runningcpu is newer than runningmachine, so assume it only exists # here, if at all $data->{forcecpu} = delete $conf->{runningcpu} if defined($conf->{runningcpu}); } else { # Note: old code did not store 'machine', so we try to be smart # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4). my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine}); $data->{forcemachine} = $machine_conf->{type} || 'pc-i440fx-1.4'; # we remove the 'machine' configuration if not explicitly specified # in the original config. delete $conf->{machine} if $snap->{vmstate} && !defined($data->{oldmachine}); } if ($conf->{vmgenid}) { # tell the VM that it's another generation, so it can react # appropriately, e.g. dirty-mark copies of distributed databases or # re-initializing its random number generator $conf->{vmgenid} = PVE::QemuServer::generate_uuid(); } $data->{'nets-host-mtu'} = delete($conf->{'running-nets-host-mtu'}); } return; } sub __snapshot_rollback_vol_possible { my ($class, $drive, $snapname, $blockers) = @_; return if PVE::QemuServer::Drive::drive_is_cdrom($drive); my $storecfg = PVE::Storage::config(); my $volid = $drive->{file}; PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname, $blockers); } sub __snapshot_rollback_vol_rollback { my ($class, $drive, $snapname) = @_; return if PVE::QemuServer::Drive::drive_is_cdrom($drive); my $storecfg = PVE::Storage::config(); PVE::Storage::volume_snapshot_rollback($storecfg, $drive->{file}, $snapname); } sub __snapshot_rollback_vm_stop { my ($class, $vmid) = @_; my $storecfg = PVE::Storage::config(); PVE::QemuServer::vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef); } sub __snapshot_rollback_vm_start { my ($class, $vmid, $vmstate, $data) = @_; my $storecfg = PVE::Storage::config(); my $params = { statefile => $vmstate, forcemachine => $data->{forcemachine}, forcecpu => $data->{forcecpu}, 'nets-host-mtu' => $data->{'nets-host-mtu'}, }; PVE::QemuServer::vm_start($storecfg, $vmid, $params); } sub __snapshot_rollback_get_unused { my ($class, $conf, $snap) = @_; my $unused = []; $class->foreach_volume( $conf, sub { my ($vs, $volume) = @_; return if PVE::QemuServer::Drive::drive_is_cdrom($volume, 1); my $found = 0; my $volid = $volume->{file}; $class->foreach_volume( $snap, sub { my ($ds, $drive) = @_; return if $found; return if PVE::QemuServer::Drive::drive_is_cdrom($drive, 1); $found = 1 if ($drive->{file} && $drive->{file} eq $volid); }, ); push @$unused, $volid if !$found; }, ); return $unused; } sub add_unused_volume { my ($class, $config, $volid) = @_; if ($volid =~ m/vm-\d+-cloudinit/) { print "found unused cloudinit disk '$volid', removing it\n"; my $storecfg = PVE::Storage::config(); PVE::Storage::vdisk_free($storecfg, $volid); return undef; } else { return $class->SUPER::add_unused_volume($config, $volid); } } sub load_current_config { my ($class, $vmid, $current) = @_; my $conf = $class->SUPER::load_current_config($vmid, $current); delete $conf->{'special-sections'}; return $conf; } sub get_derived_property { my ($class, $conf, $name) = @_; if ($name eq 'max-cpu') { my $sockets = $conf->{sockets} || PVE::QemuServer::get_default_property_value('sockets'); my $cores = $conf->{cores} || PVE::QemuServer::get_default_property_value('cores'); return $conf->{vcpus} || ($sockets * $cores); } elsif ($name eq 'max-memory') { # current usage maximum, not maximum hotpluggable return get_current_memory($conf->{memory}) * 1024 * 1024; } else { die "unknown derived property - $name\n"; } } sub write_config { my ($class, $vmid, $conf) = @_; # Dispatch to class the object was blessed with if caller invoked the method via the # 'PVE::QemuConfig' class name explicitly. This is hack, but the code currently doesn't # generally use blessed config objects. Safeguard against infinite recursion. if (blessed($conf) && !blessed($class)) { return $conf->write_config($vmid, $conf); } return $class->SUPER::write_config($vmid, $conf); } # END implemented abstract methods from PVE::AbstractConfig sub has_cloudinit { my ($class, $conf, $skip) = @_; my $found; $class->foreach_volume( $conf, sub { my ($key, $volume) = @_; return if ($skip && $skip eq $key) || $found; $found = $key if PVE::QemuServer::Drive::drive_is_cloudinit($volume); }, ); return $found; } # Caller is expected to deal with volumes from an already existing 'fleecing' special section in the # configuration first. sub record_fleecing_images { my ($vmid, $volids) = @_; return if scalar($volids->@*) == 0; PVE::QemuConfig->lock_config( $vmid, sub { my $conf = PVE::QemuConfig->load_config($vmid); $conf->{'special-sections'}->{fleecing}->{'fleecing-images'} = join(',', $volids->@*); PVE::QemuConfig->write_config($vmid, $conf); }, ); } # Will also cancel a running backup job inside QEMU. Not doing so can lead to a deadlock when # attempting to detach the fleecing image. sub cleanup_fleecing_images { my ($vmid, $storecfg, $log_func) = @_; if (!$log_func) { $log_func = sub { my ($level, $line) = @_; chomp($line); if ($level eq 'info') { print "$line\n"; } else { log_warn($line); } }; } my $volids = []; my $failed = []; # cancel left-over backup job and detach any left-over images from a running VM if (PVE::QemuServer::Helpers::vm_running_locally($vmid)) { eval { if (my $status = mon_cmd($vmid, 'query-backup')) { if ($status->{status} && $status->{status} eq 'active') { $log_func->( 'warn', "left-over backup job still running inside QEMU - canceling now", ); mon_cmd($vmid, 'backup-cancel'); } } }; $log_func->('warn', "checking/canceling old backup job failed - $@") if $@; PVE::QemuServer::Blockdev::detach_fleecing_block_nodes($vmid, $log_func); } PVE::QemuConfig->lock_config( $vmid, sub { my $conf = PVE::QemuConfig->load_config($vmid); my $special = $conf->{'special-sections'}; if (my $fleecing = $special->{fleecing}) { $volids = [PVE::Tools::split_list($fleecing->{'fleecing-images'})]; delete $fleecing->{'fleecing-images'}; delete $special->{fleecing} if !scalar(keys $fleecing->%*); PVE::QemuConfig->write_config($vmid, $conf); } }, ); for my $volid ($volids->@*) { $log_func->('info', "removing (old) fleecing image '$volid'"); eval { PVE::Storage::vdisk_free($storecfg, $volid); }; if (my $err = $@) { $log_func->('warn', "error removing fleecing image '$volid' - $err"); push $failed->@*, $volid; } } record_fleecing_images($vmid, $failed); } sub foreach_storage_used_by_vm { my ($conf, $func) = @_; my $sidhash = {}; PVE::QemuConfig->foreach_volume( $conf, sub { my ($ds, $drive) = @_; return if PVE::QemuServer::Drive::drive_is_cdrom($drive); my $volid = $drive->{file}; my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1); $sidhash->{$sid} = $sid if $sid; }, ); foreach my $sid (sort keys %$sidhash) { &$func($sid); } } # NOTE: if this logic changes, please update docs & possibly gui logic sub find_vmstate_storage { my ($conf, $storecfg) = @_; # first, return storage from conf if set return $conf->{vmstatestorage} if $conf->{vmstatestorage}; my ($target, $shared, $local); foreach_storage_used_by_vm( $conf, sub { my ($sid) = @_; my $scfg = PVE::Storage::storage_config($storecfg, $sid); my $dst = $scfg->{shared} ? \$shared : \$local; $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage }, ); # second, use shared storage where VM has at least one disk # third, use local storage where VM has at least one disk # fall back to local storage $target = $shared // $local // 'local'; return $target; } 1; ================================================ FILE: src/PVE/QemuMigrate/Helpers.pm ================================================ package PVE::QemuMigrate::Helpers; use strict; use warnings; use JSON; use PVE::Cluster; use PVE::JSONSchema qw(parse_property_string); use PVE::Mapping::Dir; use PVE::Mapping::PCI; use PVE::Mapping::USB; use PVE::QemuServer::Monitor qw(mon_cmd); use PVE::QemuServer::Virtiofs; sub check_non_migratable_resources { my ($conf, $state, $noerr) = @_; my @blockers = (); if ($state) { push @blockers, "amd-sev" if $conf->{"amd-sev"}; push @blockers, "intel-tdx" if $conf->{"intel-tdx"}; push @blockers, "virtiofs" if PVE::QemuServer::Virtiofs::virtiofs_enabled($conf); } if (scalar(@blockers) && !$noerr) { die "Cannot live-migrate, snapshot (with RAM), or hibernate a VM with: " . join(', ', @blockers) . "\n"; } return @blockers; } # test if VM uses local resources (to prevent migration) sub check_local_resources { my ($conf, $state, $noerr) = @_; my @loc_res = (); my $mapped_res = {}; my @non_migratable_resources = check_non_migratable_resources($conf, $state, $noerr); push(@loc_res, @non_migratable_resources); my $nodelist = PVE::Cluster::get_nodelist(); my $pci_map = PVE::Mapping::PCI::config(); my $usb_map = PVE::Mapping::USB::config(); my $dir_map = PVE::Mapping::Dir::config(); my $missing_mappings_by_node = { map { $_ => [] } @$nodelist }; my $add_missing_mapping = sub { my ($type, $key, $id) = @_; for my $node (@$nodelist) { my $entry; if ($type eq 'pci') { $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node); } elsif ($type eq 'usb') { $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node); } elsif ($type eq 'dir') { $entry = PVE::Mapping::Dir::get_node_mapping($dir_map, $id, $node); } if (!scalar($entry->@*)) { push @{ $missing_mappings_by_node->{$node} }, $key; } } }; push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax push @loc_res, "ivshmem" if $conf->{ivshmem}; foreach my $k (keys %$conf) { if ($k =~ m/^usb/) { my $entry = parse_property_string('pve-qm-usb', $conf->{$k}); next if $entry->{host} && $entry->{host} =~ m/^spice$/i; if (my $name = $entry->{mapping}) { $add_missing_mapping->('usb', $k, $name); $mapped_res->{$k} = { name => $name }; } } if ($k =~ m/^hostpci/) { my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k}); if (my $name = $entry->{mapping}) { $add_missing_mapping->('pci', $k, $name); my $mapped_device = { name => $name }; $mapped_res->{$k} = $mapped_device; if ($pci_map->{ids}->{$name}->{'live-migration-capable'}) { $mapped_device->{'live-migration'} = 1; # don't add mapped device with live migration as blocker next; } # don't add mapped devices as blocker for offline migration but still iterate over # all mappings above to collect on which nodes they are available. next if !$state; } } if ($k =~ m/^virtiofs/) { my $entry = parse_property_string('pve-qm-virtiofs', $conf->{$k}); $add_missing_mapping->('dir', $k, $entry->{dirid}); $mapped_res->{$k} = { name => $entry->{dirid} }; } # sockets are safe: they will recreated be on the target side post-migrate next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket'); push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel|virtiofs)\d+$/; } die "VM uses local resources\n" if scalar @loc_res && !$noerr; return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res; } sub set_migration_caps { my ($vmid, $savevm) = @_; my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") }; my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration'; my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0; my $cap_ref = []; my $enabled_cap = { "auto-converge" => 1, "xbzrle" => 1, "dirty-bitmaps" => $dirty_bitmaps, }; my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities"); for my $supported_capability (@$supported_capabilities) { push @$cap_ref, { capability => $supported_capability->{capability}, state => $enabled_cap->{ $supported_capability->{capability} } ? JSON::true : JSON::false, }; } mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref); } 1; ================================================ FILE: src/PVE/QemuMigrate/Makefile ================================================ DESTDIR= PREFIX=/usr PERLDIR=$(PREFIX)/share/perl5 SOURCES=Helpers.pm .PHONY: install install: $(SOURCES) for i in $(SOURCES); do install -D -m 0644 $$i $(DESTDIR)$(PERLDIR)/PVE/QemuMigrate/$$i; done ================================================ FILE: src/PVE/QemuMigrate.pm ================================================ package PVE::QemuMigrate; use strict; use warnings; use IO::File; use IPC::Open2; use Storable qw(dclone); use Time::HiRes qw( usleep ); use PVE::AccessControl; use PVE::Cluster; use PVE::Format qw(render_bytes); use PVE::Firewall::Helpers; use PVE::GuestHelpers qw(safe_boolean_ne safe_string_ne); use PVE::INotify; use PVE::JSONSchema; use PVE::RPCEnvironment; use PVE::Replication; use PVE::ReplicationConfig; use PVE::ReplicationState; use PVE::Storage::Plugin; use PVE::Storage; use PVE::StorageTunnel; use PVE::Tools; use PVE::Tunnel; use PVE::QemuConfig; use PVE::QemuMigrate::Helpers; use PVE::QemuServer::Agent; use PVE::QemuServer::BlockJob; use PVE::QemuServer::CPUConfig; use PVE::QemuServer::Drive qw(checked_volume_format); use PVE::QemuServer::Helpers qw(min_version); use PVE::QemuServer::Machine; use PVE::QemuServer::Monitor qw(mon_cmd vm_qmp_peer); use PVE::QemuServer::Memory qw(get_current_memory); use PVE::QemuServer::Network; use PVE::QemuServer::QMPHelpers; use PVE::QemuServer::DBusVMState; use PVE::QemuServer; use PVE::AbstractMigrate; use base qw(PVE::AbstractMigrate); # compared against remote end's minimum version our $WS_TUNNEL_VERSION = 2; sub fork_tunnel { my ($self, $ssh_forward_info) = @_; my $cmd = ['/usr/sbin/qm', 'mtunnel']; my $log = sub { my ($level, $msg) = @_; $self->log($level, $msg); }; return PVE::Tunnel::fork_ssh_tunnel($self->{rem_ssh}, $cmd, $ssh_forward_info, $log); } sub fork_websocket_tunnel { my ($self, $storages, $bridges) = @_; my $remote = $self->{opts}->{remote}; my $conn = $remote->{conn}; my $log = sub { my ($level, $msg) = @_; $self->log($level, $msg); }; my $websocket_url = "https://$conn->{host}:$conn->{port}/api2/json/nodes/$self->{node}/qemu/$remote->{vmid}/mtunnelwebsocket"; my $url = "/nodes/$self->{node}/qemu/$remote->{vmid}/mtunnel"; my $tunnel_params = { url => $websocket_url, }; my $storage_list = join(',', keys %$storages); my $bridge_list = join(',', keys %$bridges); my $req_params = { storages => $storage_list, bridges => $bridge_list, }; return PVE::Tunnel::fork_websocket_tunnel($conn, $url, $req_params, $tunnel_params, $log); } # tunnel_info: # proto: unix (secure) or tcp (insecure/legacy compat) # addr: IP or UNIX socket path # port: optional TCP port # unix_sockets: additional UNIX socket paths to forward sub start_remote_tunnel { my ($self, $tunnel_info) = @_; my $nodename = PVE::INotify::nodename(); my $migration_type = $self->{opts}->{migration_type}; if ($migration_type eq 'secure') { if ($tunnel_info->{proto} eq 'unix') { my $ssh_forward_info = []; my $unix_sockets = [keys %{ $tunnel_info->{unix_sockets} }]; push @$unix_sockets, $tunnel_info->{addr}; for my $sock (@$unix_sockets) { push @$ssh_forward_info, "$sock:$sock"; unlink $sock; } $self->{tunnel} = $self->fork_tunnel($ssh_forward_info); my $unix_socket_try = 0; # wait for the socket to become ready while ($unix_socket_try <= 100) { $unix_socket_try++; my $available = 0; foreach my $sock (@$unix_sockets) { if (-S $sock) { $available++; } } if ($available == @$unix_sockets) { last; } usleep(50000); } if ($unix_socket_try > 100) { $self->{errors} = 1; PVE::Tunnel::finish_tunnel($self->{tunnel}); die "Timeout, migration socket $tunnel_info->{addr} did not get ready"; } $self->{tunnel}->{unix_sockets} = $unix_sockets if (@$unix_sockets); } elsif ($tunnel_info->{proto} eq 'tcp') { my $ssh_forward_info = []; if ($tunnel_info->{addr} eq "localhost") { # for backwards compatibility with older qemu-server versions my $pfamily = PVE::Tools::get_host_address_family($nodename); my $lport = PVE::Tools::next_migrate_port($pfamily); push @$ssh_forward_info, "$lport:localhost:$tunnel_info->{port}"; } $self->{tunnel} = $self->fork_tunnel($ssh_forward_info); } else { die "unsupported protocol in migration URI: $tunnel_info->{proto}\n"; } } else { #fork tunnel for insecure migration, to send faster commands like resume $self->{tunnel} = $self->fork_tunnel(); } } sub lock_vm { my ($self, $vmid, $code, @param) = @_; return PVE::QemuConfig->lock_config($vmid, $code, @param); } sub target_storage_check_available { my ($self, $storecfg, $targetsid, $volid) = @_; if (!$self->{opts}->{remote}) { # check if storage is available on target node my $target_scfg = PVE::Storage::storage_check_enabled( $storecfg, $targetsid, $self->{node}, ); my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid); die "$volid: content type '$vtype' is not available on storage '$targetsid'\n" if !$target_scfg->{content}->{$vtype}; } } sub prepare { my ($self, $vmid) = @_; my $online = $self->{opts}->{online}; my $storecfg = $self->{storecfg} = PVE::Storage::config(); # updates the configuration, so ordered before saving the configuration in $self eval { PVE::QemuConfig::cleanup_fleecing_images( $vmid, $storecfg, sub { $self->log($_[0], $_[1]); }, ); }; $self->log('warn', "attempt to clean up left-over fleecing images failed - $@") if $@; # test if VM exists my $conf = $self->{vmconf} = PVE::QemuConfig->load_config($vmid); my $repl_conf = PVE::ReplicationConfig->new(); $self->{replication_jobcfg} = $repl_conf->find_local_replication_job($vmid, $self->{node}); $self->{is_replicated} = $repl_conf->check_for_existing_jobs($vmid, 1); if ($self->{replication_jobcfg} && defined($self->{replication_jobcfg}->{remove_job})) { die "refusing to migrate replicated VM whose replication job is marked for removal\n"; } PVE::QemuConfig->check_lock($conf); my $running = 0; if (my $pid = PVE::QemuServer::check_running($vmid)) { die "can't migrate running VM without --online\n" if !$online; $running = $pid; if ($self->{is_replicated} && !$self->{replication_jobcfg}) { if ($self->{opts}->{force}) { $self->log( 'warn', "WARNING: Node '$self->{node}' is not a replication target. Existing " . "replication jobs will fail after migration!\n", ); } else { die "Cannot live-migrate replicated VM to node '$self->{node}' - not a replication " . "target. Use 'force' to override.\n"; } } $self->{forcemachine} = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf); # To support abstracted CPU configurations, keep QEMU's "-cpu" parameter intact. if ($conf->{cpu} && PVE::QemuServer::CPUConfig::is_abstracted($conf->{cpu})) { $self->{forcecpu} = PVE::QemuServer::CPUConfig::get_cpu_from_running_vm($pid); } # Do not treat a suspended VM as paused, as it might wake up # during migration and remain paused after migration finishes. $self->{vm_was_paused} = 1 if PVE::QemuServer::vm_is_paused($vmid, 0); if ($self->{opts}->{'with-conntrack-state'}) { if ($self->{opts}->{remote}) { # shouldn't be reached in normal circumstances anyway, as we prevent it on # an API level $self->log( 'warn', 'conntrack state migration not supported for remote migrations, ' . 'active connections might get dropped', ); $self->{opts}->{'with-conntrack-state'} = 0; } else { PVE::QemuServer::DBusVMState::qemu_add_dbus_vmstate($vmid); } } else { $self->log( 'warn', 'conntrack state migration not supported or disabled, ' . 'active connections might get dropped', ); # In case some leftover instance is running, stop it. The target QEMU instance won't # have the 'dbus-vmstate' object, so the source must not have it either. if (defined(PVE::QemuServer::DBusVMState::qemu_del_dbus_vmstate($vmid, quiet => 1))) { $self->log('warn', "stopped left-over dbus-vmstate helper for VM $vmid"); } } } my ($loc_res, $mapped_res, $missing_mappings_by_node) = PVE::QemuMigrate::Helpers::check_local_resources($conf, $running, 1); my $blocking_resources = []; for my $res ($loc_res->@*) { if (!defined($mapped_res->{$res})) { push $blocking_resources->@*, $res; } } if (scalar($blocking_resources->@*)) { if ($self->{running} || !$self->{opts}->{force}) { die "can't migrate VM which uses local devices: " . join(", ", $blocking_resources->@*) . "\n"; } else { $self->log('info', "migrating VM which uses local devices"); } } if (scalar(keys $mapped_res->%*)) { my $missing_mappings = $missing_mappings_by_node->{ $self->{node} }; my $missing_live_mappings = []; for my $key (sort keys $mapped_res->%*) { my $res = $mapped_res->{$key}; my $name = "$key:$res->{name}"; push $missing_live_mappings->@*, $name if !$res->{'live-migration'}; } if (scalar($missing_mappings->@*)) { my $missing = join(", ", $missing_mappings->@*); die "can't migrate to '$self->{node}': missing mapped devices $missing\n"; } elsif ($running && scalar($missing_live_mappings->@*)) { my $missing = join(", ", $missing_live_mappings->@*); die "can't live migrate running VM which uses following mapped devices: $missing\n"; } else { $self->log('info', "migrating VM which uses mapped local devices"); } } my $vga = PVE::QemuServer::parse_vga($conf->{vga}); if ($running && $vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') { my $machine_version = PVE::QemuServer::Machine::get_current_qemu_machine($vmid); if (!PVE::QemuServer::Machine::is_machine_version_at_least($machine_version, 10, 1)) { die "VMs with 'clipboard' set to 'vnc' are not live migratable with" . " QEMU/machine versions older than 10.1!\n"; } } my $vollist = PVE::QemuServer::get_vm_volumes($conf); my $storages = {}; foreach my $volid (@$vollist) { my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1); # check if storage is available on source node my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid); my $targetsid = $sid; # NOTE: local ignores shared mappings, remote maps them if (!$scfg->{shared} || $self->{opts}->{remote}) { $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $sid); } $storages->{$targetsid} = 1; $self->target_storage_check_available($storecfg, $targetsid, $volid); if ($scfg->{shared}) { # PVE::Storage::activate_storage checks this for non-shared storages my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); warn "Used shared storage '$sid' is not online on source node!\n" if !$plugin->check_connection($sid, $scfg); } } if ($self->{opts}->{remote}) { # test & establish websocket connection my $bridges = map_bridges($conf, $self->{opts}->{bridgemap}, 1); my $tunnel = $self->fork_websocket_tunnel($storages, $bridges); my $min_version = $tunnel->{version} - $tunnel->{age}; $self->log('info', "local WS tunnel version: $WS_TUNNEL_VERSION"); $self->log('info', "remote WS tunnel version: $tunnel->{version}"); $self->log('info', "minimum required WS tunnel version: $min_version"); die "Remote tunnel endpoint not compatible, upgrade required\n" if $WS_TUNNEL_VERSION < $min_version; die "Remote tunnel endpoint too old, upgrade required\n" if $WS_TUNNEL_VERSION > $tunnel->{version}; print "websocket tunnel started\n"; $self->{tunnel} = $tunnel; } else { # test ssh connection my $cmd = [@{ $self->{rem_ssh} }, '/bin/true']; eval { $self->cmd_quiet($cmd); }; die "Can't connect to destination address using public key\n" if $@; } return $running; } sub scan_local_volumes { my ($self, $vmid) = @_; my $conf = $self->{vmconf}; # local volumes which have been copied # and their old_id => new_id pairs $self->{volume_map} = {}; $self->{local_volumes} = {}; my $storecfg = $self->{storecfg}; eval { # found local volumes and their origin my $local_volumes = $self->{local_volumes}; my $local_volumes_errors = {}; my $other_errors = []; my $abort = 0; my $path_to_volid = {}; my $log_error = sub { my ($msg, $volid) = @_; if (defined($volid)) { $local_volumes_errors->{$volid} = $msg; } else { push @$other_errors, $msg; } $abort = 1; }; my $replicatable_volumes = !$self->{replication_jobcfg} ? {} : PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 1); foreach my $volid (keys %{$replicatable_volumes}) { $local_volumes->{$volid}->{replicated} = 1; } my $test_volid = sub { my ($volid, $attr) = @_; if ($volid =~ m|^/|) { return if $attr->{shared}; $local_volumes->{$volid}->{ref} = 'config'; die "local file/device\n"; } my $snaprefs = $attr->{referenced_in_snapshot}; if ($attr->{cdrom}) { if ($volid eq 'cdrom') { my $msg = "can't migrate local cdrom drive"; if (defined($snaprefs) && !$attr->{is_attached}) { my $snapnames = join(', ', sort keys %$snaprefs); $msg .= " (referenced in snapshot - $snapnames)"; } &$log_error("$msg\n"); return; } return if $volid eq 'none'; } my ($sid, $volname) = PVE::Storage::parse_volume_id($volid); # check if storage is available on both nodes my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid); my $targetsid = $sid; # NOTE: local ignores shared mappings, remote maps them if (!$scfg->{shared} || $self->{opts}->{remote}) { $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $sid); } $self->target_storage_check_available($storecfg, $targetsid, $volid); return if $scfg->{shared} && !$self->{opts}->{remote}; $local_volumes->{$volid}->{ref} = 'pending' if $attr->{referenced_in_pending}; $local_volumes->{$volid}->{ref} = 'snapshot' if $attr->{referenced_in_snapshot}; $local_volumes->{$volid}->{ref} = 'unused' if $attr->{is_unused}; $local_volumes->{$volid}->{ref} = 'attached' if $attr->{is_attached}; $local_volumes->{$volid}->{ref} = 'generated' if $attr->{is_tpmstate}; $local_volumes->{$volid}->{bwlimit} = $self->get_bwlimit($sid, $targetsid); $local_volumes->{$volid}->{targetsid} = $targetsid; $local_volumes->{$volid}->@{qw(size format)} = PVE::Storage::volume_size_info($storecfg, $volid); $local_volumes->{$volid}->{is_vmstate} = $attr->{is_vmstate} ? 1 : 0; $local_volumes->{$volid}->{is_cloudinit} = $attr->{is_cloudinit} ? 1 : 0; $local_volumes->{$volid}->{drivename} = $attr->{drivename} if $attr->{drivename}; # If with_snapshots is not set for storage migrate, it tries to use # a raw+size stream, but on-the-fly conversion from qcow2 to raw+size # back to qcow2 is currently not possible. $local_volumes->{$volid}->{snapshots} = ($local_volumes->{$volid}->{format} =~ /^(?:qcow2|vmdk)$/); if ($attr->{cdrom}) { if ($volid =~ /vm-\d+-cloudinit/) { $local_volumes->{$volid}->{ref} = 'generated'; return; } die "local cdrom image\n"; } my ($path, $owner) = PVE::Storage::path($storecfg, $volid); die "owned by other VM (owner = VM $owner)\n" if !$owner || ($owner != $vmid); $path_to_volid->{$path}->{$volid} = 1; return if $attr->{is_vmstate}; if (defined($snaprefs)) { $local_volumes->{$volid}->{snapshots} = 1; # we cannot migrate snapshots on local storage # exceptions: 'zfspool' or 'qcow2' files (on directory storage) # # Note that only qcow2 with in-qcow2 snapshots work - for # backing-chain snapshots we'd need to copy the entire snapshot # list (or support replication) die "online storage migration not possible if non-replicated snapshot exists\n" if $self->{running} && !$local_volumes->{$volid}->{replicated}; die "remote migration with snapshots not supported yet\n" if $self->{opts}->{remote}; if (!( $scfg->{type} eq 'zfspool' || ($scfg->{type} eq 'btrfs' && $local_volumes->{$volid}->{format} eq 'raw') || ($local_volumes->{$volid}->{format} eq 'qcow2' && !$scfg->{'snapshot-as-volume-chain'}) )) { die "non-migratable snapshot exists\n"; } } die "referenced by linked clone(s)\n" if PVE::Storage::volume_is_base_and_used($storecfg, $volid); }; PVE::QemuServer::foreach_volid( $conf, sub { my ($volid, $attr) = @_; eval { $test_volid->($volid, $attr); }; if (my $err = $@) { &$log_error($err, $volid); } }, ); for my $path (keys %$path_to_volid) { my @volids = keys $path_to_volid->{$path}->%*; die "detected not supported aliased volumes: '" . join("', '", @volids) . "'\n" if (scalar(@volids) > 1); } foreach my $vol (sort keys %$local_volumes) { my $type = $replicatable_volumes->{$vol} ? 'local, replicated' : 'local'; my $ref = $local_volumes->{$vol}->{ref}; if ($ref eq 'attached') { &$log_error( "can't live migrate attached local disks without with-local-disks option\n", $vol, ) if $self->{running} && !$self->{opts}->{"with-local-disks"}; $self->log('info', "found $type disk '$vol' (attached)\n"); } elsif ($ref eq 'unused') { $self->log('info', "found $type disk '$vol' (unused)\n"); } elsif ($ref eq 'snapshot') { $self->log('info', "found $type disk '$vol' (referenced by snapshot(s))\n"); } elsif ($ref eq 'pending') { $self->log('info', "found $type disk '$vol' (pending change)\n"); } elsif ($ref eq 'generated') { $self->log('info', "found generated disk '$vol' (in current VM config)\n"); } else { $self->log('info', "found $type disk '$vol'\n"); } } foreach my $vol (sort keys %$local_volumes_errors) { $self->log('warn', "can't migrate local disk '$vol': $local_volumes_errors->{$vol}"); } foreach my $err (@$other_errors) { $self->log('warn', "$err"); } if ($abort) { die "can't migrate VM - check log\n"; } # additional checks for local storage foreach my $volid (keys %$local_volumes) { my ($sid, $volname) = PVE::Storage::parse_volume_id($volid); my $scfg = PVE::Storage::storage_config($storecfg, $sid); my $migratable = $scfg->{type} =~ /^(?:dir|btrfs|zfspool|lvmthin|lvm)$/; # TODO: what is this even here for? $migratable = 1 if $self->{opts}->{remote}; die "can't migrate '$volid' - storage type '$scfg->{type}' not supported\n" if !$migratable; # image is a linked clone on local storage, se we can't migrate. if (my $basename = (PVE::Storage::parse_volname($storecfg, $volid))[3]) { die "can't migrate '$volid' as it's a clone of '$basename'"; } } foreach my $volid (sort keys %$local_volumes) { my $ref = $local_volumes->{$volid}->{ref}; if ($self->{running} && $ref eq 'attached') { $local_volumes->{$volid}->{migration_mode} = 'online'; } elsif ($self->{running} && $ref eq 'generated') { # offline migrate the cloud-init ISO and don't regenerate on VM start # # tpmstate will also be offline migrated first, and in case of # live migration then updated by QEMU/swtpm if necessary $local_volumes->{$volid}->{migration_mode} = 'offline'; } else { $local_volumes->{$volid}->{migration_mode} = 'offline'; } } }; die "Problem found while scanning volumes - $@" if $@; } sub handle_replication { my ($self, $vmid) = @_; my $conf = $self->{vmconf}; my $local_volumes = $self->{local_volumes}; return if !$self->{replication_jobcfg}; die "can't migrate VM with replicated volumes to remote cluster/node\n" if $self->{opts}->{remote}; if ($self->{running}) { my @live_replicatable_volumes = $self->filter_local_volumes('online', 1); foreach my $volid (@live_replicatable_volumes) { my $drive = $local_volumes->{$volid}->{drivename}; die "internal error - no drive for '$volid'\n" if !defined($drive); my $bitmap = "repl_$drive"; # start tracking before replication to get full delta + a few duplicates $self->log('info', "$drive: start tracking writes using block-dirty-bitmap '$bitmap'"); mon_cmd($vmid, 'block-dirty-bitmap-add', node => "drive-$drive", name => $bitmap); # other info comes from target node in phase 2 $self->{target_drive}->{$drive}->{bitmap} = $bitmap; } } $self->log('info', "replicating disk images"); my $start_time = time(); my $logfunc = sub { $self->log('info', shift) }; my $actual_replicated_volumes = PVE::Replication::run_replication( 'PVE::QemuConfig', $self->{replication_jobcfg}, $start_time, $start_time, $logfunc, ); # extra safety check my @replicated_volumes = $self->filter_local_volumes(undef, 1); foreach my $volid (@replicated_volumes) { die "expected volume '$volid' to get replicated, but it wasn't\n" if !$actual_replicated_volumes->{$volid}; } } sub config_update_local_disksizes { my ($self) = @_; my $conf = $self->{vmconf}; my $local_volumes = $self->{local_volumes}; PVE::QemuConfig->foreach_volume( $conf, sub { my ($key, $drive) = @_; # skip special disks, will be handled later return if $key eq 'efidisk0'; return if $key eq 'tpmstate0'; my $volid = $drive->{file}; return if !defined($local_volumes->{$volid}); # only update sizes for local volumes my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $local_volumes->{$volid}->{size}); if (defined($updated)) { $conf->{$key} = PVE::QemuServer::print_drive($updated); $self->log('info', "drive '$key': $msg"); } }, ); # we want to set the efidisk size in the config to the size of the # real OVMF_VARS.fd image, else we can create a too big image, which does not work if (defined($conf->{efidisk0})) { PVE::QemuServer::update_efidisk_size($conf); } # TPM state might have an irregular filesize, to avoid problems on transfer # we always assume the static size of 4M to allocate on the target if (defined($conf->{tpmstate0})) { PVE::QemuServer::update_tpmstate_size($conf); } } sub filter_local_volumes { my ($self, $migration_mode, $replicated) = @_; my $volumes = $self->{local_volumes}; my @filtered_volids; foreach my $volid (sort keys %{$volumes}) { next if defined($migration_mode) && safe_string_ne($volumes->{$volid}->{migration_mode}, $migration_mode); next if defined($replicated) && safe_boolean_ne($volumes->{$volid}->{replicated}, $replicated); push @filtered_volids, $volid; } return @filtered_volids; } sub sync_offline_local_volumes { my ($self) = @_; my $local_volumes = $self->{local_volumes}; my @volids = $self->filter_local_volumes('offline', 0); my $storecfg = $self->{storecfg}; my $opts = $self->{opts}; $self->log('info', "copying local disk images") if scalar(@volids); foreach my $volid (@volids) { my $new_volid; my $opts = $self->{opts}; if ($opts->{remote}) { my $log = sub { my ($level, $msg) = @_; $self->log($level, $msg); }; $new_volid = PVE::StorageTunnel::storage_migrate( $self->{tunnel}, $storecfg, $volid, $self->{vmid}, $opts->{remote}->{vmid}, $local_volumes->{$volid}, $log, ); } else { my $targetsid = $local_volumes->{$volid}->{targetsid}; my $bwlimit = $local_volumes->{$volid}->{bwlimit}; $bwlimit = $bwlimit * 1024 if defined($bwlimit); # storage_migrate uses bps my $preserve_name = $local_volumes->{$volid}->{is_vmstate} || $local_volumes->{$volid}->{is_cloudinit}; my $storage_migrate_opts = { 'ratelimit_bps' => $bwlimit, 'insecure' => $opts->{migration_type} eq 'insecure', 'with_snapshots' => $local_volumes->{$volid}->{snapshots}, 'allow_rename' => !$preserve_name, }; my $logfunc = sub { $self->log('info', $_[0]); }; $new_volid = eval { PVE::Storage::storage_migrate( $storecfg, $volid, $self->{ssh_info}, $targetsid, $storage_migrate_opts, $logfunc, ); }; if (my $err = $@) { die "storage migration for '$volid' to storage '$targetsid' failed - $err\n"; } } $self->{volume_map}->{$volid} = $new_volid; $self->log('info', "volume '$volid' is '$new_volid' on the target\n"); eval { PVE::Storage::deactivate_volumes($storecfg, [$volid]); }; if (my $err = $@) { $self->log('warn', $err); } } } sub cleanup_remotedisks { my ($self) = @_; if ($self->{opts}->{remote}) { PVE::Tunnel::finish_tunnel($self->{tunnel}, 1); delete $self->{tunnel}; return; } my $local_volumes = $self->{local_volumes}; foreach my $volid (values %{ $self->{volume_map} }) { # don't clean up replicated disks! next if $local_volumes->{$volid}->{replicated}; my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); my $cmd = [@{ $self->{rem_ssh} }, 'pvesm', 'free', "$storeid:$volname"]; eval { PVE::Tools::run_command($cmd, outfunc => sub { }, errfunc => sub { }); }; if (my $err = $@) { $self->log('err', $err); $self->{errors} = 1; } } } sub cleanup_bitmaps { my ($self) = @_; foreach my $drive (keys %{ $self->{target_drive} }) { my $bitmap = $self->{target_drive}->{$drive}->{bitmap}; next if !$bitmap; $self->log('info', "$drive: removing block-dirty-bitmap '$bitmap'"); mon_cmd( $self->{vmid}, 'block-dirty-bitmap-remove', node => "drive-$drive", name => $bitmap, ); } } sub phase1 { my ($self, $vmid) = @_; $self->log('info', "starting migration of VM $vmid to node '$self->{node}' ($self->{nodeip})"); my $conf = $self->{vmconf}; # set migrate lock in config file $conf->{lock} = 'migrate'; PVE::QemuConfig->write_config($vmid, $conf); $self->scan_local_volumes($vmid); # fix disk sizes to match their actual size and write changes, # so that the target allocates the correct volumes $self->config_update_local_disksizes(); PVE::QemuConfig->write_config($vmid, $conf); $self->handle_replication($vmid); $self->sync_offline_local_volumes(); $self->phase1_remote($vmid) if $self->{opts}->{remote}; } sub map_bridges { my ($conf, $map, $scan_only) = @_; my $bridges = {}; foreach my $opt (keys %$conf) { next if $opt !~ m/^net\d+$/; next if !$conf->{$opt}; my $d = PVE::QemuServer::Network::parse_net($conf->{$opt}); next if !$d || !$d->{bridge}; my $target_bridge = PVE::JSONSchema::map_id($map, $d->{bridge}); $bridges->{$target_bridge}->{$opt} = $d->{bridge}; next if $scan_only; $d->{bridge} = $target_bridge; $conf->{$opt} = PVE::QemuServer::Network::print_net($d); } return $bridges; } sub phase1_remote { my ($self, $vmid) = @_; my $remote_conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->update_volume_ids($remote_conf, $self->{volume_map}); my $bridges = map_bridges($remote_conf, $self->{opts}->{bridgemap}); for my $target (keys $bridges->%*) { for my $nic (keys $bridges->{$target}->%*) { $self->log('info', "mapped: $nic from $bridges->{$target}->{$nic} to $target"); } } my @online_local_volumes = $self->filter_local_volumes('online'); my $storage_map = $self->{opts}->{storagemap}; $self->{nbd} = {}; PVE::QemuConfig->foreach_volume( $remote_conf, sub { my ($ds, $drive) = @_; # TODO eject CDROM? return if PVE::QemuServer::drive_is_cdrom($drive); my $volid = $drive->{file}; return if !$volid; return if !grep { $_ eq $volid } @online_local_volumes; my ($storeid) = PVE::Storage::parse_volume_id($volid); my $source_format = checked_volume_format($self->{storecfg}, $volid); # set by target cluster my $oldvolid = delete $drive->{file}; delete $drive->{format}; my $targetsid = PVE::JSONSchema::map_id($storage_map, $storeid); my $params = { format => $source_format, storage => $targetsid, drive => $drive, }; $self->log( 'info', "Allocating volume for drive '$ds' on remote storage '$targetsid'..", ); my $res = PVE::Tunnel::write_tunnel($self->{tunnel}, 600, 'disk', $params); $self->log('info', "volume '$oldvolid' is '$res->{volid}' on the target\n"); $remote_conf->{$ds} = $res->{drivestr}; $self->{nbd}->{$ds} = $res; }, ); my $conf_str = PVE::QemuServer::write_vm_config("remote", $remote_conf); # TODO expose in PVE::Firewall? my $vm_fw_conf_path = "/etc/pve/firewall/$vmid.fw"; my $fw_conf_str; $fw_conf_str = PVE::Tools::file_get_contents($vm_fw_conf_path) if -e $vm_fw_conf_path; my $params = { conf => $conf_str, 'firewall-config' => $fw_conf_str, }; PVE::Tunnel::write_tunnel($self->{tunnel}, 120, 'config', $params); } sub phase1_cleanup { my ($self, $vmid, $err) = @_; $self->log('info', "aborting phase 1 - cleanup resources"); my $conf = $self->{vmconf}; delete $conf->{lock}; eval { PVE::QemuConfig->write_config($vmid, $conf) }; if (my $err = $@) { $self->log('err', $err); } eval { $self->cleanup_remotedisks() }; if (my $err = $@) { $self->log('err', $err); } eval { $self->cleanup_bitmaps() }; if (my $err = $@) { $self->log('err', $err); } if ($self->{running} && $self->{opts}->{'with-conntrack-state'}) { # if the VM is running, that means we also tried to migrate additional # state via our dbus-vmstate helper # only need to locally stop it, on the target the VM cleanup will # handle it PVE::QemuServer::DBusVMState::qemu_del_dbus_vmstate($vmid); } } sub phase2_start_local_cluster { my ($self, $vmid, $params) = @_; my $conf = $self->{vmconf}; my $local_volumes = $self->{local_volumes}; my @online_local_volumes = $self->filter_local_volumes('online'); my $start = $params->{start_params}; my $migrate = $params->{migrate_opts}; $self->log('info', "starting VM $vmid on remote node '$self->{node}'"); my $tunnel_info = {}; ## start on remote node my $cmd = [@{ $self->{rem_ssh} }]; push @$cmd, 'qm', 'start', $vmid; if ($start->{skiplock}) { push @$cmd, '--skiplock'; } push @$cmd, '--migratedfrom', $migrate->{migratedfrom}; push @$cmd, '--migration_type', $migrate->{type}; push @$cmd, '--migration_network', $migrate->{network} if $migrate->{network}; push @$cmd, '--stateuri', $start->{statefile}; if ($start->{forcemachine}) { push @$cmd, '--machine', $start->{forcemachine}; } if ($start->{forcecpu}) { push @$cmd, '--force-cpu', $start->{forcecpu}; } if ($start->{'nets-host-mtu'}) { push @$cmd, '--nets-host-mtu', $start->{'nets-host-mtu'}; } if ($self->{storage_migration}) { push @$cmd, '--targetstorage', ($self->{opts}->{targetstorage} // '1'); } if ($self->{opts}->{'with-conntrack-state'}) { push @$cmd, '--with-conntrack-state'; } my $spice_port; my $input = "nbd_protocol_version: $migrate->{nbd_proto_version}\n"; my @offline_local_volumes = $self->filter_local_volumes('offline'); for my $volid (@offline_local_volumes) { my $drivename = $local_volumes->{$volid}->{drivename}; next if !$drivename || !$conf->{$drivename}; my $new_volid = $self->{volume_map}->{$volid}; next if !$new_volid || $volid eq $new_volid; $input .= "offline_volume: $drivename: $new_volid\n"; } $input .= "spice_ticket: $migrate->{spice_ticket}\n" if $migrate->{spice_ticket}; my @online_replicated_volumes = $self->filter_local_volumes('online', 1); foreach my $volid (@online_replicated_volumes) { $input .= "replicated_volume: $volid\n"; } my $handle_storage_migration_listens = sub { my ($drive_key, $drivestr, $nbd_uri) = @_; $self->{stopnbd} = 1; $self->{target_drive}->{$drive_key}->{drivestr} = $drivestr; $self->{target_drive}->{$drive_key}->{nbd_uri} = $nbd_uri; my $source_drive = PVE::QemuServer::parse_drive($drive_key, $conf->{$drive_key}); my $target_drive = PVE::QemuServer::parse_drive($drive_key, $drivestr); my $source_volid = $source_drive->{file}; my $target_volid = $target_drive->{file}; $self->{volume_map}->{$source_volid} = $target_volid; $self->log('info', "volume '$source_volid' is '$target_volid' on the target\n"); }; my $target_replicated_volumes = {}; my $target_nets_host_mtu_not_supported; # Note: We try to keep $spice_ticket secret (do not pass via command line parameter) # instead we pipe it through STDIN my $exitcode = PVE::Tools::run_command( $cmd, input => $input, outfunc => sub { my $line = shift; if ($line =~ m/^migration listens on (tcp):(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/ ) { $tunnel_info->{addr} = $2; $tunnel_info->{port} = int($3); $tunnel_info->{proto} = $1; } elsif ($line =~ m!^migration listens on (unix):(/run/qemu-server/(\d+)\.migrate)$! ) { $tunnel_info->{addr} = $2; die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $3; $tunnel_info->{proto} = $1; } elsif ($line =~ m/^migration listens on port (\d+)$/) { $tunnel_info->{addr} = "localhost"; $tunnel_info->{port} = int($1); $tunnel_info->{proto} = "tcp"; } elsif ($line =~ m/^spice listens on port (\d+)$/) { $spice_port = int($1); } elsif ($line =~ m/^storage migration listens on nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+) volume:(\S+)$/ ) { my $drivestr = $4; my $nbd_uri = "nbd:$1:$2:exportname=$3"; my $targetdrive = $3; $targetdrive =~ s/drive-//g; $handle_storage_migration_listens->($targetdrive, $drivestr, $nbd_uri); } elsif ($line =~ m!^storage migration listens on nbd:unix:(/run/qemu-server/(\d+)_nbd\.migrate):exportname=(\S+) volume:(\S+)$! ) { my $drivestr = $4; die "Destination UNIX socket's VMID does not match source VMID" if $vmid ne $2; my $nbd_unix_addr = $1; my $nbd_uri = "nbd:unix:$nbd_unix_addr:exportname=$3"; my $targetdrive = $3; $targetdrive =~ s/drive-//g; $handle_storage_migration_listens->($targetdrive, $drivestr, $nbd_uri); $tunnel_info->{unix_sockets}->{$nbd_unix_addr} = 1; } elsif ($line =~ m/^re-using replicated volume: (\S+) - (.*)$/) { my $drive = $1; my $volid = $2; $target_replicated_volumes->{$volid} = $drive; } elsif ($line =~ m/^QEMU: (.*)$/) { $self->log('info', "[$self->{node}] $1\n"); } }, errfunc => sub { my $line = shift; $target_nets_host_mtu_not_supported = 1 if $line =~ m/^Unknown option: nets-host-mtu/; $self->log('info', "[$self->{node}] $line"); }, noerr => 1, ); die "target node $self->{node} is too old for preserving VirtIO-net MTU, please upgrade\n" if $target_nets_host_mtu_not_supported; die "remote command failed with exit code $exitcode\n" if $exitcode; die "unable to detect remote migration address\n" if !$tunnel_info->{addr} || !$tunnel_info->{proto}; if (scalar(keys %$target_replicated_volumes) != scalar(@online_replicated_volumes)) { die "number of replicated disks on source and target node do not match - target node too old?\n"; } return ($tunnel_info, $spice_port); } sub phase2_start_remote_cluster { my ($self, $vmid, $params) = @_; die "insecure migration to remote cluster not implemented\n" if $params->{migrate_opts}->{type} ne 'websocket'; my $remote_vmid = $self->{opts}->{remote}->{vmid}; # like regular start but with some overhead accounted for my $memory = get_current_memory($self->{vmconf}->{memory}); my $timeout = PVE::QemuServer::Helpers::config_aware_timeout($self->{vmconf}, $memory) + 10; my $res = PVE::Tunnel::write_tunnel($self->{tunnel}, $timeout, "start", $params); foreach my $drive (keys %{ $res->{drives} }) { $self->{stopnbd} = 1; $self->{target_drive}->{$drive}->{drivestr} = $res->{drives}->{$drive}->{drivestr}; my $nbd_uri = $res->{drives}->{$drive}->{nbd_uri}; die "unexpected NBD uri for '$drive': $nbd_uri\n" if $nbd_uri !~ s!/run/qemu-server/$remote_vmid\_!/run/qemu-server/$vmid\_!; $self->{target_drive}->{$drive}->{nbd_uri} = $nbd_uri; } return ($res->{migrate}, $res->{spice_port}); } my $migrate_downtime_max = 2000 * 1000; # as defined in QEMU's migration/options.c my sub cap_migrate_downtime { my ($self, $migrate_downtime) = @_; if ($migrate_downtime > $migrate_downtime_max) { $self->log('info', "capping downtime limit to maximum possible: $migrate_downtime_max ms"); return $migrate_downtime_max; } return $migrate_downtime; } my sub increase_migrate_downtime { my ($self, $vmid, $migrate_downtime) = @_; return $migrate_downtime_max if $migrate_downtime >= $migrate_downtime_max; $migrate_downtime *= 2; $migrate_downtime = cap_migrate_downtime($self, $migrate_downtime); $self->log( 'info', "auto-increased downtime to continue migration: $migrate_downtime ms", ); eval { # migrate-set-parameters does not touch values not # specified, so this only changes downtime-limit mon_cmd( $vmid, "migrate-set-parameters", 'downtime-limit' => int($migrate_downtime), ); }; $self->log('info', "migrate-set-parameters error: $@") if $@; return $migrate_downtime; } sub phase2 { my ($self, $vmid) = @_; my $conf = $self->{vmconf}; my $local_volumes = $self->{local_volumes}; # version > 0 for unix socket support my $nbd_protocol_version = 1; my $spice_ticket; if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) { my $res = mon_cmd($vmid, 'query-spice'); $spice_ticket = $res->{ticket}; } my $migration_type = $self->{opts}->{migration_type}; my $state_uri = $migration_type eq 'insecure' ? 'tcp' : 'unix'; my $params = { start_params => { statefile => $state_uri, forcemachine => $self->{forcemachine}, forcecpu => $self->{forcecpu}, skiplock => 1, }, migrate_opts => { spice_ticket => $spice_ticket, type => $migration_type, network => $self->{opts}->{migration_network}, storagemap => $self->{opts}->{storagemap}, migratedfrom => PVE::INotify::nodename(), nbd_proto_version => $nbd_protocol_version, nbd => $self->{nbd}, }, }; if (my $nets_host_mtu = PVE::QemuServer::Network::get_nets_host_mtu($vmid, $conf)) { $params->{start_params}->{'nets-host-mtu'} = $nets_host_mtu; } my ($tunnel_info, $spice_port); my @online_local_volumes = $self->filter_local_volumes('online'); $self->{storage_migration} = 1 if scalar(@online_local_volumes); if (my $remote = $self->{opts}->{remote}) { my $remote_vmid = $remote->{vmid}; $params->{migrate_opts}->{remote_node} = $self->{node}; ($tunnel_info, $spice_port) = $self->phase2_start_remote_cluster($vmid, $params); die "only UNIX sockets are supported for remote migration\n" if $tunnel_info->{proto} ne 'unix'; # untaint my ($remote_socket) = $tunnel_info->{addr} =~ m|^(/run/qemu-server/\d+\.migrate)$| or die "unexpected socket address '$tunnel_info->{addr}'\n"; my $local_socket = $remote_socket; $local_socket =~ s/$remote_vmid/$vmid/g; $tunnel_info->{addr} = $local_socket; $self->log('info', "Setting up tunnel for '$local_socket'"); PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket); foreach my $remote_socket (@{ $tunnel_info->{unix_sockets} }) { # untaint ($remote_socket) = $remote_socket =~ m|^(/run/qemu-server/(?:(?!\.\./).)+\.migrate)$| or die "unexpected socket address '$remote_socket'\n"; my $local_socket = $remote_socket; $local_socket =~ s/$remote_vmid/$vmid/g; next if $self->{tunnel}->{forwarded}->{$local_socket}; $self->log('info', "Setting up tunnel for '$local_socket'"); PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket); } } else { ($tunnel_info, $spice_port) = $self->phase2_start_local_cluster($vmid, $params); $self->log('info', "start remote tunnel"); $self->start_remote_tunnel($tunnel_info); } my $migrate_uri = "$tunnel_info->{proto}:$tunnel_info->{addr}"; $migrate_uri .= ":$tunnel_info->{port}" if defined($tunnel_info->{port}); if ($self->{storage_migration}) { $self->{storage_migration_jobs} = {}; $self->log('info', "starting storage migration"); die "The number of local disks does not match between the source and the destination.\n" if (scalar(keys %{ $self->{target_drive} }) != scalar(@online_local_volumes)); foreach my $drive (keys %{ $self->{target_drive} }) { my $target = $self->{target_drive}->{$drive}; my $nbd_uri = $target->{nbd_uri}; my $source_drive = PVE::QemuServer::parse_drive($drive, $conf->{$drive}); my $source_volid = $source_drive->{file}; my $bwlimit = $self->{local_volumes}->{$source_volid}->{bwlimit}; my $bitmap = $target->{bitmap}; $self->log('info', "$drive: start migration to $nbd_uri"); my $source_info = { vmid => $vmid, drive => $source_drive }; $source_info->{bitmap} = $bitmap if defined($bitmap); my $dest_info = { volid => $nbd_uri }; my $mirror_opts = {}; $mirror_opts->{bwlimit} = $bwlimit if defined($bwlimit); PVE::QemuServer::BlockJob::mirror( $source_info, $dest_info, $self->{storage_migration_jobs}, 'skip', $mirror_opts, ); } if (PVE::QemuServer::QMPHelpers::runs_at_least_qemu_version($vmid, 8, 2)) { $self->log('info', "switching mirror jobs to actively synced mode"); PVE::QemuServer::BlockJob::qemu_drive_mirror_switch_to_active_mode( $vmid, $self->{storage_migration_jobs}, ); } } $self->log('info', "starting online/live migration on $migrate_uri"); $self->{livemigration} = 1; # load_defaults my $defaults = PVE::QemuServer::load_defaults(); $self->log('info', "set migration capabilities"); eval { PVE::QemuMigrate::Helpers::set_migration_caps($vmid) }; warn $@ if $@; my $qemu_migrate_params = {}; # migrate speed can be set via bwlimit (datacenter.cfg and API) and via the # migrate_speed parameter in qm.conf - take the lower of the two. my $bwlimit = $self->get_bwlimit(); my $migrate_speed = $conf->{migrate_speed} // 0; $migrate_speed *= 1024; # migrate_speed is in MB/s, bwlimit in KB/s if ($bwlimit && $migrate_speed) { $migrate_speed = ($bwlimit < $migrate_speed) ? $bwlimit : $migrate_speed; } else { $migrate_speed ||= $bwlimit; } $migrate_speed ||= ($defaults->{migrate_speed} || 0) * 1024; if ($migrate_speed) { $migrate_speed *= 1024; # qmp takes migrate_speed in B/s. $self->log('info', "migration speed limit: " . render_bytes($migrate_speed, 1) . "/s"); } else { # always set migrate speed as QEMU default to 128 MiBps == 1 Gbps, use 16 GiBps == 128 Gbps $migrate_speed = (16 << 30); } $qemu_migrate_params->{'max-bandwidth'} = int($migrate_speed); my $migrate_downtime = $defaults->{migrate_downtime}; $migrate_downtime = $conf->{migrate_downtime} if defined($conf->{migrate_downtime}); # migrate-set-parameters expects limit in ms $migrate_downtime *= 1000; $migrate_downtime = cap_migrate_downtime($self, $migrate_downtime); $self->log('info', "migration downtime limit: $migrate_downtime ms"); $qemu_migrate_params->{'downtime-limit'} = int($migrate_downtime); # set cachesize to 10% of the total memory my $memory = get_current_memory($conf->{memory}); my $cachesize = int($memory * 1048576 / 10); $cachesize = round_powerof2($cachesize); $self->log('info', "migration cachesize: " . render_bytes($cachesize, 1)); $qemu_migrate_params->{'xbzrle-cache-size'} = int($cachesize); $self->log('info', "set migration parameters"); eval { mon_cmd($vmid, "migrate-set-parameters", %{$qemu_migrate_params}); }; $self->log('info', "migrate-set-parameters error: $@") if $@; if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && !$self->{opts}->{remote}) { my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $target_version = PVE::QemuServer::Helpers::get_node_pvecfg_version($self->{node}); my $ticket_port = undef; # Check if target is new enough for having the port encoded in the proxy ticket. if ( $target_version && PVE::QemuServer::Helpers::pvecfg_min_version($target_version, 9, 1, 9) ) { $ticket_port = $spice_port; } my (undef, $proxyticket) = PVE::AccessControl::assemble_spice_ticket( $authuser, $vmid, $self->{node}, $ticket_port, ); my $filename = "/etc/pve/nodes/$self->{node}/pve-ssl.pem"; my $subject = PVE::AccessControl::read_x509_subject_spice($filename); $self->log('info', "spice client_migrate_info"); eval { mon_cmd( $vmid, "client_migrate_info", protocol => 'spice', hostname => $proxyticket, 'port' => 0, 'tls-port' => $spice_port, 'cert-subject' => $subject, ); }; $self->log('info', "client_migrate_info error: $@") if $@; } my $start = time(); $self->log('info', "start migrate command to $migrate_uri"); eval { mon_cmd($vmid, "migrate", uri => $migrate_uri); }; my $merr = $@; $self->log('info', "migrate uri => $migrate_uri failed: $merr") if $merr; my $last_mem_transferred = 0; my $last_vfio_transferred = 0; my $usleep = 1000000; my $i = 0; my $err_count = 0; my $lastrem = undef; my $downtimecounter = 0; while (1) { $i++; my $avglstat = $last_mem_transferred ? $last_mem_transferred / $i : 0; usleep($usleep); my $stat = eval { mon_cmd($vmid, "query-migrate") }; if (my $err = $@) { $err_count++; warn "query migrate failed: $err\n"; $self->log('info', "query migrate failed: $err"); if ($err_count <= 5) { usleep(1_000_000); next; } die "too many query migrate failures - aborting\n"; } my $status = $stat->{status}; if (defined($status) && $status =~ m/^(cancelling|setup|wait-unplug)$/im) { $self->log('info', "migration in status '$status' - waiting for transition"); sleep(1); next; } if (!defined($status) || $status !~ m/^(active|cancelled|completed|device|failed)$/im) { die $merr if $merr; die "unable to parse migration status '$status' - aborting\n"; } $merr = undef; $err_count = 0; my $memstat = $stat->{ram}; my $mem_transferred = $memstat->{transferred} || 0; my $vfio_transferred = $stat->{vfio}->{transferred} || 0; if ($status eq 'completed') { my $delay = time() - $start; if ($delay > 0) { my $total = $memstat->{total} || 0; my $avg_speed = render_bytes($total / $delay, 1); my $downtime = $stat->{downtime} || 0; $self->log('info', "average migration speed: $avg_speed/s - downtime $downtime ms"); } if ($mem_transferred > 0 || $vfio_transferred > 0) { my $transferred_h = render_bytes($mem_transferred, 1); my $summary = "transferred $transferred_h VM-state"; if ($vfio_transferred > 0) { my $vfio_h = render_bytes($vfio_transferred, 1); $summary .= " (+ $vfio_h VFIO-state)"; } $self->log('info', "migration $status, $summary"); } } if ($status eq 'failed' || $status eq 'cancelled') { my $message = $stat->{'error-desc'} ? "$status - $stat->{'error-desc'}" : $status; $self->log('info', "migration status error: $message"); die "aborting\n"; } if ($status ne 'active' && $status ne 'device') { $self->log('info', "migration status: $status"); last; } if ( $mem_transferred ne $last_mem_transferred || $vfio_transferred ne $last_vfio_transferred ) { my $rem = $memstat->{remaining} || 0; my $total = $memstat->{total} || 0; my $speed = ($memstat->{'pages-per-second'} // 0) * ($memstat->{'page-size'} // 0); my $dirty_rate = ($memstat->{'dirty-pages-rate'} // 0) * ($memstat->{'page-size'} // 0); # reduce sleep if remaining memory is lower than the average transfer speed $usleep = 100_000 if $avglstat && $rem < $avglstat; # also reduce logging if we poll more frequent my $should_log = $usleep > 100_000 ? 1 : ($i % 10) == 0; my $total_h = render_bytes($total, 1); my $transferred_h = render_bytes($mem_transferred, 1); my $speed_h = render_bytes($speed, 1); my $progress = "transferred $transferred_h of $total_h VM-state, ${speed_h}/s"; if ($vfio_transferred > 0) { my $vfio_h = render_bytes($vfio_transferred, 1); $progress .= " (+ $vfio_h VFIO-state)"; } if ($dirty_rate > $speed) { my $dirty_rate_h = render_bytes($dirty_rate, 1); $progress .= ", VM dirties lots of memory: $dirty_rate_h/s"; } $self->log('info', "migration $status, $progress") if $should_log; my $xbzrle = $stat->{"xbzrle-cache"} || {}; my ($xbzrlebytes, $xbzrlepages) = $xbzrle->@{ 'bytes', 'pages' }; if ($xbzrlebytes || $xbzrlepages) { my $bytes_h = render_bytes($xbzrlebytes, 1); my $msg = "send updates to $xbzrlepages pages in $bytes_h encoded memory"; $msg .= sprintf(", cache-miss %.2f%%", $xbzrle->{'cache-miss-rate'} * 100) if $xbzrle->{'cache-miss-rate'}; $msg .= ", overflow $xbzrle->{overflow}" if $xbzrle->{overflow}; $self->log('info', "xbzrle: $msg") if $should_log; } if (($lastrem && $rem > $lastrem) || ($rem == 0)) { $downtimecounter++; } $lastrem = $rem; if ($downtimecounter > 5) { $downtimecounter = 0; $migrate_downtime = increase_migrate_downtime($self, $vmid, $migrate_downtime); } } $last_mem_transferred = $mem_transferred; $last_vfio_transferred = $vfio_transferred; } if ($self->{storage_migration}) { # finish block-job with block-job-cancel, to disconnect source VM from NBD # to avoid it trying to re-establish it. We are in blockjob ready state, # thus, this command changes to it to blockjob complete (see qapi docs) eval { PVE::QemuServer::BlockJob::monitor( vm_qmp_peer($vmid), undef, $self->{storage_migration_jobs}, 'cancel', ); }; if (my $err = $@) { die "Failed to complete storage migration: $err\n"; } } } sub phase2_cleanup { my ($self, $vmid, $err) = @_; return if !$self->{errors}; $self->{phase2errors} = 1; $self->log('info', "aborting phase 2 - cleanup resources"); $self->log('info', "migrate_cancel"); eval { mon_cmd($vmid, "migrate_cancel"); }; $self->log('info', "migrate_cancel error: $@") if $@; my $vm_status = eval { mon_cmd($vmid, 'query-status')->{status} or die "no 'status' in result\n"; }; $self->log('err', "query-status error: $@") if $@; # Can end up in POSTMIGRATE state if failure occurred after convergence. Try going back to # original state. Unfortunately, direct transition from POSTMIGRATE to PAUSED is not possible. if ($vm_status && $vm_status eq 'postmigrate') { if (!$self->{vm_was_paused}) { eval { mon_cmd($vmid, 'cont'); }; $self->log('err', "resuming VM failed: $@") if $@; } else { $self->log('err', "VM was paused, but ended in postmigrate state"); } } my $conf = $self->{vmconf}; delete $conf->{lock}; eval { PVE::QemuConfig->write_config($vmid, $conf) }; if (my $err = $@) { $self->log('err', $err); } # cleanup resources on target host if ($self->{storage_migration}) { eval { PVE::QemuServer::BlockJob::qemu_blockjobs_cancel( vm_qmp_peer($vmid), $self->{storage_migration_jobs}, ); }; if (my $err = $@) { $self->log('err', $err); } } eval { $self->cleanup_bitmaps() }; if (my $err = $@) { $self->log('err', $err); } my $nodename = PVE::INotify::nodename(); if ($self->{tunnel} && $self->{tunnel}->{version} >= 2) { PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'stop'); } else { my $cmd = [@{ $self->{rem_ssh} }, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename]; eval { PVE::Tools::run_command($cmd, outfunc => sub { }, errfunc => sub { }); }; if (my $err = $@) { $self->log('err', $err); $self->{errors} = 1; } } # cleanup after stopping, otherwise disks might be in-use by target VM! eval { PVE::QemuMigrate::cleanup_remotedisks($self) }; if (my $err = $@) { $self->log('err', $err); } if ($self->{running} && $self->{opts}->{'with-conntrack-state'}) { # if the VM is running, that means we also tried to migrate additional # state via our dbus-vmstate helper # only need to locally stop it, on the target the VM cleanup will # handle it PVE::QemuServer::DBusVMState::qemu_del_dbus_vmstate($vmid); } if ($self->{tunnel}) { eval { PVE::Tunnel::finish_tunnel($self->{tunnel}); }; if (my $err = $@) { $self->log('err', $err); $self->{errors} = 1; } } } sub phase3 { my ($self, $vmid) = @_; return; } sub phase3_cleanup { my ($self, $vmid, $err) = @_; my $conf = $self->{vmconf}; return if $self->{phase2errors}; my $tunnel = $self->{tunnel}; # we'll need an unmodified copy of the config later for the cleanup my $oldconf = dclone($conf); if ($self->{volume_map} && !$self->{opts}->{remote}) { my $target_drives = $self->{target_drive}; # FIXME: for NBD storage migration we now only update the volid, and # not the full drivestr from the target node. Workaround that until we # got some real rescan, to avoid things like wrong format in the drive delete $conf->{$_} for keys %$target_drives; PVE::QemuConfig->update_volume_ids($conf, $self->{volume_map}); for my $drive (keys %$target_drives) { $conf->{$drive} = $target_drives->{$drive}->{drivestr}; } PVE::QemuConfig->write_config($vmid, $conf); } # transfer replication state before move config if (!$self->{opts}->{remote}) { $self->transfer_replication_state() if $self->{is_replicated}; PVE::QemuConfig->move_config_to_node($vmid, $self->{node}); $self->switch_replication_job_target() if $self->{is_replicated}; } if ($self->{livemigration}) { if ($self->{stopnbd}) { $self->log('info', "stopping NBD storage migration server on target."); # stop nbd server on remote vm - requirement for resume since 2.9 if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 2) { eval { PVE::Tunnel::write_tunnel($tunnel, 30, 'nbdstop'); }; if (my $err = $@) { $self->log('err', $err); $self->{errors} = 1; } } else { my $cmd = [@{ $self->{rem_ssh} }, 'qm', 'nbdstop', $vmid]; eval { PVE::Tools::run_command($cmd, outfunc => sub { }, errfunc => sub { }); }; if (my $err = $@) { $self->log('err', $err); $self->{errors} = 1; } } } # deletes local FDB entries if learning is disabled, they'll be re-added on target on resume PVE::QemuServer::Network::del_nets_bridge_fdb($conf, $vmid); if (!$self->{vm_was_paused}) { # config moved and nbd server stopped - now we can resume vm on target if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 1) { my $cmd = $tunnel->{version} == 1 ? "resume $vmid" : "resume"; eval { PVE::Tunnel::write_tunnel($tunnel, 30, $cmd); }; if (my $err = $@) { $self->log('err', $err); $self->{errors} = 1; } } else { # nocheck in case target node hasn't processed the config move/rename yet my $cmd = [@{ $self->{rem_ssh} }, 'qm', 'resume', $vmid, '--skiplock', '--nocheck']; my $logf = sub { my $line = shift; $self->log('err', $line); }; eval { PVE::Tools::run_command($cmd, outfunc => sub { }, errfunc => $logf); }; if (my $err = $@) { $self->log('err', $err); $self->{errors} = 1; } } } if ( $self->{storage_migration} && PVE::QemuServer::Agent::get_qga_key($conf, 'fstrim_cloned_disks') && $self->{running} ) { if (!$self->{vm_was_paused}) { $self->log('info', "issuing guest fstrim"); if ($self->{opts}->{remote}) { PVE::Tunnel::write_tunnel($self->{tunnel}, 600, 'fstrim'); } else { my $cmd = [@{ $self->{rem_ssh} }, 'qm', 'guest', 'cmd', $vmid, 'fstrim']; eval { PVE::Tools::run_command($cmd, outfunc => sub { }, errfunc => sub { }); }; if (my $err = $@) { $self->log('err', "fstrim failed - $err"); $self->{errors} = 1; } } } else { $self->log('info', "skipping guest fstrim, because VM is paused"); } } if ($self->{running} && $self->{opts}->{'with-conntrack-state'}) { # if the VM is running, that means we also migrated additional # state via our dbus-vmstate helper $self->log('info', 'stopping migration dbus-vmstate helpers'); # first locally my $num = PVE::QemuServer::DBusVMState::qemu_del_dbus_vmstate($vmid); if (defined($num)) { my $plural = $num != 1 ? "entries" : "entry"; $self->log('info', "migrated $num conntrack state $plural"); } # .. and then remote my $targetnode = $self->{node}; eval { # FIXME: introduce proper way to call API methods on another node? # See also e.g. pve-network/src/PVE/API2/Network/SDN.pm, which # does something similar. PVE::Tools::run_command([ 'pvesh', 'create', "/nodes/$targetnode/qemu/$vmid/dbus-vmstate", '--action', 'stop', ]); }; if (my $err = $@) { $self->log('warn', "failed to stop dbus-vmstate on $targetnode: $err\n"); } # also flush now-old local conntrack entries for the migrated VM $self->log('info', 'flushing conntrack state for guest on source node'); PVE::Firewall::Helpers::flush_fw_ct_entries_by_mark($vmid); } } # close tunnel on successful migration, on error phase2_cleanup closed it if ($tunnel && $tunnel->{version} == 1) { eval { PVE::Tunnel::finish_tunnel($tunnel); }; if (my $err = $@) { $self->log('err', $err); $self->{errors} = 1; } $tunnel = undef; delete $self->{tunnel}; } eval { my $timer = 0; if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && $self->{running}) { $self->log('info', "Waiting for spice server migration"); while (1) { my $res = mon_cmd($vmid, 'query-spice'); last if int($res->{'migrated'}) == 1; last if $timer > 50; $timer++; usleep(200000); } } }; # always stop local VM with nocheck, since config is moved already eval { PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1, 1); }; if (my $err = $@) { $self->log('err', "stopping vm failed - $err"); $self->{errors} = 1; } # stop with nocheck does not do a cleanup, so do it here with the original config eval { PVE::QemuServer::vm_stop_cleanup($self->{storecfg}, $vmid, $oldconf) }; if (my $err = $@) { $self->log('err', "Cleanup after stopping VM failed - $err"); $self->{errors} = 1; } my @not_replicated_volumes = $self->filter_local_volumes(undef, 0); # destroy local copies foreach my $volid (@not_replicated_volumes) { # remote is cleaned up below next if $self->{opts}->{remote}; eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); }; if (my $err = $@) { $self->log('err', "removing local copy of '$volid' failed - $err"); $self->{errors} = 1; last if $err =~ /^interrupted by signal$/; } } # clear migrate lock if ($tunnel && $tunnel->{version} >= 2) { PVE::Tunnel::write_tunnel($tunnel, 10, "unlock"); PVE::Tunnel::finish_tunnel($tunnel); } else { my $cmd = [@{ $self->{rem_ssh} }, 'qm', 'unlock', $vmid]; $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock"); } if ($self->{opts}->{remote} && $self->{opts}->{delete}) { eval { PVE::QemuServer::destroy_vm($self->{storecfg}, $vmid, 1, undef, 0) }; warn "Failed to remove source VM - $@\n" if $@; } } sub final_cleanup { my ($self, $vmid) = @_; # nothing to do } sub round_powerof2 { return 1 if $_[0] < 2; return 2 << int(log($_[0] - 1) / log(2)); } 1; ================================================ FILE: src/PVE/QemuServer/Agent.pm ================================================ package PVE::QemuServer::Agent; use strict; use warnings; use JSON; use MIME::Base64 qw(decode_base64 encode_base64); use PVE::JSONSchema; use PVE::QemuServer::Helpers; use PVE::QemuServer::Monitor; use base 'Exporter'; our @EXPORT_OK = qw( check_agent_error agent_cmd get_qga_key parse_guest_agent qga_check_running ); our $agent_fmt = { enabled => { description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.", type => 'boolean', default => 0, default_key => 1, }, fstrim_cloned_disks => { description => "Run fstrim after moving a disk or migrating the VM.", type => 'boolean', optional => 1, default => 0, }, # keep for old backup restore compatibility 'freeze-fs-on-backup' => { alias => 'freeze-fs' }, # TODO: was only on test repo, drop with PVE 10. 'guest-fsfreeze' => { alias => 'freeze-fs' }, 'freeze-fs' => { description => "Freeze guest filesystems through QGA for consistent disk state on" . " operations such as snapshots, backups, replications and clones.", verbose_description => "Whether to issue the guest-fsfreeze-freeze and guest-fsfreeze-thaw QEMU guest agent" . " commands. Backups in snapshot mode, clones, snapshots without RAM, importing" . " disks from a running guest, and replications normally issue a guest-fsfreeze-freeze" . " and a respective thaw command when the QEMU Guest agent option is enabled in the" . " guest's configuration and the agent is running inside of the guest.\n\nThe deprecated" . " 'freeze-fs-on-backup' setting is treated as an alias for this setting.", type => 'boolean', optional => 1, default => 1, }, type => { description => "Select the agent type", type => 'string', default => 'virtio', optional => 1, enum => [qw(virtio isa)], }, }; sub parse_guest_agent { my ($conf) = @_; return {} if !defined($conf->{agent}); my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $conf->{agent}) }; warn $@ if $@; # if the agent is disabled ignore the other potentially set properties return {} if !$res->{enabled}; return $res; } sub get_qga_key { my ($conf, $key) = @_; return undef if !defined($conf->{agent}); my $agent = parse_guest_agent($conf); return $agent->{$key}; } sub qga_check_running { my ($vmid, $nowarn) = @_; eval { PVE::QemuServer::Monitor::mon_cmd($vmid, "guest-ping", timeout => 3); }; if ($@) { warn "QEMU Guest Agent is not running - $@" if !$nowarn; return 0; } return 1; } sub check_agent_error { my ($result, $errmsg, $noerr) = @_; $errmsg //= ''; my $error = ''; if (ref($result) eq 'HASH' && $result->{error} && $result->{error}->{desc}) { $error = "Agent error: $result->{error}->{desc}\n"; } elsif (!defined($result)) { $error = "Agent error: $errmsg\n"; } if ($error) { die $error if !$noerr; warn $error; return; } return 1; } sub assert_agent_available { my ($vmid, $conf) = @_; die "No QEMU guest agent configured\n" if !defined($conf->{agent}); die "VM $vmid is not running\n" if !PVE::QemuServer::Helpers::vm_running_locally($vmid); die "QEMU guest agent is not running\n" if !qga_check_running($vmid, 1); } # loads config, checks if available, executes command, checks for errors sub agent_cmd { my ($vmid, $conf, $cmd, $params, $errormsg) = @_; assert_agent_available($vmid, $conf); my $res = PVE::QemuServer::Monitor::mon_cmd($vmid, "guest-$cmd", %$params); check_agent_error($res, $errormsg); return $res; } sub qemu_exec { my ($vmid, $conf, $input_data, $cmd) = @_; my $args = { 'capture-output' => JSON::true, }; if ($cmd) { $args->{path} = shift @$cmd; $args->{arg} = $cmd; } $args->{'input-data'} = encode_base64($input_data, '') if defined($input_data); die "command or input-data (or both) required\n" if !defined($args->{'input-data'}) && !defined($args->{path}); my $errmsg = "can't execute command"; if ($cmd) { $errmsg .= " ($args->{path} $args->{arg})"; } if (defined($input_data)) { $errmsg .= " (input-data given)"; } my $res = agent_cmd($vmid, $conf, "exec", $args, $errmsg); return $res; } sub qemu_exec_status { my ($vmid, $conf, $pid) = @_; my $res = agent_cmd($vmid, $conf, "exec-status", { pid => $pid }, "can't get exec status for '$pid'"); if ($res->{'out-data'}) { my $decoded = eval { decode_base64($res->{'out-data'}) }; warn $@ if $@; if (defined($decoded)) { $res->{'out-data'} = $decoded; } } if ($res->{'err-data'}) { my $decoded = eval { decode_base64($res->{'err-data'}) }; warn $@ if $@; if (defined($decoded)) { $res->{'err-data'} = $decoded; } } # convert JSON::Boolean to 1/0 foreach my $d (keys %$res) { if (JSON::is_bool($res->{$d})) { $res->{$d} = ($res->{$d}) ? 1 : 0; } } return $res; } =head3 should_fs_freeze Returns whether guest filesystem freeze/thaw should be attempted based on the agent configuration. Does B check whether the agent is actually running. =cut sub should_fs_freeze { my ($conf) = @_; my $agent = parse_guest_agent($conf); return 0 if !$agent->{enabled}; return $agent->{'freeze-fs'} // 1; } =head3 guest_fsfreeze guest_fsfreeze($vmid); Freeze the file systems of the guest C<$vmid>. Check that the guest agent is enabled and running before calling this function. Dies if the file systems cannot be frozen. With C, it can happen that a guest agent command is read, but then the guest agent never sends an answer, because the service in the guest is stopped/killed. For example, if a guest reboot happens before the command can be successfully executed. This is usually not problematic, but the fsfreeze-freeze command should use a timeout of 1 hour, so the guest agent socket would be blocked for that amount of time, waiting on a command that is not being executed anymore. This function uses a lower timeout for the initial fsfreeze-freeze command, and issues an fsfreeze-status command afterwards, which will return immediately if the fsfreeze-freeze command already finished, and which will be queued if not. This is used as a proxy to determine whether the fsfreeze-freeze command is still running and to check whether it was successful. Using a too low timeout would mean stuffing/queuing many fsfreeze-status commands while the guest agent might still be busy actually doing the freeze. In total, fsfreeze-freeze is still allowed to take 1 hour, but the time the socket is blocked after a lost command is at most 10 minutes. =cut sub guest_fsfreeze { my ($vmid) = @_; my $timeout = 10 * 60; my $result = eval { PVE::QemuServer::Monitor::mon_cmd($vmid, 'guest-fsfreeze-freeze', timeout => $timeout); }; if ($result && ref($result) eq 'HASH' && $result->{error}) { my $error = $result->{error}->{desc} // 'unknown'; die "unable to freeze guest fs - $error\n"; } elsif (defined($result)) { return; # command successful } my $status; eval { my ($i, $last_iteration) = (0, 5); while ($i < $last_iteration && !defined($status)) { print "still waiting on guest fs freeze - timeout in " . ($timeout * ($last_iteration - $i) / 60) . " minutes\n"; $i++; $status = PVE::QemuServer::Monitor::mon_cmd( $vmid, 'guest-fsfreeze-status', timeout => $timeout, noerr => 1, ); if ($status && ref($status) eq 'HASH' && $status->{'error-is-timeout'}) { $status = undef; } else { check_agent_error($status, 'unknown error'); } } if (!defined($status)) { die "timeout after " . ($timeout * ($last_iteration + 1) / 60) . " minutes\n"; } }; die "querying status after freezing guest fs failed - $@" if $@; die "unable to freeze guest fs - unexpected status '$status'\n" if $status ne 'frozen'; } =head3 guest_fsthaw guest_fsthaw($vmid); Thaws the file systems of the guest C<$vmid>. Dies if the file systems cannot be thawed. See C<$guest_fsfreeze> for more details. =cut sub guest_fsthaw { my ($vmid) = @_; my $res = PVE::QemuServer::Monitor::mon_cmd($vmid, "guest-fsfreeze-thaw"); check_agent_error($res, "unable to thaw guest filesystem"); return; } 1; ================================================ FILE: src/PVE/QemuServer/BlockJob.pm ================================================ package PVE::QemuServer::BlockJob; use strict; use warnings; use JSON; use Storable qw(dclone); use PVE::Format qw(render_duration render_bytes); use PVE::RESTEnvironment qw(log_warn); use PVE::Storage; use PVE::QemuServer::Agent qw(qga_check_running); use PVE::QemuServer::Blockdev; use PVE::QemuServer::Drive qw(checked_volume_format); use PVE::QemuServer::Monitor qw(mon_cmd qmp_cmd vm_qmp_peer); use PVE::QemuServer::RunState; # If the job was started with auto-dismiss=false, it's necessary to dismiss it manually. Using this # option is useful to get the error for failed jobs here. QEMU's job lock should make it impossible # to see a job in 'concluded' state when auto-dismiss=true. # $qmp_info is the 'BlockJobInfo' for the job returned by query-block-jobs. # $job is the information about the job recorded on the PVE-side. # A block node $job->{'detach-node-name'} will be detached if present. sub qemu_handle_concluded_blockjob { my ($qmp_peer, $job_id, $qmp_info, $job) = @_; eval { qmp_cmd($qmp_peer, 'job-dismiss', id => $job_id); }; log_warn("$job_id: failed to dismiss job - $@") if $@; # If there was an error or if the job was cancelled, always detach the target. This is correct # even when the job was cancelled after completion, because then the disk is not switched over # to use the target. $job->{'detach-node-name'} = $job->{'target-node-name'} if $qmp_info->{error} || $job->{cancel}; if (my $node_name = $job->{'detach-node-name'}) { eval { PVE::QemuServer::Blockdev::detach($qmp_peer, $node_name); }; log_warn($@) if $@; } die "$job_id: $qmp_info->{error} (io-status: $qmp_info->{'io-status'})\n" if $qmp_info->{error}; } sub qemu_blockjobs_cancel { my ($qmp_peer, $jobs) = @_; foreach my $job (keys %$jobs) { print "$job: Cancelling block job\n"; eval { qmp_cmd($qmp_peer, "block-job-cancel", device => $job); }; $jobs->{$job}->{cancel} = 1; } while (1) { my $stats = qmp_cmd($qmp_peer, "query-block-jobs"); my $running_jobs = {}; foreach my $stat (@$stats) { $running_jobs->{ $stat->{device} } = $stat; } foreach my $job (keys %$jobs) { my $info = $running_jobs->{$job}; eval { qemu_handle_concluded_blockjob($qmp_peer, $job, $info, $jobs->{$job}) if $info && $info->{status} eq 'concluded'; }; log_warn($@) if $@; # only warn and proceed with canceling other jobs if (defined($jobs->{$job}->{cancel}) && !defined($info)) { print "$job: Done.\n"; delete $jobs->{$job}; } } last if scalar(keys %$jobs) == 0; sleep 1; } } # $completion can be either # 'complete': wait until all jobs are ready, job-complete them (default) # 'cancel': wait until all jobs are ready, block-job-cancel them # 'skip': wait until all jobs are ready, return with block jobs in ready state # 'auto': wait until all jobs disappear, only use for jobs which complete automatically sub monitor { my ($qmp_peer, $vmiddst, $jobs, $completion, $qga, $op) = @_; die "drive mirror: different destination is only supported when peer is main QEMU instance\n" if $vmiddst && $qmp_peer->{type} ne 'qmp'; $completion //= 'complete'; $op //= "mirror"; eval { my $err_complete = 0; my $starttime = time(); while (1) { die "block job ('$op') timed out\n" if $err_complete > 300; my $stats = qmp_cmd($qmp_peer, "query-block-jobs"); my $ctime = time(); my $running_jobs = {}; for my $stat (@$stats) { next if $stat->{type} ne $op; $running_jobs->{ $stat->{device} } = $stat; } my $readycounter = 0; for my $job_id (sort keys %$jobs) { my $job = $running_jobs->{$job_id}; my $vanished = !defined($job); my $complete = defined($jobs->{$job_id}->{complete}) && $vanished; if ($complete || ($vanished && $completion eq 'auto')) { print "$job_id: $op-job finished\n"; delete $jobs->{$job_id}; next; } die "$job_id: '$op' has been cancelled\n" if !defined($job); if ($job && $job->{status} eq 'concluded') { qemu_handle_concluded_blockjob($qmp_peer, $job_id, $job, $jobs->{$job_id}); } my $busy = $job->{busy}; my $ready = $job->{ready}; if (my $total = $job->{len}) { my $transferred = $job->{offset} || 0; my $remaining = $total - $transferred; my $percent = sprintf "%.2f", ($transferred * 100 / $total); my $duration = $ctime - $starttime; my $total_h = render_bytes($total, 1); my $transferred_h = render_bytes($transferred, 1); my $status = sprintf( "transferred $transferred_h of $total_h ($percent%%) in %s", render_duration($duration), ); if ($ready) { if ($busy) { $status .= ", still busy"; # shouldn't even happen? but mirror is weird } else { $status .= ", ready"; } } print "$job_id: $status\n" if !$jobs->{$job_id}->{ready}; $jobs->{$job_id}->{ready} = $ready; } $readycounter++ if $job->{ready}; } last if scalar(keys %$jobs) == 0; if ($readycounter == scalar(keys %$jobs)) { print "all '$op' jobs are ready\n"; # do the complete later (or has already been done) last if $completion eq 'skip' || $completion eq 'auto'; if ($qmp_peer->{type} eq 'qmp' && $vmiddst && $vmiddst != $qmp_peer->{id}) { my $vmid = $qmp_peer->{id}; my $should_fsfreeze = $qga && qga_check_running($vmid); if ($should_fsfreeze) { print "issuing guest agent 'guest-fsfreeze-freeze' command\n"; eval { PVE::QemuServer::Agent::guest_fsfreeze($vmid); }; warn $@ if $@; } else { if (!$qga) { print "skipping guest-agent 'guest-fsfreeze-freeze', disabled in VM" . " options\n"; } else { print "skipping guest agent 'guest-fsfreeze-freeze' command: the" . " agent is not running inside of the guest\n"; } print "suspend vm\n"; eval { PVE::QemuServer::RunState::vm_suspend($vmid, 1); }; warn $@ if $@; } # if we clone a disk for a new target vm, we don't switch the disk qemu_blockjobs_cancel($qmp_peer, $jobs); if ($should_fsfreeze) { print "issuing guest agent 'guest-fsfreeze-thaw' command\n"; eval { PVE::QemuServer::Agent::guest_fsthaw($vmid); }; warn $@ if $@; } else { print "resume vm\n"; eval { PVE::QemuServer::RunState::vm_resume($vmid, 1, 1); }; warn $@ if $@; } last; } else { for my $job_id (sort keys %$jobs) { # try to switch the disk if source and destination are on the same guest print "$job_id: Completing block job...\n"; # For blockdev, need to detach appropriate node. QEMU will only drop it if # it was implicitly added (e.g. as the child of a top throttle node), but # not if it was explicitly added via blockdev-add (e.g. as a previous mirror # target). my $detach_node_name; eval { if ($completion eq 'complete') { $detach_node_name = $jobs->{$job_id}->{'source-node-name'}; qmp_cmd($qmp_peer, 'job-complete', id => $job_id); } elsif ($completion eq 'cancel') { $detach_node_name = $jobs->{$job_id}->{'target-node-name'}; qmp_cmd($qmp_peer, 'block-job-cancel', device => $job_id); } else { die "invalid completion value: $completion\n"; } }; my $err = $@; if ($err && $err =~ m/cannot be completed/) { print "$job_id: block job cannot be completed, trying again.\n"; $err_complete++; } elsif ($err) { die "$job_id: block job cannot be completed - $err\n"; } else { $jobs->{$job_id}->{'detach-node-name'} = $detach_node_name if $detach_node_name; print "$job_id: Completed successfully.\n"; $jobs->{$job_id}->{complete} = 1; } } } } sleep 1; } }; my $err = $@; if ($err) { eval { qemu_blockjobs_cancel($qmp_peer, $jobs) }; die "block job ($op) error: $err"; } } my sub common_mirror_qmp_options { my ($device_id, $qemu_target, $src_bitmap, $bwlimit) = @_; my $opts = { timeout => 10, device => "$device_id", sync => "full", target => $qemu_target, 'auto-dismiss' => JSON::false, }; if (defined($src_bitmap)) { $opts->{sync} = 'incremental'; $opts->{bitmap} = $src_bitmap; print "drive mirror re-using dirty bitmap '$src_bitmap'\n"; } if (defined($bwlimit)) { $opts->{speed} = $bwlimit * 1024; print "drive mirror is starting for $device_id with bandwidth limit: ${bwlimit} KB/s\n"; } else { print "drive mirror is starting for $device_id\n"; } return $opts; } sub qemu_drive_mirror { my ( $vmid, $drive_id, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap, ) = @_; my $device_id = "drive-$drive_id"; $jobs = {} if !$jobs; my $qemu_target; my $format; $jobs->{$device_id} = {}; if ($dst_volid =~ /^nbd:/) { $qemu_target = $dst_volid; $format = "nbd"; } else { my $storecfg = PVE::Storage::config(); $format = checked_volume_format($storecfg, $dst_volid); my $dst_path = PVE::Storage::path($storecfg, $dst_volid); $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path; } my $opts = common_mirror_qmp_options($device_id, $qemu_target, $src_bitmap, $bwlimit); $opts->{mode} = "existing"; $opts->{format} = $format if $format; # if a job already runs for this device we get an error, catch it for cleanup eval { mon_cmd($vmid, "drive-mirror", %$opts); }; if (my $err = $@) { eval { qemu_blockjobs_cancel(vm_qmp_peer($vmid), $jobs) }; warn "$@\n" if $@; die "mirroring error: $err\n"; } monitor(vm_qmp_peer($vmid), $vmiddst, $jobs, $completion, $qga); } # Callers should version guard this (only available with a binary >= QEMU 8.2) sub qemu_drive_mirror_switch_to_active_mode { my ($vmid, $jobs) = @_; my $switching = {}; for my $job (sort keys $jobs->%*) { print "$job: switching to actively synced mode\n"; eval { mon_cmd( $vmid, "block-job-change", id => $job, type => 'mirror', 'copy-mode' => 'write-blocking', ); $switching->{$job} = 1; }; die "could not switch mirror job $job to active mode - $@\n" if $@; } while (1) { my $stats = mon_cmd($vmid, "query-block-jobs"); my $running_jobs = {}; $running_jobs->{ $_->{device} } = $_ for $stats->@*; for my $job (sort keys $switching->%*) { die "$job: vanished while switching to active mode\n" if !$running_jobs->{$job}; my $info = $running_jobs->{$job}; if ($info->{status} eq 'concluded') { qemu_handle_concluded_blockjob(vm_qmp_peer($vmid), $job, $info, $jobs->{$job}); # The 'concluded' state should occur here if and only if the job failed, so the # 'die' below should be unreachable, but play it safe. die "$job: expected job to have failed, but no error was set\n"; } if ($info->{'actively-synced'}) { print "$job: successfully switched to actively synced mode\n"; delete $switching->{$job}; } } last if scalar(keys $switching->%*) == 0; sleep 1; } } =pod =head3 blockdev_mirror blockdev_mirror($source, $dest, $jobs, $completion, $options) Mirrors the volume of a running VM specified by C<$source> to destination C<$dest>. =over =item C<$source>: The source information consists of: =over =item C<< $source->{vmid} >>: The ID of the running VM the source volume belongs to. =item C<< $source->{drive} >>: The drive configuration of the source volume as currently attached to the VM. =item C<< $source->{bitmap} >>: (optional) Use incremental mirroring based on the specified bitmap. =back =item C<$dest>: The destination information consists of: =over =item C<< $dest->{volid} >>: The volume ID of the target volume. =item C<< $dest->{vmid} >>: (optional) The ID of the VM the target volume belongs to. Defaults to C<< $source->{vmid} >>. =item C<< $dest->{'zero-initialized'} >>: (optional) True, if the target volume is zero-initialized. =back =item C<$jobs>: (optional) Other jobs in the transaction when multiple volumes should be mirrored. All jobs must be ready before completion can happen. =item C<$completion>: Completion mode, default is C: =over =item C: Wait until all jobs are ready, job-complete them (default). This means switching the original drive to use the new target. =item C: Wait until all jobs are ready, block-job-cancel them. This means not switching the original drive to use the new target. =item C: Wait until all jobs are ready, return with block jobs in ready state. =item C: Wait until all jobs disappear, only use for jobs which complete automatically. =back =item C<$options>: Further options: =over =item C<< $options->{'guest-agent'} >>: If the guest agent is configured for the VM. It will be used to freeze and thaw the filesystems for consistency when the target belongs to a different VM. =item C<< $options->{'bwlimit'} >>: The bandwidth limit to use for the mirroring operation, in KiB/s. =back =back =cut sub blockdev_mirror { my ($source, $dest, $jobs, $completion, $options) = @_; my $vmid = $source->{vmid}; my $drive_id = PVE::QemuServer::Drive::get_drive_id($source->{drive}); my $device_id = "drive-$drive_id"; my $storecfg = PVE::Storage::config(); # Need to replace the node below the top node. This is not necessarily a format node, for # example, it can also be a zeroinit node by a previous mirror! So query QEMU itself. my $source_node_name = PVE::QemuServer::Blockdev::get_node_name_below_throttle(vm_qmp_peer($vmid), $device_id, 1); # Copy original drive config (aio, cache, discard, ...): my $dest_drive = dclone($source->{drive}); delete($dest_drive->{format}); # cannot use the source's format $dest_drive->{file} = $dest->{volid}; # Mirror happens below the throttle filter, so if the target is for the same VM, it will end up # below the source's throttle filter, which is inserted for the drive device. my $attach_dest_opts = { 'no-throttle' => 1 }; $attach_dest_opts->{'zero-initialized'} = 1 if $dest->{'zero-initialized'}; # Source and target need to have the exact same virtual size, see bug #3227. # However, it won't be possible to resize a disk with 'size' explicitly set afterwards, so only # set it for EFI disks. if ($drive_id eq 'efidisk0' && !PVE::QemuServer::Blockdev::is_nbd($dest_drive)) { my ($storeid) = PVE::Storage::parse_volume_id($dest_drive->{file}, 1); if ( $storeid && PVE::QemuServer::Drive::checked_volume_format($storecfg, $dest->{volid}) eq 'raw' ) { my $block_info = PVE::QemuServer::Blockdev::get_block_info($vmid); if (my $size = $block_info->{$drive_id}->{inserted}->{image}->{'virtual-size'}) { $attach_dest_opts->{size} = $size; } else { log_warn("unable to determine source block node size - continuing anyway"); } } } # Note that if 'aio' is not explicitly set, i.e. default, it can change if source and target # don't both allow or both not allow 'io_uring' as the default. my ($target_node_name) = PVE::QemuServer::Blockdev::attach($storecfg, $vmid, $dest_drive, $attach_dest_opts); $jobs = {} if !$jobs; my $jobid = "mirror-$drive_id"; $jobs->{$jobid} = { 'source-node-name' => $source_node_name, 'target-node-name' => $target_node_name, }; my $qmp_opts = common_mirror_qmp_options( $device_id, $target_node_name, $source->{bitmap}, $options->{bwlimit}, ); $qmp_opts->{'job-id'} = "$jobid"; $qmp_opts->{replaces} = "$source_node_name"; # if a job already runs for this device we get an error, catch it for cleanup eval { mon_cmd($vmid, "blockdev-mirror", $qmp_opts->%*); }; if (my $err = $@) { eval { qemu_blockjobs_cancel(vm_qmp_peer($vmid), $jobs) }; log_warn("unable to cancel block jobs - $@"); eval { PVE::QemuServer::Blockdev::detach(vm_qmp_peer($vmid), $target_node_name); }; log_warn("unable to delete blockdev '$target_node_name' - $@"); die "error starting blockdev mirrror - $err"; } monitor( vm_qmp_peer($vmid), $dest->{vmid}, $jobs, $completion, $options->{'guest-agent'}, 'mirror', ); } sub mirror { my ($source, $dest, $jobs, $completion, $options) = @_; # for the switch to -blockdev my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($source->{vmid}); if (PVE::QemuServer::Machine::is_machine_version_at_least($machine_type, 10, 0)) { blockdev_mirror($source, $dest, $jobs, $completion, $options); } else { my $drive_id = PVE::QemuServer::Drive::get_drive_id($source->{drive}); qemu_drive_mirror( $source->{vmid}, $drive_id, $dest->{volid}, $dest->{vmid}, $dest->{'zero-initialized'}, $jobs, $completion, $options->{'guest-agent'}, $options->{bwlimit}, $source->{bitmap}, ); } } 1; ================================================ FILE: src/PVE/QemuServer/Blockdev.pm ================================================ package PVE::QemuServer::Blockdev; use strict; use warnings; use Digest::SHA; use Fcntl qw(S_ISBLK S_ISCHR); use File::stat; use JSON; use PVE::JSONSchema qw(json_bool); use PVE::Storage; use PVE::QemuServer::Drive qw(drive_is_cdrom); use PVE::QemuServer::Helpers; use PVE::QemuServer::Machine; use PVE::QemuServer::Monitor qw(mon_cmd qmp_cmd qsd_qmp_peer vm_qmp_peer); use base qw(Exporter); our @EXPORT_OK = qw( generate_file_blockdev generate_format_blockdev ); # gives ($host, $port, $export) my $NBD_TCP_PATH_RE_3 = qr/nbd:(\S+):(\d+):exportname=(\S+)/; my $NBD_UNIX_PATH_RE_2 = qr/nbd:unix:(\S+):exportname=(\S+)/; sub is_nbd { my ($drive) = @_; return 1 if $drive->{file} =~ $NBD_TCP_PATH_RE_3; return 1 if $drive->{file} =~ $NBD_UNIX_PATH_RE_2; return 0; } my sub tpm_backup_node_name { my ($type, $drive_id) = @_; if ($type eq 'fmt') { return "drive-$drive_id-backup"; # this is the top node } elsif ($type eq 'file') { return "$drive_id-backup-file"; # drop the "drive-" prefix to be sure, max length is 31 } die "unknown node type '$type' for TPM backup node"; } my sub fleecing_node_name { my ($type, $drive_id, $options) = @_; $drive_id .= '-backup' if $options->{'tpm-backup'}; if ($type eq 'fmt') { return "drive-$drive_id-fleecing"; # this is the top node for fleecing } elsif ($type eq 'file') { return "$drive_id-fleecing-file"; # drop the "drive-" prefix to be sure, max length is 31 } die "unknown node type '$type' for fleecing"; } my sub is_fleecing_top_node { my ($node_name) = @_; return $node_name =~ m/-fleecing$/ ? 1 : 0; } sub qdev_id_to_drive_id { my ($qdev_id) = @_; if ($qdev_id =~ m|^/machine/peripheral/(virtio(\d+))/virtio-backend$|) { return $1; } elsif ($qdev_id =~ m|^/machine/system\.flash0$|) { return 'pflash0'; } elsif ($qdev_id =~ m|^/machine/system\.flash1$|) { return 'efidisk0'; } return $qdev_id; # for SCSI/SATA/IDE it's the same } =pod =head3 get_block_info my $block_info = get_block_info($vmid); my $inserted = $block_info->{$drive_key}->{inserted}; my $node_name = $inserted->{'node-name'}; my $block_node_size = $inserted->{image}->{'virtual-size'}; Returns a hash reference with the information from the C QMP command indexed by configuration drive keys like C. See the QMP documentation for details. Parameters: =over =item C<$vmid>: The ID of the virtual machine to query. =back =cut sub get_block_info { my ($vmid) = @_; my $block_info = {}; my $qmp_block_info = mon_cmd($vmid, "query-block"); for my $info ($qmp_block_info->@*) { my $qdev_id = $info->{qdev} or next; my $drive_id = qdev_id_to_drive_id($qdev_id); $block_info->{$drive_id} = $info; } return $block_info; } sub get_node_name { my ($type, $drive_id, $volid, $options) = @_; return fleecing_node_name($type, $drive_id, $options) if $options->{fleecing}; return tpm_backup_node_name($type, $drive_id) if $options->{'tpm-backup'}; my $snap = $options->{'snapshot-name'}; my $info = "drive=$drive_id,"; $info .= "snap=$snap," if defined($snap); $info .= "volid=$volid"; my $hash = substr(Digest::SHA::sha256_hex($info), 0, 30); my $prefix = ""; if ($type eq 'alloc-track') { $prefix = 'a'; } elsif ($type eq 'file') { $prefix = 'e'; } elsif ($type eq 'fmt') { $prefix = 'f'; } elsif ($type eq 'zeroinit') { $prefix = 'z'; } else { die "unknown node type '$type'"; } # node-name must start with an alphabetical character return "${prefix}${hash}"; } sub parse_top_node_name { my ($node_name) = @_; if ($node_name =~ m/^drive-(.+)$/) { my $drive_id = $1; return $drive_id if PVE::QemuServer::Drive::is_valid_drivename($drive_id); } return; } sub top_node_name { my ($drive_id) = @_; return "drive-$drive_id"; } sub get_node_name_below_throttle { my ($qmp_peer, $device_id, $assert_top_is_throttle) = @_; my $top; if ($qmp_peer->{type} eq 'qmp') { # get_block_info() only works if there are front-end devices. my $block_info = get_block_info($qmp_peer->{id}); my $drive_id = $device_id =~ s/^drive-//r; $top = $block_info->{$drive_id}->{inserted}; } else { my $named_block_node_info = qmp_cmd($qmp_peer, 'query-named-block-nodes'); for my $info ($named_block_node_info->@*) { next if $info->{'node-name'} ne $device_id; $top = $info; last; } } die "no block node found for drive '$device_id'\n" if !$top; if ($top->{drv} ne 'throttle') { die "$device_id: unexpected top node $top->{'node-name'} ($top->{drv})\n" if $assert_top_is_throttle; # before the switch to -blockdev, the top node was not throttle return $top->{'node-name'}; } my $children = { map { $_->{child} => $_ } $top->{children}->@* }; if (my $node_name = $children->{file}->{'node-name'}) { return $node_name; } die "$device_id: throttle node without file child node name!\n"; } my sub read_only_json_option { my ($drive, $options) = @_; return json_bool($drive->{ro} || drive_is_cdrom($drive) || $options->{'read-only'}); } # Common blockdev options that need to be set across the whole throttle->fmt->file chain. my sub add_common_options { my ($blockdev, $drive, $options) = @_; if (!drive_is_cdrom($drive)) { $blockdev->{discard} = $drive->{discard} && $drive->{discard} eq 'on' ? 'unmap' : 'ignore'; $blockdev->{'detect-zeroes'} = PVE::QemuServer::Drive::detect_zeroes_cmdline_option($drive); } $blockdev->{'read-only'} = read_only_json_option($drive, $options); } my sub throttle_group_id { my ($drive_id) = @_; return "throttle-drive-$drive_id"; } sub generate_throttle_group { my ($drive) = @_; my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive); my $limits = {}; for my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) { my ($dir, $qmpname) = @$type; if (my $v = $drive->{"mbps$dir"}) { $limits->{"bps$qmpname"} = int($v * 1024 * 1024); } if (my $v = $drive->{"mbps${dir}_max"}) { $limits->{"bps$qmpname-max"} = int($v * 1024 * 1024); } if (my $v = $drive->{"bps${dir}_max_length"}) { $limits->{"bps$qmpname-max-length"} = int($v); } if (my $v = $drive->{"iops${dir}"}) { $limits->{"iops$qmpname"} = int($v); } if (my $v = $drive->{"iops${dir}_max"}) { $limits->{"iops$qmpname-max"} = int($v); } if (my $v = $drive->{"iops${dir}_max_length"}) { $limits->{"iops$qmpname-max-length"} = int($v); } } return { id => throttle_group_id($drive_id), limits => $limits, 'qom-type' => 'throttle-group', }; } my sub generate_blockdev_drive_cache { my ($drive, $scfg) = @_; my $cache_direct = PVE::QemuServer::Drive::drive_uses_cache_direct($drive, $scfg); return { direct => json_bool($cache_direct), 'no-flush' => json_bool($drive->{cache} && $drive->{cache} eq 'unsafe'), }; } sub generate_file_blockdev { my ($storecfg, $drive, $machine_version, $options) = @_; my $blockdev = {}; my $scfg = undef; delete $options->{'snapshot-name'} if $options->{'snapshot-name'} && $options->{'snapshot-name'} eq 'current'; die "generate_file_blockdev called without volid/path\n" if !$drive->{file}; die "generate_file_blockdev called with 'none'\n" if $drive->{file} eq 'none'; # FIXME use overlay and new config option to define storage for temp write device die "'snapshot' option is not yet supported for '-blockdev'\n" if $drive->{snapshot}; my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive); if ($drive->{file} =~ m/^$NBD_UNIX_PATH_RE_2$/) { my $server = { type => 'unix', path => "$1" }; $blockdev = { driver => 'nbd', server => $server, export => "$2" }; } elsif ($drive->{file} =~ m/^$NBD_TCP_PATH_RE_3$/) { my ($host, $port, $export) = ($1, $2, $3); if ($host =~ m/^\[(.*)\]$/) { # IPv6 address needs to be passed without square brackets $host = $1; } # port is also a string in QAPI my $server = { type => 'inet', host => "$host", port => "$port" }; $blockdev = { driver => 'nbd', server => $server, export => "$export" }; } elsif ($drive->{file} eq 'cdrom') { my $path = PVE::QemuServer::Drive::get_iso_path($storecfg, $drive->{file}); $blockdev = { driver => 'host_cdrom', filename => "$path" }; } elsif ($drive->{file} =~ m|^/|) { my $path = $drive->{file}; # The 'file' driver only works for regular files. The check below is taken from # block/file-posix.c:hdev_probe_device() in QEMU. To detect CD-ROM host devices, QEMU issues # an ioctl, while the code here relies on the media=cdrom flag instead. my $st = File::stat::stat($path) or die "stat for '$path' failed - $!\n"; my $driver = 'file'; if (S_ISCHR($st->mode) || S_ISBLK($st->mode)) { $driver = drive_is_cdrom($drive) ? 'host_cdrom' : 'host_device'; } $blockdev = { driver => "$driver", filename => "$path" }; } else { my $volid = $drive->{file}; my ($storeid) = PVE::Storage::parse_volume_id($volid); my $vtype = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[0]; die "$drive_id: explicit media parameter is required for iso images\n" if !defined($drive->{media}) && defined($vtype) && $vtype eq 'iso'; my $storage_opts = { hints => {} }; $storage_opts->{hints}->{'efi-disk'} = 1 if $drive->{interface} eq 'efidisk'; $storage_opts->{'snapshot-name'} = $options->{'snapshot-name'} if defined($options->{'snapshot-name'}); $blockdev = PVE::Storage::qemu_blockdev_options($storecfg, $volid, $machine_version, $storage_opts); $scfg = PVE::Storage::storage_config($storecfg, $storeid); } # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329 # It also needs the rbd_cache_policy set to 'writeback' on the RBD side, which is done by the # storage layer. if ($blockdev->{driver} eq 'rbd' && $drive->{interface} eq 'efidisk') { $blockdev->{cache} = { direct => JSON::false, 'no-flush' => JSON::false }; } else { $blockdev->{cache} = generate_blockdev_drive_cache($drive, $scfg); } my $driver = $blockdev->{driver}; # only certain drivers have the aio setting if ($driver eq 'file' || $driver eq 'host_cdrom' || $driver eq 'host_device') { $blockdev->{aio} = PVE::QemuServer::Drive::aio_cmdline_option($scfg, $drive, $blockdev->{cache}->{direct}); } $blockdev->{'node-name'} = get_node_name('file', $drive_id, $drive->{file}, $options); add_common_options($blockdev, $drive, $options); return $blockdev; } sub generate_format_blockdev { my ($storecfg, $drive, $child, $options) = @_; die "generate_format_blockdev called without volid/path\n" if !$drive->{file}; die "generate_format_blockdev called with 'none'\n" if $drive->{file} eq 'none'; die "generate_format_blockdev called with NBD path\n" if is_nbd($drive); delete($options->{'snapshot-name'}) if $options->{'snapshot-name'} && $options->{'snapshot-name'} eq 'current'; my $scfg; my $format; my $volid = $drive->{file}; my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive); my ($storeid) = PVE::Storage::parse_volume_id($volid, 1); # For PVE-managed volumes, use the format from the storage layer and prevent overrides via the # drive's 'format' option. For unmanaged volumes, fallback to 'raw' to avoid auto-detection by # QEMU. if ($storeid) { $scfg = PVE::Storage::storage_config($storecfg, $storeid); $format = PVE::QemuServer::Drive::checked_volume_format($storecfg, $volid); if ($drive->{format} && $drive->{format} ne $format) { die "drive '$drive->{interface}$drive->{index}' - volume '$volid'" . " - 'format=$drive->{format}' option different from storage format '$format'\n"; } } else { $format = $drive->{format} // 'raw'; } my $node_name = get_node_name('fmt', $drive_id, $drive->{file}, $options); my $blockdev = { 'node-name' => "$node_name", driver => "$format", file => $child, cache => $child->{cache}, # define cache option on both format && file node like libvirt }; add_common_options($blockdev, $drive, $options); if (defined($options->{size})) { die "blockdev: 'size' is only supported for 'raw' format" if $format ne 'raw'; $blockdev->{size} = int($options->{size}); } # see bug #6543: without this option, fragmentation can lead to the qcow2 file growing larger # than what qemu-img measure reports, which is problematic for qcow2-on-top-of-LVM # TODO test and consider enabling this in general if ($scfg && $scfg->{'snapshot-as-volume-chain'}) { $blockdev->{'discard-no-unref'} = JSON::true if $format eq 'qcow2'; } return $blockdev; } my sub generate_backing_blockdev { use feature 'current_sub'; my ($storecfg, $snapshots, $deviceid, $drive, $machine_version, $options) = @_; my $snap_id = $options->{'snapshot-name'}; my $snapshot = $snapshots->{$snap_id}; my $parentid = $snapshot->{parent}; my $volid = $drive->{file}; my $snap_file_blockdev = generate_file_blockdev($storecfg, $drive, $machine_version, $options); $snap_file_blockdev->{filename} = $snapshot->{file}; my $snap_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $snap_file_blockdev, $options); if ($parentid) { my $options = { 'snapshot-name' => $parentid }; $snap_fmt_blockdev->{backing} = __SUB__->( $storecfg, $snapshots, $deviceid, $drive, $machine_version, $options, ); } return $snap_fmt_blockdev; } my sub generate_backing_chain_blockdev { my ($storecfg, $deviceid, $drive, $machine_version) = @_; my $volid = $drive->{file}; my $snapshots = PVE::Storage::volume_snapshot_info($storecfg, $volid); my $parentid = $snapshots->{'current'}->{parent}; return undef if !$parentid; my $options = { 'snapshot-name' => $parentid }; return generate_backing_blockdev( $storecfg, $snapshots, $deviceid, $drive, $machine_version, $options, ); } sub generate_throttle_blockdev { my ($drive, $child, $options) = @_; my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive); my $blockdev = { driver => "throttle", 'node-name' => top_node_name($drive_id), 'throttle-group' => throttle_group_id($drive_id), file => $child, }; add_common_options($blockdev, $drive, $options); return $blockdev; } sub generate_drive_blockdev { my ($storecfg, $drive, $machine_version, $options) = @_; my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive); die "generate_drive_blockdev called without volid/path\n" if !$drive->{file}; die "generate_drive_blockdev called with 'none'\n" if $drive->{file} eq 'none'; my $child = generate_file_blockdev($storecfg, $drive, $machine_version, $options); if (!is_nbd($drive)) { $child = generate_format_blockdev($storecfg, $drive, $child, $options); my $support_qemu_snapshots = PVE::Storage::volume_qemu_snapshot_method($storecfg, $drive->{file}); if ($support_qemu_snapshots && $support_qemu_snapshots eq 'mixed') { my $backing_chain = generate_backing_chain_blockdev( $storecfg, "drive-$drive_id", $drive, $machine_version, ); $child->{backing} = $backing_chain if $backing_chain; } } if ($options->{'zero-initialized'}) { my $node_name = get_node_name('zeroinit', $drive_id, $drive->{file}, $options); $child = { driver => 'zeroinit', file => $child, 'node-name' => "$node_name" }; } if (my $live_restore = $options->{'live-restore'}) { my $node_name = get_node_name('alloc-track', $drive_id, $drive->{file}, $options); $child = { driver => 'alloc-track', 'auto-remove' => JSON::true, backing => $live_restore->{blockdev}, file => $child, 'node-name' => "$node_name", }; } if ($drive->{scsiblock}) { # When using scsi-block for the front-end device, throttling would not work in any case, and # the throttle block driver doesn't allow doing the necessary ioctls(), so don't attach a # throttle filter. Implementing live mirroring for such disks would require special care! $child->{'node-name'} = top_node_name($drive_id); return $child; } # for fleecing and TPM backup, this is already the top node return $child if $options->{fleecing} || $options->{'tpm-backup'} || $options->{'no-throttle'}; # this is the top filter entry point, use $drive-drive_id as nodename return generate_throttle_blockdev($drive, $child, $options); } sub generate_pbs_blockdev { my ($pbs_conf, $pbs_name) = @_; my $blockdev = { driver => 'pbs', 'node-name' => "$pbs_name", 'read-only' => JSON::true, archive => "$pbs_conf->{archive}", repository => "$pbs_conf->{repository}", snapshot => "$pbs_conf->{snapshot}", }; $blockdev->{namespace} = "$pbs_conf->{namespace}" if $pbs_conf->{namespace}; $blockdev->{keyfile} = "$pbs_conf->{keyfile}" if $pbs_conf->{keyfile}; return $blockdev; } my sub blockdev_add { my ($qmp_peer, $blockdev) = @_; eval { qmp_cmd($qmp_peer, 'blockdev-add', $blockdev->%*); }; if (my $err = $@) { my $node_name = $blockdev->{'node-name'} // 'undefined'; die "adding blockdev '$node_name' failed : $err\n" if $@; } return; } =pod =head3 attach my ($node_name, $read_only) = attach($storecfg, $id, $drive, $options); Attach the drive C<$drive> to the VM C<$id> considering the additional options C<$options>. Returns the node name of the (topmost) attached block device node and whether the node is read-only. Parameters: =over =item C<$storecfg>: The storage configuration. =item C<$id>: The ID of the virtual machine or QEMU storage daemon. =item C<$drive>: The drive as parsed from a virtual machine configuration. =item C<$options>: A hash reference with additional options. =over =item C<< $options->{fleecing} >>: Generate and attach a block device for backup fleecing. =item C<< $options->{'no-throttle'} >>: Do not insert a throttle node as the top node. =item C<< $options->{'read-only'} >>: Attach the image as read-only irrespective of the configuration in C<$drive>. =item C<< $options->{size} >>: Attach the image with this virtual size. Must be smaller than the actual size of the image. The image format must be C. =item C<< $options->{'snapshot-name'} >>: Attach this snapshot of the volume C<< $drive->{file} >>, rather than the volume itself. =item C<< $options->{'tpm-backup'} >>: Generate and attach a block device for backing up the TPM state image. =item C<< $options->{'qsd'} >>: Rather than attaching to a VM, attach to a QEMU storage daemon. =back =back =cut sub attach { my ($storecfg, $id, $drive, $options) = @_; my $qmp_peer = $options->{qsd} ? qsd_qmp_peer($id) : vm_qmp_peer($id); my $machine_version; if ($options->{qsd}) { # qemu-storage-daemon runs with the installed binary version $machine_version = 'pc-i440fx-' . PVE::QemuServer::Machine::latest_installed_machine_version(); } else { $machine_version = PVE::QemuServer::Machine::get_current_qemu_machine($id); } my $blockdev = generate_drive_blockdev($storecfg, $drive, $machine_version, $options); my $throttle_group_id; if (parse_top_node_name($blockdev->{'node-name'})) { # device top nodes need a throttle group my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive); $throttle_group_id = throttle_group_id($drive_id); } eval { if ($throttle_group_id) { # Try to remove potential left-over. qmp_cmd($qmp_peer, 'object-del', id => $throttle_group_id, noerr => 1); my $throttle_group = generate_throttle_group($drive); qmp_cmd($qmp_peer, 'object-add', $throttle_group->%*); } blockdev_add($qmp_peer, $blockdev); }; if (my $err = $@) { if ($throttle_group_id) { eval { qmp_cmd($qmp_peer, 'object-del', id => $throttle_group_id); }; } die $err; } return ($blockdev->{'node-name'}, $blockdev->{'read-only'}); } =pod =head3 detach detach($qmp_peer, $node_name); Detach the block device C<$node_name> from the QMP peer C<$qmp_peer>. Also removes associated child block nodes. Parameters: =over =item C<$qmp_peer>: QMP peer information. =item C<$node_name>: The node name identifying the block node in QEMU. =back =cut sub detach { my ($qmp_peer, $node_name) = @_; die "Blockdev::detach - no node name\n" if !$node_name; my $block_info = qmp_cmd($qmp_peer, "query-named-block-nodes"); $block_info = { map { $_->{'node-name'} => $_ } $block_info->@* }; my $remove_throttle_group_id; if ((my $drive_id = parse_top_node_name($node_name)) && $block_info->{$node_name}) { $remove_throttle_group_id = throttle_group_id($drive_id); } while ($node_name) { last if !$block_info->{$node_name}; # already gone my $res = qmp_cmd($qmp_peer, 'blockdev-del', 'node-name' => "$node_name", noerr => 1); if (my $err = $res->{error}) { last if $err =~ m/Failed to find node with node-name/; # already gone die "deleting blockdev '$node_name' failed : $err\n"; } my $children = { map { $_->{child} => $_ } $block_info->{$node_name}->{children}->@* }; # Recursively remove 'file' child nodes. QEMU will auto-remove implicitly added child nodes, # but e.g. the child of the top throttle node might have been explicitly added as a mirror # target, and needs to be removed manually. $node_name = $children->{file}->{'node-name'}; } if ($remove_throttle_group_id) { eval { qmp_cmd($qmp_peer, 'object-del', id => $remove_throttle_group_id); }; die "removing throttle group failed - $@\n" if $@; } return; } sub detach_tpm_backup_node { my ($vmid) = @_; detach(vm_qmp_peer($vmid), "drive-tpmstate0-backup"); } sub detach_fleecing_block_nodes { my ($vmid, $log_func) = @_; my $block_info = mon_cmd($vmid, "query-named-block-nodes"); for my $info ($block_info->@*) { my $node_name = $info->{'node-name'}; next if !is_fleecing_top_node($node_name); $log_func->('info', "detaching (old) fleecing image '$node_name'"); eval { detach(vm_qmp_peer($vmid), $node_name) }; $log_func->('warn', "error detaching (old) fleecing image '$node_name' - $@") if $@; } } sub resize { my ($vmid, $deviceid, $storecfg, $volid, $size) = @_; my $running = PVE::QemuServer::Helpers::vm_running_locally($vmid); PVE::Storage::volume_resize($storecfg, $volid, $size, $running); return if !$running; my $block_info = get_block_info($vmid); my $drive_id = $deviceid =~ s/^drive-//r; my $inserted = $block_info->{$drive_id}->{inserted} or die "no block node inserted for drive '$drive_id'\n"; my $padding = (1024 - $size % 1024) % 1024; $size = $size + $padding; mon_cmd( $vmid, "block_resize", # Need to use the top throttle node, not the node below, because QEMU won't update the size # of the top node otherwise, even though it's a filter node (as of QEMU 10.0). For legacy # -drive, there is no top throttle node, so this also is the correct node. 'node-name' => "$inserted->{'node-name'}", size => int($size), timeout => 60, ); } my sub blockdev_change_medium { my ($storecfg, $vmid, $qdev_id, $drive) = @_; # force eject if locked mon_cmd($vmid, "blockdev-open-tray", force => JSON::true, id => "$qdev_id"); mon_cmd($vmid, "blockdev-remove-medium", id => "$qdev_id"); detach(vm_qmp_peer($vmid), "drive-$qdev_id"); return if $drive->{file} eq 'none'; attach($storecfg, $vmid, $drive, {}); mon_cmd($vmid, "blockdev-insert-medium", id => "$qdev_id", 'node-name' => "drive-$qdev_id"); mon_cmd($vmid, "blockdev-close-tray", id => "$qdev_id"); } sub change_medium { my ($storecfg, $vmid, $qdev_id, $drive) = @_; my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid); # for the switch to -blockdev if (PVE::QemuServer::Machine::is_machine_version_at_least($machine_type, 10, 0)) { blockdev_change_medium($storecfg, $vmid, $qdev_id, $drive); } else { # force eject if locked mon_cmd($vmid, "eject", force => JSON::true, id => "$qdev_id"); my ($path, $format) = PVE::QemuServer::Drive::get_path_and_format($storecfg, $drive); if ($path) { # no path for 'none' mon_cmd( $vmid, "blockdev-change-medium", id => "$qdev_id", filename => "$path", format => "$format", ); } } } sub set_io_throttle { my ( $vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr, $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max, $bps_max_length, $bps_rd_max_length, $bps_wr_max_length, $iops_max_length, $iops_rd_max_length, $iops_wr_max_length, ) = @_; return if !PVE::QemuServer::Helpers::vm_running_locally($vmid); my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid); # for the switch to -blockdev if (PVE::QemuServer::Machine::is_machine_version_at_least($machine_type, 10, 0)) { mon_cmd( $vmid, 'qom-set', path => "throttle-$deviceid", property => "limits", value => { 'bps-total' => int($bps), 'bps-read' => int($bps_rd), 'bps-write' => int($bps_wr), 'iops-total' => int($iops), 'iops-read' => int($iops_rd), 'iops-write' => int($iops_wr), 'bps-total-max' => int($bps_max), 'bps-read-max' => int($bps_rd_max), 'bps-write-max' => int($bps_wr_max), 'iops-total-max' => int($iops_max), 'iops-read-max' => int($iops_rd_max), 'iops-write-max' => int($iops_wr_max), 'bps-total-max-length' => int($bps_max_length), 'bps-read-max-length' => int($bps_rd_max_length), 'bps-write-max-length' => int($bps_wr_max_length), 'iops-total-max-length' => int($iops_max_length), 'iops-read-max-length' => int($iops_rd_max_length), 'iops-write-max-length' => int($iops_wr_max_length), }, ); } else { mon_cmd( $vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr), bps_max => int($bps_max), bps_rd_max => int($bps_rd_max), bps_wr_max => int($bps_wr_max), iops_max => int($iops_max), iops_rd_max => int($iops_rd_max), iops_wr_max => int($iops_wr_max), bps_max_length => int($bps_max_length), bps_rd_max_length => int($bps_rd_max_length), bps_wr_max_length => int($bps_wr_max_length), iops_max_length => int($iops_max_length), iops_rd_max_length => int($iops_rd_max_length), iops_wr_max_length => int($iops_wr_max_length), ); } } 1; ================================================ FILE: src/PVE/QemuServer/CGroup.pm ================================================ package PVE::QemuServer::CGroup; use strict; use warnings; use Net::DBus qw(dbus_uint64 dbus_boolean); use PVE::Systemd; use base('PVE::CGroup'); sub get_subdir { my ($self, $controller, $limiting) = @_; my $vmid = $self->{vmid}; return "qemu.slice/$vmid.scope/"; } sub scope { my ($self) = @_; return $self->{vmid} . '.scope'; } my sub set_unit_properties : prototype($$) { my ($self, $properties) = @_; PVE::Systemd::systemd_call(sub { my ($if, $reactor, $finish_cb) = @_; $if->SetUnitProperties($self->scope(), dbus_boolean(1), $properties); return 1; }); } # Update the 'cpulimit' of VM. # Note that this is now the systemd API and we expect a value for `CPUQuota` as # set on VM startup, rather than cgroup values. sub change_cpu_quota { my ($self, $quota, $period) = @_; die "period is not controlled for VMs\n" if defined($period); $quota = dbus_uint64(defined($quota) ? ($quota * 10_000) : -1); set_unit_properties($self, [[CPUQuotaPerSecUSec => $quota]]); return; } # Update the 'cpuunits' of a VM. # Note that this is now the systemd API and we expect a value for `CPUQuota` as # set on VM startup, rather than cgroup values. sub change_cpu_shares { my ($self, $shares) = @_; $shares //= -1; if (PVE::CGroup::cgroup_mode() == 2) { set_unit_properties($self, [[CPUWeight => dbus_uint64($shares)]]); } else { set_unit_properties($self, [[CPUShares => dbus_uint64($shares)]]); } return; } 1; ================================================ FILE: src/PVE/QemuServer/CPUConfig.pm ================================================ package PVE::QemuServer::CPUConfig; use strict; use warnings; use JSON; use PVE::JSONSchema qw(json_bool); use PVE::Cluster qw(cfs_register_file cfs_read_file); use PVE::ProcFSTools; use PVE::RESTEnvironment qw(log_warn); use PVE::Tools qw(run_command); use PVE::QemuServer::Helpers qw(min_version get_host_arch); use base qw(PVE::SectionConfig Exporter); our @EXPORT_OK = qw( print_cpu_device get_cpu_options get_cpu_bitness is_native_arch get_amd_sev_object get_intel_tdx_object get_cvm_type ); my $arch_desc = { description => "Virtual processor architecture. Defaults to the host architecture.", type => 'string', enum => [qw(x86_64 aarch64)], }; PVE::JSONSchema::register_standard_option("pve-qm-cpu-arch", $arch_desc); # under certain race-conditions, this module might be loaded before pve-cluster # has started completely, so ensure we don't prevent the FUSE mount with our dir if (PVE::Cluster::check_cfs_is_mounted(1)) { mkdir "/etc/pve/virtual-guest"; } my $default_filename = "virtual-guest/cpu-models.conf"; cfs_register_file( $default_filename, sub { PVE::QemuServer::CPUConfig->parse_config(@_); }, sub { PVE::QemuServer::CPUConfig->write_config(@_); }, ); sub load_custom_model_conf { return cfs_read_file($default_filename); } #builtin models : reported-model is mandatory my $builtin_models_by_arch = { x86_64 => { 'x86-64-v2' => { 'reported-model' => 'qemu64', flags => "+popcnt;+pni;+sse4.1;+sse4.2;+ssse3", }, 'x86-64-v2-AES' => { 'reported-model' => 'qemu64', flags => "+aes;+popcnt;+pni;+sse4.1;+sse4.2;+ssse3", }, 'x86-64-v3' => { 'reported-model' => 'qemu64', flags => "+aes;+popcnt;+pni;+sse4.1;+sse4.2;+ssse3;+avx;+avx2;+bmi1;+bmi2;+f16c;+fma;+abm;+movbe;+xsave", }, 'x86-64-v4' => { 'reported-model' => 'qemu64', flags => "+aes;+popcnt;+pni;+sse4.1;+sse4.2;+ssse3;+avx;+avx2;+bmi1;+bmi2;+f16c;+fma;+abm;+movbe;+xsave;+avx512f;+avx512bw;+avx512cd;+avx512dq;+avx512vl", }, }, aarch64 => {}, }; my $all_builtin_models; for my $arch (keys $builtin_models_by_arch->%*) { for my $model (keys $builtin_models_by_arch->{$arch}->%*) { $all_builtin_models->{$model} = $builtin_models_by_arch->{$arch}->{$model}; } } my $depreacated_cpu_map = { # there never was such a client CPU, so map it to the server one for backward compat 'Icelake-Client' => 'Icelake-Server', 'Icelake-Client-noTSX' => 'Icelake-Server-noTSX', }; my $cputypes_32bit = { '486' => 1, 'pentium' => 1, 'pentium2' => 1, 'pentium3' => 1, 'coreduo' => 1, 'athlon' => 1, 'kvm32' => 1, 'qemu32' => 1, }; my $cpu_models_by_arch; my $all_cpu_models; # helper to make it easier for testing # initializes both '$cpu_models_by_arch' and '$all_cpu_models' sub initialize_cpu_models { $cpu_models_by_arch = { x86_64 => { # Intel CPUs 486 => 'GenuineIntel', pentium => 'GenuineIntel', pentium2 => 'GenuineIntel', pentium3 => 'GenuineIntel', coreduo => 'GenuineIntel', core2duo => 'GenuineIntel', Conroe => 'GenuineIntel', Penryn => 'GenuineIntel', Nehalem => 'GenuineIntel', 'Nehalem-IBRS' => 'GenuineIntel', Westmere => 'GenuineIntel', 'Westmere-IBRS' => 'GenuineIntel', SandyBridge => 'GenuineIntel', 'SandyBridge-IBRS' => 'GenuineIntel', IvyBridge => 'GenuineIntel', 'IvyBridge-IBRS' => 'GenuineIntel', Haswell => 'GenuineIntel', 'Haswell-IBRS' => 'GenuineIntel', 'Haswell-noTSX' => 'GenuineIntel', 'Haswell-noTSX-IBRS' => 'GenuineIntel', Broadwell => 'GenuineIntel', 'Broadwell-IBRS' => 'GenuineIntel', 'Broadwell-noTSX' => 'GenuineIntel', 'Broadwell-noTSX-IBRS' => 'GenuineIntel', 'Skylake-Client' => 'GenuineIntel', 'Skylake-Client-IBRS' => 'GenuineIntel', 'Skylake-Client-noTSX-IBRS' => 'GenuineIntel', 'Skylake-Client-v4' => 'GenuineIntel', 'Skylake-Server' => 'GenuineIntel', 'Skylake-Server-IBRS' => 'GenuineIntel', 'Skylake-Server-noTSX-IBRS' => 'GenuineIntel', 'Skylake-Server-v4' => 'GenuineIntel', 'Skylake-Server-v5' => 'GenuineIntel', 'Cascadelake-Server' => 'GenuineIntel', 'Cascadelake-Server-v2' => 'GenuineIntel', 'Cascadelake-Server-noTSX' => 'GenuineIntel', 'Cascadelake-Server-v4' => 'GenuineIntel', 'Cascadelake-Server-v5' => 'GenuineIntel', 'Cooperlake' => 'GenuineIntel', 'Cooperlake-v2' => 'GenuineIntel', KnightsMill => 'GenuineIntel', 'Icelake-Client' => 'GenuineIntel', # depreacated, removed with QEMU 7.1 'Icelake-Client-noTSX' => 'GenuineIntel', # depreacated, removed with QEMU 7.1 'Icelake-Server' => 'GenuineIntel', 'Icelake-Server-noTSX' => 'GenuineIntel', 'Icelake-Server-v3' => 'GenuineIntel', 'Icelake-Server-v4' => 'GenuineIntel', 'Icelake-Server-v5' => 'GenuineIntel', 'Icelake-Server-v6' => 'GenuineIntel', 'Icelake-Server-v7' => 'GenuineIntel', 'SapphireRapids' => 'GenuineIntel', 'SapphireRapids-v2' => 'GenuineIntel', 'SapphireRapids-v3' => 'GenuineIntel', 'SapphireRapids-v4' => 'GenuineIntel', 'GraniteRapids' => 'GenuineIntel', 'GraniteRapids-v2' => 'GenuineIntel', 'GraniteRapids-v3' => 'GenuineIntel', 'SierraForest' => 'GenuineIntel', 'SierraForest-v2' => 'GenuineIntel', 'SierraForest-v3' => 'GenuineIntel', 'ClearwaterForest' => 'GenuineIntel', # AMD CPUs athlon => 'AuthenticAMD', phenom => 'AuthenticAMD', Opteron_G1 => 'AuthenticAMD', Opteron_G2 => 'AuthenticAMD', Opteron_G3 => 'AuthenticAMD', Opteron_G4 => 'AuthenticAMD', Opteron_G5 => 'AuthenticAMD', EPYC => 'AuthenticAMD', 'EPYC-IBPB' => 'AuthenticAMD', 'EPYC-v3' => 'AuthenticAMD', 'EPYC-v4' => 'AuthenticAMD', 'EPYC-v5' => 'AuthenticAMD', 'EPYC-Rome' => 'AuthenticAMD', 'EPYC-Rome-v2' => 'AuthenticAMD', 'EPYC-Rome-v3' => 'AuthenticAMD', 'EPYC-Rome-v4' => 'AuthenticAMD', 'EPYC-Rome-v5' => 'AuthenticAMD', 'EPYC-Milan' => 'AuthenticAMD', 'EPYC-Milan-v2' => 'AuthenticAMD', 'EPYC-Milan-v3' => 'AuthenticAMD', 'EPYC-Genoa' => 'AuthenticAMD', 'EPYC-Genoa-v2' => 'AuthenticAMD', 'EPYC-Turin' => 'AuthenticAMD', # generic types, use vendor from host node kvm32 => 'default', kvm64 => 'default', qemu32 => 'default', qemu64 => 'default', max => 'default', }, aarch64 => { 'a64fx' => 'ARM', 'cortex-a35' => 'ARM', 'cortex-a53' => 'ARM', 'cortex-a55' => 'ARM', 'cortex-a57' => 'ARM', 'cortex-a710' => 'ARM', 'cortex-a72' => 'ARM', 'cortex-a76' => 'ARM', 'neoverse-n1' => 'ARM', 'neoverse-n2' => 'ARM', 'neoverse-v1' => 'ARM', # 32 bit and deprecated models were not added max => 'default', }, }; my $host_arch = get_host_arch(); # The host CPU model only exists if the arch matches $cpu_models_by_arch->{$host_arch}->{host} = 'default'; $all_cpu_models = {}; for my $arch (keys $cpu_models_by_arch->%*) { for my $model (keys $cpu_models_by_arch->{$arch}->%*) { $all_cpu_models->{$model} = $cpu_models_by_arch->{$arch}->{$model}; } } } =head3 get_cpu_models_by_arch my $cpu_vendor = get_cpu_models_by_arch($arch)->{$cpu_model}; Returns the a hash reference with all available CPU models for the given architecture C<$arch> as keys. The associated value is the vendor of the CPU model. The parameter C<$arch> must be specified. =cut sub get_cpu_models_by_arch { my ($arch) = @_; die "get_cpu_models_by_arch: required parameter 'arch' not specified\n" if !defined($arch); initialize_cpu_models() if !defined($cpu_models_by_arch); return $cpu_models_by_arch->{$arch}; } =head3 get_all_cpu_models my $cpu_vendor = get_all_cpu_models()->{$cpu_model}; Returns a hash reference with all available CPU models as keys. The associated value is the vendor of the CPU model. =cut sub get_all_cpu_models { initialize_cpu_models() if !defined($all_cpu_models); return $all_cpu_models; } my $supported_cpu_flags_by_arch = { x86_64 => [ { name => 'nested-virt', description => "Controls nested virtualization, namely 'svm' for AMD CPUs and 'vmx' for" . " Intel CPUs. Live migration still only works if it's the same flag on both sides." . " Use a CPU model similar to the host, with the same vendor, not x86-64-vX!", }, { name => 'md-clear', description => "Required to let the guest OS know if MDS is mitigated correctly.", }, { name => 'pcid', description => "Meltdown fix cost reduction on Westmere, Sandy-, and IvyBridge Intel CPUs.", }, { name => 'spec-ctrl', description => "Allows improved Spectre mitigation with Intel CPUs.", }, { name => 'ssbd', description => "Protection for 'Speculative Store Bypass' for Intel models.", }, { name => 'ibpb', description => "Allows improved Spectre mitigation with AMD CPUs.", }, { name => 'virt-ssbd', description => "Basis for 'Speculative Store Bypass' protection for AMD models.", }, { name => 'amd-ssbd', description => "Improves Spectre mitigation performance with AMD CPUs, best used with" . " 'virt-ssbd'.", }, { name => 'amd-no-ssb', description => "Notifies guest OS that host is not vulnerable for Spectre on AMD CPUs.", }, { name => 'pdpe1gb', description => "Allow guest OS to use 1GB size pages, if host HW supports it.", }, { name => 'hv-tlbflush', description => "Improve performance in overcommitted Windows guests. May lead to guest" . " bluescreens on old CPUs.", }, { name => 'hv-evmcs', description => "Improve performance for nested virtualization. Only supported on Intel" . " CPUs.", }, { name => 'aes', description => "Activate AES instruction set for HW acceleration.", }, ], aarch64 => [], }; sub get_supported_cpu_flags { my ($arch) = @_; $arch = get_host_arch() if !defined($arch); return $supported_cpu_flags_by_arch->{$arch}; } my $all_supported_cpu_flags = {}; for my $arch ($supported_cpu_flags_by_arch->%*) { for my $flag ($supported_cpu_flags_by_arch->{$arch}->@*) { $all_supported_cpu_flags->{ $flag->{name} } = 1; } } my @supported_cpu_flags_names = sort keys $all_supported_cpu_flags->%*; my $cpu_flag_supported_re = qr/([+-])(@{[join('|', @supported_cpu_flags_names)]})/; my $cpu_flag_any_re = qr/([+-])([a-zA-Z0-9\-_\.]+)/; our $qemu_cmdline_cpu_re = qr/^((?>[+-]?[\w\-\._=]+,?)+)$/; my $cpu_fmt = { cputype => { description => "Emulated CPU type. Can be default or custom name (custom model names must be prefixed with 'custom-').", type => 'string', format_description => 'string', default => 'kvm64', default_key => 1, optional => 1, }, 'reported-model' => { description => "CPU model and vendor to report to the guest. Must be a QEMU/KVM supported model." . " Only valid for custom CPU model definitions, default models will always report themselves to the guest OS.", type => 'string', enum => [sort { lc("$a") cmp lc("$b") } keys get_all_cpu_models()->%*], default => 'kvm64', optional => 1, }, hidden => { description => "Do not identify as a KVM virtual machine. Only affects vCPUs with x86-64" . " architecture.", type => 'boolean', optional => 1, default => 0, }, 'hv-vendor-id' => { type => 'string', pattern => qr/[a-zA-Z0-9]{1,12}/, format_description => 'vendor-id', description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.', optional => 1, }, flags => { description => "List of additional CPU flags separated by ';'. Use '+FLAG' to enable," . " '-FLAG' to disable a flag. There is a special 'nested-virt' shorthand which" . " controls nested virtualization for the current CPU ('svm' for AMD and 'vmx' for" . " Intel). Custom CPU models can specify any flag supported by QEMU/KVM, VM-specific" . " flags must be from the following set for security reasons: " . join(', ', @supported_cpu_flags_names), format_description => '+FLAG[;-FLAG...]', type => 'string', pattern => qr/$cpu_flag_any_re(;$cpu_flag_any_re)*/, optional => 1, }, 'guest-phys-bits' => { type => 'integer', minimum => 32, # see target/i386/cpu.c in QEMU maximum => 64, description => "Number of physical address bits available to the guest.", optional => 1, }, 'phys-bits' => { type => 'string', format => 'pve-phys-bits', format_description => '8-64|host', description => "The physical memory address bits that are reported to the guest OS. Should" . " be smaller or equal to the host's. Set to 'host' to use value from host CPU, but" . " note that doing so will break live migration to CPUs with other values.", optional => 1, }, }; my $sev_fmt = { type => { description => "Enable standard SEV with type='std' or enable" . " experimental SEV-ES with the 'es' option or enable" . " experimental SEV-SNP with the 'snp' option.", type => 'string', default_key => 1, format_description => "sev-type", enum => ['std', 'es', 'snp'], maxLength => 3, }, 'no-debug' => { description => "Sets policy bit to disallow debugging of guest", type => 'boolean', default => 0, optional => 1, }, 'no-key-sharing' => { description => "Sets policy bit to disallow key sharing with other guests (Ignored for SEV-SNP)", type => 'boolean', default => 0, optional => 1, }, 'allow-smt' => { description => "Sets policy bit to allow Simultaneous Multi Threading (SMT) (Ignored unless for SEV-SNP)", type => 'boolean', default => 1, optional => 1, }, "kernel-hashes" => { description => "Add kernel hashes to guest firmware for measured linux kernel launch", type => 'boolean', default => 0, optional => 1, }, }; PVE::JSONSchema::register_format('pve-qemu-sev-fmt', $sev_fmt); my $tdx_fmt = { type => { description => "Enable TDX", type => 'string', default_key => 1, format_description => "tdx-type", enum => ['tdx'], }, 'attestation' => { description => "Enable TDX attestation by including quote-generation-socket", type => 'boolean', default => 1, }, 'vsock-cid' => { type => 'integer', minimum => 2, default => 2, optional => 1, description => "CID for vsock of Quote Generation Service", }, 'vsock-port' => { type => 'integer', minimum => 0, default => 4050, optional => 1, description => "Port for vsock of Quote Generation Service", }, }; PVE::JSONSchema::register_format('pve-qemu-tdx-fmt', $tdx_fmt); PVE::JSONSchema::register_format('pve-phys-bits', \&parse_phys_bits); sub parse_phys_bits { my ($str, $noerr) = @_; my $err_msg = "value must be an integer between 8 and 64 or 'host'\n"; if ($str !~ m/^(host|\d{1,2})$/) { die $err_msg if !$noerr; return; } if ($str =~ m/^\d+$/ && (int($str) < 8 || int($str) > 64)) { die $err_msg if !$noerr; return; } return $str; } # $cpu_fmt describes both the CPU config passed as part of a VM config, as well # as the definition of a custom CPU model. There are some slight differences # though, which we catch in the custom validation functions below. PVE::JSONSchema::register_format('pve-cpu-conf', $cpu_fmt, \&validate_cpu_conf); sub validate_cpu_conf { my ($cpu) = @_; # required, but can't be forced in schema since it's encoded in section header for custom models die "CPU is missing cputype\n" if !$cpu->{cputype}; return $cpu; } PVE::JSONSchema::register_format('pve-vm-cpu-conf', $cpu_fmt, \&validate_vm_cpu_conf); sub validate_vm_cpu_conf { my ($cpu) = @_; validate_cpu_conf($cpu); my $cputype = $cpu->{cputype}; # a VM-specific config is only valid if the cputype exists if (is_custom_model($cputype)) { # dies on unknown model get_custom_model($cputype); } elsif ( !defined(get_all_cpu_models()->{$cputype}) && !defined($all_builtin_models->{$cputype}) ) { die "Built-in cputype '$cputype' is not defined (missing 'custom-' prefix?)\n"; } # in a VM-specific config, certain properties are limited/forbidden if ($cpu->{flags} && $cpu->{flags} !~ m/^$cpu_flag_supported_re(;$cpu_flag_supported_re)*$/) { die "VM-specific CPU flags must be a subset of: " . join(', ', @supported_cpu_flags_names) . "\n"; } if (defined($cpu->{'reported-model'})) { die "Property 'reported-model' not allowed in VM-specific CPU config.\n"; } return $cpu; } # Section config settings my $defaultData = { # shallow copy, since SectionConfig modifies propertyList internally propertyList => {%$cpu_fmt}, }; sub private { return $defaultData; } sub options { return {%$cpu_fmt}; } sub type { return 'cpu-model'; } sub parse_section_header { my ($class, $line) = @_; my ($type, $sectionId, $errmsg, $config) = $class->SUPER::parse_section_header($line); return if !$type; return ( $type, $sectionId, $errmsg, { # name is given by section header, and we can always prepend 'custom-' # since we're reading the custom CPU file cputype => "custom-$sectionId", }, ); } sub write_config { my ($class, $filename, $cfg) = @_; mkdir "/etc/pve/virtual-guest"; for my $model (keys %{ $cfg->{ids} }) { my $model_conf = $cfg->{ids}->{$model}; if (!is_custom_model($model_conf->{cputype})) { die "internal error: tried saving built-in CPU model (or missing prefix):" . " $model_conf->{cputype}\n"; } if ("custom-$model" ne $model_conf->{cputype}) { die "internal error: tried saving custom cpumodel with cputype (ignoring prefix:" . " $model_conf->{cputype}) not equal to \$cfg->ids entry ($model)\n"; } # saved in section header delete $model_conf->{cputype}; } $class->SUPER::write_config($filename, $cfg); } sub add_cpu_json_properties { my ($prop) = @_; foreach my $opt (keys %$cpu_fmt) { $prop->{$opt} = $cpu_fmt->{$opt}; } return $prop; } sub get_cpu_models { my ($include_custom, $arch) = @_; $arch = get_host_arch() if !defined($arch); my $cpu_vendor_list = get_cpu_models_by_arch($arch); my $models = []; for my $default_model (keys %{$cpu_vendor_list}) { push @$models, { name => $default_model, custom => 0, vendor => $cpu_vendor_list->{$default_model}, }; } my $builtin_models = $builtin_models_by_arch->{$arch}; for my $model (keys %{$builtin_models}) { my $reported_model = $builtin_models->{$model}->{'reported-model'}; my $vendor = $cpu_vendor_list->{$reported_model}; push @$models, { name => $model, custom => 0, vendor => $vendor, }; } return $models if !$include_custom; my $conf = load_custom_model_conf(); for my $custom_model (keys %{ $conf->{ids} }) { my $reported_model = $conf->{ids}->{$custom_model}->{'reported-model'}; $reported_model //= $cpu_fmt->{'reported-model'}->{default}; my $vendor = get_all_cpu_models()->{$reported_model}; push @$models, { name => "custom-$custom_model", custom => 1, vendor => $vendor, }; } return $models; } sub is_custom_model { my ($cputype) = @_; return $cputype =~ m/^custom-/; } # Use this to get a single model in the format described by $cpu_fmt. # Allows names with and without custom- prefix. sub get_custom_model { my ($name, $noerr) = @_; $name =~ s/^custom-//; my $conf = load_custom_model_conf(); my $entry = $conf->{ids}->{$name}; if (!defined($entry)) { die "Custom cputype '$name' not found\n" if !$noerr; return; } my $model = {}; for my $property (keys %$cpu_fmt) { if (my $value = $entry->{$property}) { $model->{$property} = $value; } } return $model; } # Print a QEMU device node for a given VM configuration for hotplugging CPUs sub print_cpu_device { my ($conf, $arch, $id) = @_; # FIXME: hot plugging other architectures like our unofficial aarch64 support? die "Hotplug of non x86_64 CPU not yet supported" if $arch ne 'x86_64'; my $kvm = $conf->{kvm} // is_native_arch($arch); my $cpu = get_default_cpu_type('x86_64', $kvm); if (my $cputype = $conf->{cpu}) { my $cpuconf = PVE::JSONSchema::parse_property_string('pve-vm-cpu-conf', $cputype) or die "Cannot parse cpu description: $cputype\n"; $cpu = $cpuconf->{cputype}; my $builtin_models = $builtin_models_by_arch->{$arch}; if (my $model = $builtin_models->{$cpu}) { $cpu = $model->{'reported-model'}; } elsif (is_custom_model($cputype)) { my $custom_cpu = get_custom_model($cpu); $cpu = $custom_cpu->{'reported-model'} // $cpu_fmt->{'reported-model'}->{default}; } if (my $replacement_type = $depreacated_cpu_map->{$cpu}) { $cpu = $replacement_type; } } my $cores = $conf->{cores} || 1; my $current_core = ($id - 1) % $cores; my $current_socket = int(($id - 1 - $current_core) / $cores); return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0"; } =head3 is_abstracted if (is_abstracted($conf->{cpu})) { # better check running CPU of QEMU instance } Check if the configured CPU is abstracted in any way, meaning that the actual CPU model cannot be determined from the configuration alone. Possible abstractions: =over =item custom model: can change with updates to the custom model configuration =item abstract flag: for example, nested-virt changes with the host =back =cut sub is_abstracted { my ($cpu_property_string) = @_; my $cpu_conf = PVE::JSONSchema::parse_property_string('pve-cpu-conf', $cpu_property_string); return 1 if is_custom_model($cpu_conf->{cputype}); return if !$cpu_conf->{flags}; for my $flag (split(";", $cpu_conf->{flags})) { if ($flag =~ $cpu_flag_supported_re) { return 1 if $2 eq 'nested-virt'; } } return; } # Resolves multiple arrays of hashes representing CPU flags with metadata to a # single string in QEMU "-cpu" compatible format. Later arrays have higher # priority. # # Hashes take the following format: # { # aes => { # op => "+", # defaults to "" if undefined # reason => "to support AES acceleration", # for override warnings # value => "" # needed for kvm=off (value: off) etc... # }, # ... # } my sub resolve_cpu_flags { my @flag_hashes = @_; my $flags = {}; my $nested_flag; my $nested_flag_resolved; my $resolve_nested_flag = sub { if (!$nested_flag_resolved) { my $host_cpu_flags = PVE::ProcFSTools::read_cpuinfo()->{flags}; if ($host_cpu_flags =~ m/\s(svm|vmx)\s/) { $nested_flag = $1; } else { log_warn( "ignoring 'nested-virt' CPU flag - unable to resolve from host CPU flags"); } $nested_flag_resolved = 1; } return $nested_flag; }; for my $hash (@flag_hashes) { for my $flag_name (keys %$hash) { if ($flag_name eq 'nested-virt') { my $nested_flag_name = $resolve_nested_flag->() or next; if ($hash->{$nested_flag_name}) { warn "warning: CPU flag '$flag_name' overrides '$nested_flag_name'\n"; } else { print "CPU flag '$flag_name' resolved to '$nested_flag_name'\n"; } $hash->{$nested_flag_name} = delete($hash->{$flag_name}); $flag_name = $nested_flag_name; } my $flag = $hash->{$flag_name}; my $old_flag = $flags->{$flag_name}; $flag->{op} //= ""; $flag->{reason} //= "unknown origin"; if ($old_flag) { my $value_changed = (defined($flag->{value}) != defined($old_flag->{value})) || (defined($flag->{value}) && $flag->{value} ne $old_flag->{value}); if ($old_flag->{op} eq $flag->{op} && !$value_changed) { $flags->{$flag_name}->{reason} .= " & $flag->{reason}"; next; } my $old = print_cpuflag_hash($flag_name, $flags->{$flag_name}); my $new = print_cpuflag_hash($flag_name, $flag); warn "warning: CPU flag/setting $new overwrites $old\n"; } $flags->{$flag_name} = $flag; } } return $flags; } my sub print_cpu_flags { my ($flags) = @_; my $flag_str = ''; # sort for command line stability for my $flag_name (sort keys %$flags) { $flag_str .= ','; $flag_str .= $flags->{$flag_name}->{op}; $flag_str .= $flag_name; $flag_str .= "=$flags->{$flag_name}->{value}" if $flags->{$flag_name}->{value}; } return $flag_str; } sub print_cpuflag_hash { my ($flag_name, $flag) = @_; my $formatted = "'$flag->{op}$flag_name"; $formatted .= "=$flag->{value}" if defined($flag->{value}); $formatted .= "'"; $formatted .= " ($flag->{reason})" if defined($flag->{reason}); return $formatted; } sub parse_cpuflag_list { my ($re, $reason, $flaglist) = @_; my $res = {}; return $res if !$flaglist; foreach my $flag (split(";", $flaglist)) { if ($flag =~ m/^$re$/) { $res->{$2} = { op => $1, reason => $reason }; } } return $res; } my sub check_phys_bits_above_40_compat { my ($bios, $cpu_type, $cpu_flags) = @_; # Would need to check CPU model expansion for others, but that information is not cheap to get # right now. Checking with 'qemu64' and 'kvm64' should cover most problematic scenarios. return if $cpu_type ne 'qemu64' && $cpu_type ne 'kvm64'; return if !$bios || $bios ne 'ovmf'; if (!$cpu_flags->{pdpe1gb} || $cpu_flags->{pdpe1gb}->{op} eq '-') { log_warn("OVMF firmware might limit CPU 'phys-bits' to 40" . " - enable the 'pdpe1gb' CPU flag to avoid this"); } } # Calculate QEMU's '-cpu' argument from a given VM configuration sub get_cpu_options { my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_; my $cputype = get_default_cpu_type($arch, $kvm); my $cpu = {}; my $custom_cpu; my $builtin_cpu; my $hv_vendor_id; if (my $cpu_prop_str = $conf->{cpu}) { $cpu = PVE::JSONSchema::parse_property_string('pve-vm-cpu-conf', $cpu_prop_str) or die "Cannot parse cpu description: $cpu_prop_str\n"; $cputype = $cpu->{cputype}; my $builtin_models = $builtin_models_by_arch->{$arch}; if (my $model = $builtin_models->{$cputype}) { $cputype = $model->{'reported-model'}; $builtin_cpu->{flags} = $model->{'flags'}; } elsif (is_custom_model($cputype)) { $custom_cpu = get_custom_model($cputype); $cputype = $custom_cpu->{'reported-model'} // $cpu_fmt->{'reported-model'}->{default}; $kvm_off = $custom_cpu->{hidden} if defined($custom_cpu->{hidden}); $hv_vendor_id = $custom_cpu->{'hv-vendor-id'}; } if (my $replacement_type = $depreacated_cpu_map->{$cputype}) { $cputype = $replacement_type; } # VM-specific settings override custom CPU config $kvm_off = $cpu->{hidden} if defined($cpu->{hidden}); $hv_vendor_id = $cpu->{'hv-vendor-id'} if defined($cpu->{'hv-vendor-id'}); } die "CPU model '$cputype' does not exist for configured vCPU architecture '$arch'\n" if !defined(get_cpu_models_by_arch($arch)->{$cputype}); my $pve_flags = get_pve_cpu_flags($conf, $kvm, $cputype, $arch, $machine_version); my $hv_flags; if ($kvm && $arch eq 'x86_64') { $hv_flags = get_hyperv_enlightenments( $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id, ); } my $builtin_cputype_flags = parse_cpuflag_list($cpu_flag_any_re, "set by builtin CPU model", $builtin_cpu->{flags}); my $custom_cputype_flags = parse_cpuflag_list($cpu_flag_any_re, "set by custom CPU model", $custom_cpu->{flags}); my $vm_flags = parse_cpuflag_list($cpu_flag_supported_re, "manually set for VM", $cpu->{flags}); my $pve_forced_flags = {}; if ($cputype ne 'host' && $kvm && $arch eq 'x86_64') { $pve_forced_flags->{'enforce'} = { reason => "error if requested CPU settings not available", }; } if ($kvm_off && $arch eq 'x86_64') { $pve_forced_flags->{'kvm'} = { value => "off", reason => "hide KVM virtualization from guest", }; } # For aarch64, QEMU does not have a vendor property for the -cpu commandline. if ($arch eq 'x86_64') { # $cputype is the "reported-model" for custom types, so we can just look up # the vendor in the default list my $cpu_vendor = get_cpu_models_by_arch($arch)->{$cputype} or die "internal error"; $pve_forced_flags->{'vendor'} = { value => $cpu_vendor } if $cpu_vendor ne 'default'; } my $cpu_str = $cputype; # will be resolved in parameter order my $resolved_flags = resolve_cpu_flags( $pve_flags, $hv_flags, $builtin_cputype_flags, $custom_cputype_flags, $vm_flags, $pve_forced_flags, ); $cpu_str .= print_cpu_flags($resolved_flags); my $using_phys_bits_above_40; for my $phys_bits_opt (qw(guest-phys-bits phys-bits)) { my $phys_bits = ''; for my $cpu_conf ($custom_cpu, $cpu) { next if !defined($cpu_conf); my $conf_val = $cpu_conf->{$phys_bits_opt}; next if !$conf_val; if ($conf_val eq 'host') { die "unexpected value 'host' for guest-phys-bits" if $phys_bits_opt eq 'guest-phys-bits'; $phys_bits = ",host-phys-bits=true"; my $host_phys_bits = PVE::QemuServer::Helpers::get_host_phys_address_bits(); $using_phys_bits_above_40 = 1 if defined($host_phys_bits) && $host_phys_bits > 40; } else { $phys_bits = ",${phys_bits_opt}=${conf_val}"; $using_phys_bits_above_40 = 1 if $phys_bits_opt eq 'phys-bits' && $conf_val > 40; } } $cpu_str .= $phys_bits; } check_phys_bits_above_40_compat($conf->{bios}, $cputype, $resolved_flags) if $using_phys_bits_above_40; return ('-cpu', $cpu_str); } # Some hardcoded flags required by certain configurations sub get_pve_cpu_flags { my ($conf, $kvm, $cputype, $arch, $machine_version) = @_; my $pve_flags = {}; my $pve_msg = "set by PVE;"; if ($cputype eq 'kvm64' && $arch eq 'x86_64') { $pve_flags->{'lahf_lm'} = { op => '+', reason => "$pve_msg to support Windows 8.1+", }; } if ($conf->{ostype} && $conf->{ostype} eq 'solaris') { $pve_flags->{'x2apic'} = { op => '-', reason => "$pve_msg incompatible with Solaris", }; } if ($cputype eq 'kvm64' || $cputype eq 'kvm32') { $pve_flags->{'sep'} = { op => '+', reason => "$pve_msg to support Windows 8+ and improve Windows XP+", }; } if ($cputype =~ m/^Opteron/) { $pve_flags->{'rdtscp'} = { op => '-', reason => "$pve_msg broken on AMD Opteron", }; } if (min_version($machine_version, 2, 3) && $kvm && $arch eq 'x86_64') { $pve_flags->{'kvm_pv_unhalt'} = { op => '+', reason => "$pve_msg to improve Linux guest spinlock performance", }; $pve_flags->{'kvm_pv_eoi'} = { op => '+', reason => "$pve_msg to improve Linux guest interrupt performance", }; } return $pve_flags; } sub get_hyperv_enlightenments { my ($winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_; return if $winversion < 6; return if $bios && $bios eq 'ovmf' && $winversion < 8; my $flags = {}; my $default_reason = "automatic Hyper-V enlightenment for Windows"; my $flagfn = sub { my ($flag, $value, $reason) = @_; $flags->{$flag} = { reason => $reason // $default_reason, value => $value, }; }; my $hv_vendor_set = defined($hv_vendor_id); if ($gpu_passthrough || $hv_vendor_set) { $hv_vendor_id //= 'proxmox'; $flagfn->( 'hv_vendor_id', $hv_vendor_id, $hv_vendor_set ? "custom hv_vendor_id set" : "NVIDIA workaround for GPU passthrough", ); } if (min_version($machine_version, 2, 3)) { $flagfn->('hv_spinlocks', '0x1fff'); $flagfn->('hv_vapic'); $flagfn->('hv_time'); } else { $flagfn->('hv_spinlocks', '0xffff'); } if (min_version($machine_version, 2, 6)) { $flagfn->('hv_reset'); $flagfn->('hv_vpindex'); $flagfn->('hv_runtime'); } if ($winversion >= 7) { my $win7_reason = $default_reason . " 7 and higher"; $flagfn->('hv_relaxed', undef, $win7_reason); if (min_version($machine_version, 2, 12)) { $flagfn->('hv_synic', undef, $win7_reason); $flagfn->('hv_stimer', undef, $win7_reason); } if (min_version($machine_version, 3, 1)) { $flagfn->('hv_ipi', undef, $win7_reason); } } return $flags; } sub get_cpu_from_running_vm { my ($pid) = @_; my $cmdline = PVE::QemuServer::Helpers::parse_cmdline($pid); die "could not read commandline of running machine\n" if !$cmdline->{cpu}->{value}; # sanitize and untaint value $cmdline->{cpu}->{value} =~ $qemu_cmdline_cpu_re; return $1; } sub get_default_cpu_type { my ($arch, $kvm) = @_; my $cputype = $kvm ? 'kvm64' : 'qemu64'; $cputype = 'cortex-a57' if $arch eq 'aarch64'; return $cputype; } sub is_native_arch($) { my ($arch) = @_; return get_host_arch() eq $arch; } sub get_cpu_bitness { my ($cpu_prop_str, $arch) = @_; $arch //= get_host_arch(); my $cputype = get_default_cpu_type($arch, 0); if ($cpu_prop_str) { my $cpu = PVE::JSONSchema::parse_property_string('pve-vm-cpu-conf', $cpu_prop_str) or die "Cannot parse cpu description: $cpu_prop_str\n"; $cputype = $cpu->{cputype}; my $builtin_models = $builtin_models_by_arch->{$arch}; if (my $model = $builtin_models->{$cputype}) { $cputype = $model->{'reported-model'}; } elsif (is_custom_model($cputype)) { my $custom_cpu = get_custom_model($cputype); $cputype = $custom_cpu->{'reported-model'} // $cpu_fmt->{'reported-model'}->{default}; } } return $cputypes_32bit->{$cputype} ? 32 : 64 if $arch eq 'x86_64'; return 64 if $arch eq 'aarch64'; die "unsupported architecture '$arch'\n"; } sub get_hw_capabilities { # Get reduced-phys-bits & cbitpos from host-hw-capabilities.json # TODO: Find better location than /run/qemu-server/ my $filename = '/run/qemu-server/host-hw-capabilities.json'; if (!-e $filename) { die "$filename does not exist. Please check the status of query-machine-capabilities: " . "systemctl status query-machine-capabilities\n"; } my $json_text = PVE::Tools::file_get_contents($filename); ($json_text) = $json_text =~ /(.*)/; # untaint json text my $hw_capabilities = eval { decode_json($json_text) }; if (my $err = $@) { die $err; } return $hw_capabilities; } sub get_cvm_type { my ($conf) = @_; if ($conf->{'amd-sev'}) { my $sev = PVE::JSONSchema::parse_property_string($sev_fmt, $conf->{'amd-sev'}); return $sev->{type}; } elsif ($conf->{'intel-tdx'}) { my $tdx = PVE::JSONSchema::parse_property_string($tdx_fmt, $conf->{'intel-tdx'}); return $tdx->{type}; } else { return undef; } } sub get_amd_sev_object { my ($amd_sev, $bios) = @_; my $amd_sev_conf = PVE::JSONSchema::parse_property_string($sev_fmt, $amd_sev); my $sev_hw_caps = get_hw_capabilities()->{'amd-sev'}; if (!$sev_hw_caps->{'sev-support'}) { die "Your CPU does not support AMD SEV.\n"; } if ($amd_sev_conf->{type} eq 'es' && !$sev_hw_caps->{'sev-support-es'}) { die "Your CPU does not support AMD SEV-ES.\n"; } if ($amd_sev_conf->{type} eq 'snp' && !$sev_hw_caps->{'sev-support-snp'}) { die "Your CPU does not support AMD SEV-SNP.\n"; } if (!$bios || $bios ne 'ovmf') { die "To use AMD SEV, you need to change the BIOS to OVMF.\n"; } my $sev_mem_object = ''; my $policy; if ($amd_sev_conf->{type} eq 'es' || $amd_sev_conf->{type} eq 'std') { $sev_mem_object .= 'sev-guest,id=sev0'; $sev_mem_object .= ',cbitpos=' . $sev_hw_caps->{cbitpos}; $sev_mem_object .= ',reduced-phys-bits=' . $sev_hw_caps->{'reduced-phys-bits'}; # guest policy bit calculation as described here: # https://documentation.suse.com/sles/15-SP5/html/SLES-amd-sev/article-amd-sev.html#table-guestpolicy $policy = 0; $policy |= 1 << 0 if $amd_sev_conf->{'no-debug'}; $policy |= 1 << 1 if $amd_sev_conf->{'no-key-sharing'}; $policy |= 1 << 2 if $amd_sev_conf->{type} eq 'es'; # disable migration with bit 3 nosend to prevent amd-sev-migration-attack $policy |= 1 << 3; } elsif ($amd_sev_conf->{type} eq 'snp') { $sev_mem_object .= 'sev-snp-guest,id=sev0'; $sev_mem_object .= ',cbitpos=' . $sev_hw_caps->{cbitpos}; $sev_mem_object .= ',reduced-phys-bits=' . $sev_hw_caps->{'reduced-phys-bits'}; # guest policy bit calculation as described in chapter 4.3: # https://www.amd.com/system/files/TechDocs/56860.pdf # Reserved bit must be one $policy = 1 << 17; $policy |= 1 << 16 if !defined($amd_sev_conf->{'allow-smt'}) || $amd_sev_conf->{'allow-smt'}; $policy |= 1 << 19 if !$amd_sev_conf->{'no-debug'}; } $sev_mem_object .= ',policy=' . sprintf("%#x", $policy); $sev_mem_object .= ',kernel-hashes=on' if ($amd_sev_conf->{'kernel-hashes'}); return $sev_mem_object; } sub get_quote_generation_socket { my ($conf) = @_; my $socket = { type => 'vsock' }; die "Missing cid for vsock.\n" if !defined($conf->{'vsock-cid'}); die "Missing port for vsock.\n" if !defined($conf->{'vsock-port'}); # Both are strings in the QMP schema $socket->{'cid'} = "$conf->{'vsock-cid'}"; $socket->{'port'} = "$conf->{'vsock-port'}"; return $socket; } sub get_intel_tdx_object { my ($intel_tdx, $bios) = @_; my $intel_tdx_conf = PVE::JSONSchema::parse_property_string($tdx_fmt, $intel_tdx); my $tdx_hw_caps = get_hw_capabilities()->{'intel-tdx'}; if (!$tdx_hw_caps->{'tdx-support'}) { die "Your CPU does not support Intel TDX.\n"; } if (!$bios || $bios ne 'ovmf') { die "To use Intel TDX, you need to change the BIOS to OVMF.\n"; } my $tdx_object = { 'qom-type' => 'tdx-guest', id => 'tdx0', }; $tdx_object->{'quote-generation-socket'} = get_quote_generation_socket($intel_tdx_conf) if $intel_tdx_conf->{'attestation'}; return $tdx_object; } __PACKAGE__->register(); __PACKAGE__->init(); 1; ================================================ FILE: src/PVE/QemuServer/Cfg2Cmd/Makefile ================================================ DESTDIR= PREFIX=/usr PERLDIR=$(PREFIX)/share/perl5 SOURCES=Timer.pm .PHONY: install install: $(SOURCES) for i in $(SOURCES); do install -D -m 0644 $$i $(DESTDIR)$(PERLDIR)/PVE/QemuServer/Cfg2Cmd/$$i; done ================================================ FILE: src/PVE/QemuServer/Cfg2Cmd/Timer.pm ================================================ package PVE::QemuServer::Cfg2Cmd::Timer; use warnings; use strict; sub generate { my ($cfg2cmd) = @_; my $time_drift_fix = $cfg2cmd->get_prop('tdf', 1); my $acpi = $cfg2cmd->get_prop('acpi'); my $localtime = $cfg2cmd->get_prop('localtime', 1); my $startdate = $cfg2cmd->get_prop('startdate'); if ($cfg2cmd->windows_version() >= 5) { # windows $localtime = 1 if !defined($localtime); # use time drift fix when acpi is enabled, but prefer explicitly set value $time_drift_fix = 1 if $acpi && !defined($time_drift_fix); } if ($cfg2cmd->windows_version() >= 6) { $cfg2cmd->add_global_flag('kvm-pit.lost_tick_policy=discard'); $cfg2cmd->add_machine_flag_if_supported('hpet', 'off'); } elsif ($cfg2cmd->is_linux() && $cfg2cmd->version_guard(10, 1, 0)) { $cfg2cmd->add_machine_flag_if_supported('hpet', 'off'); } $cfg2cmd->add_rtc_flag('driftfix=slew') if $time_drift_fix; if ($startdate ne 'now') { $cfg2cmd->add_rtc_flag("base=$startdate"); } elsif ($localtime) { $cfg2cmd->add_rtc_flag('base=localtime'); } return; } 1; ================================================ FILE: src/PVE/QemuServer/Cfg2Cmd.pm ================================================ package PVE::QemuServer::Cfg2Cmd; use warnings; use strict; use PVE::QemuServer::Cfg2Cmd::Timer; use PVE::QemuServer::Helpers; use PVE::QemuServer::Machine; sub new { my ($class, $conf, $defaults, $version_guard, $opts) = @_; my $self = bless { conf => $conf, defaults => $defaults, 'version-guard' => $version_guard, }, $class; $self->{ostype} = $self->get_prop('ostype'); $self->{'windows-version'} = PVE::QemuServer::Helpers::windows_version($self->{ostype}); $self->{'machine-type'} = PVE::QemuServer::Machine::get_vm_machine($conf, $opts->{forcemachine}); return $self; } =head3 get_prop my $value = $self->get_prop($prop); Return the configured value for the property C<$prop>. If no fallback to the default value should be made, use C<$only_explicit>. Note that any such usage is likely an indication that the default value is not actually a static default, but that the default depends on context. =cut sub get_prop { my ($self, $prop, $only_explicit) = @_; my ($conf, $defaults) = $self->@{qw(conf defaults)}; return $conf->{$prop} if $only_explicit; return defined($conf->{$prop}) ? $conf->{$prop} : $defaults->{$prop}; } sub add_global_flag { my ($self, $flag) = @_; push $self->{'global-flags'}->@*, $flag; } sub global_flags { my ($self) = @_; return $self->{'global-flags'}; } =head3 add_machine_flag_if_supported my $success = $self->add_machine_flag_if_supported($flag_name, $value); Add flag C<$flag_name> with value C<$value> to the machine flags if the current machine type supports it. Returns whether the flag was added or not. =cut sub add_machine_flag_if_supported { my ($self, $flag_name, $value) = @_; return if !PVE::QemuServer::Machine::machine_supports_flag($self->{'machine-type'}, $flag_name); push $self->{'machine-flags'}->@*, "${flag_name}=${value}"; return 1; } sub machine_flags { my ($self) = @_; return $self->{'machine-flags'}; } sub add_rtc_flag { my ($self, $flag) = @_; push $self->{'rtc-flags'}->@*, $flag; } sub rtc_flags { my ($self) = @_; return $self->{'rtc-flags'}; } =head3 is_linux if ($self->is_linux()) { do_something_for_linux_vms(); } Check if the virtual machine is configured for running Linux. Does not include the C os type by default. Specify C<$include_l24> if that is desired. =cut sub is_linux { my ($self, $include_l24) = @_; return $self->{ostype} eq 'l26' || ($include_l24 && $self->{ostype} eq 'l24'); } sub windows_version { my ($self) = @_; return $self->{'windows-version'}; } sub version_guard { my ($self, $major, $minor, $pve) = @_; $self->{'version-guard'}->($major, $minor, $pve); } sub generate { my ($self) = @_; PVE::QemuServer::Cfg2Cmd::Timer::generate($self); return $self; } 1; ================================================ FILE: src/PVE/QemuServer/Cloudinit.pm ================================================ package PVE::QemuServer::Cloudinit; use strict; use warnings; use File::Path; use Digest::SHA; use URI::Escape; use MIME::Base64 qw(encode_base64); use Storable qw(dclone); use JSON; use PVE::Tools qw(run_command file_set_contents); use PVE::Storage; use PVE::QemuServer::Drive qw(checked_volume_format); use PVE::QemuServer::Helpers; use PVE::QemuServer::Network; use constant CLOUDINIT_DISK_SIZE => 4 * 1024 * 1024; # 4MiB in bytes sub commit_cloudinit_disk { my ($conf, $vmid, $drive, $volname, $storeid, $files, $label) = @_; my $path = "/run/pve/cloudinit/$vmid/"; mkpath $path; foreach my $filepath (keys %$files) { if ($filepath !~ m@^(.*)\/[^/]+$@) { die "internal error: bad file name in cloud-init image: $filepath\n"; } my $dirname = $1; mkpath "$path/$dirname"; my $contents = $files->{$filepath}; file_set_contents("$path/$filepath", $contents); } my $storecfg = PVE::Storage::config(); my $iso_path = PVE::Storage::path($storecfg, $drive->{file}); my $scfg = PVE::Storage::storage_config($storecfg, $storeid); my $format = checked_volume_format($storecfg, $drive->{file}); my $size = eval { PVE::Storage::volume_size_info($storecfg, $drive->{file}) }; if (!defined($size) || $size <= 0) { $volname =~ m/(vm-$vmid-cloudinit(.\Q$format\E)?)/; my $name = $1; $size = 4 * 1024; PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, $name, $size); $size *= 1024; # vdisk alloc takes KB, qemu-img dd's osize takes byte } my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); $plugin->activate_volume($storeid, $scfg, $volname); print "generating cloud-init ISO\n"; eval { run_command([ ['genisoimage', '-quiet', '-iso-level', '3', '-J', '-R', '-V', $label, $path], [ 'qemu-img', 'dd', '-n', '-f', 'raw', '-O', $format, 'isize=0', "osize=$size", "of=$iso_path", ], ]); }; my $err = $@; rmtree($path); die $err if $err; } sub get_cloudinit_format { my ($conf) = @_; if (defined(my $format = $conf->{citype})) { return $format; } # No format specified, default based on ostype because windows' # cloudbased-init only supports configdrivev2, whereas on linux we need # to use mac addresses because regular cloudinit doesn't map 'ethX' to # the new predictable network device naming scheme. if (defined(my $ostype = $conf->{ostype})) { return 'configdrive2' if PVE::QemuServer::Helpers::windows_version($ostype); } return 'nocloud'; } sub get_hostname_fqdn { my ($conf, $vmid) = @_; my $hostname = $conf->{name} // "VM$vmid"; my $fqdn; if ($hostname =~ /\./) { $fqdn = $hostname; $hostname =~ s/\..*$//; } elsif (my $search = $conf->{searchdomain}) { $fqdn = "$hostname.$search"; } else { $fqdn = $hostname; } return ($hostname, $fqdn); } sub get_dns_conf { my ($conf) = @_; # Same logic as in pve-container, but without the testcase special case my $host_resolv_conf = PVE::INotify::read_file('resolvconf'); my $searchdomains = [split(/\s+/, $conf->{searchdomain} // $host_resolv_conf->{search} // '')]; my $nameserver = $conf->{nameserver}; if (!defined($nameserver)) { $nameserver = [grep { $_ } $host_resolv_conf->@{qw(dns1 dns2 dns3)}]; } else { $nameserver = [split(/\s+/, $nameserver)]; } return ($searchdomains, $nameserver); } sub cloudinit_userdata { my ($conf, $vmid, $mask_password) = @_; my ($hostname, $fqdn) = get_hostname_fqdn($conf, $vmid); my $content = "#cloud-config\n"; $content .= "hostname: $hostname\n"; $content .= "manage_etc_hosts: true\n"; $content .= "fqdn: $fqdn\n"; my $username = $conf->{ciuser}; my $password = $conf->{cipassword}; $content .= "user: $username\n" if defined($username); $content .= "disable_root: False\n" if defined($username) && $username eq 'root'; if (defined($password)) { if ($mask_password) { $content .= "password: **********\n"; } else { $content .= "password: $password\n"; } } if (defined(my $keys = $conf->{sshkeys})) { $keys = URI::Escape::uri_unescape($keys); $keys = [map { my $key = $_; chomp $key; $key } split(/\n/, $keys)]; $keys = [grep { /\S/ } @$keys]; $content .= "ssh_authorized_keys:\n"; foreach my $k (@$keys) { $content .= " - $k\n"; } } $content .= "chpasswd:\n"; $content .= " expire: False\n"; if (!defined($username) || $username ne 'root') { $content .= "users:\n"; $content .= " - default\n"; } $content .= "package_upgrade: true\n" if !defined($conf->{ciupgrade}) || $conf->{ciupgrade}; return $content; } sub split_ip4 { my ($ip) = @_; my ($addr, $mask) = split('/', $ip); die "not a CIDR: $ip\n" if !defined $mask; return ($addr, $PVE::Network::ipv4_reverse_mask->[$mask]); } sub configdrive2_network { my ($conf) = @_; my $content = "auto lo\n"; $content .= "iface lo inet loopback\n\n"; my ($searchdomains, $nameservers) = get_dns_conf($conf); if ($nameservers && @$nameservers) { $nameservers = join(' ', @$nameservers); $content .= " dns_nameservers $nameservers\n"; } if ($searchdomains && @$searchdomains) { $searchdomains = join(' ', @$searchdomains); $content .= " dns_search $searchdomains\n"; } my @ifaces = grep { /^net(\d+)$/ } keys %$conf; foreach my $iface (sort @ifaces) { (my $id = $iface) =~ s/^net//; next if !$conf->{"ipconfig$id"}; my $net = PVE::QemuServer::Network::parse_ipconfig($conf->{"ipconfig$id"}); $id = "eth$id"; $content .= "auto $id\n"; if ($net->{ip}) { if ($net->{ip} eq 'dhcp') { $content .= "iface $id inet dhcp\n"; } else { my ($addr, $mask) = split_ip4($net->{ip}); $content .= "iface $id inet static\n"; $content .= " address $addr\n"; $content .= " netmask $mask\n"; $content .= " gateway $net->{gw}\n" if $net->{gw}; } } if ($net->{ip6}) { if ($net->{ip6} =~ /^(auto|dhcp)$/) { $content .= "iface $id inet6 $1\n"; } else { my ($addr, $mask) = split('/', $net->{ip6}); $content .= "iface $id inet6 static\n"; $content .= " address $addr\n"; $content .= " netmask $mask\n"; $content .= " gateway $net->{gw6}\n" if $net->{gw6}; } } } return $content; } sub configdrive2_gen_metadata { my ($user, $network) = @_; my $uuid_str = Digest::SHA::sha1_hex($user . $network); return configdrive2_metadata($uuid_str); } sub configdrive2_metadata { my ($uuid) = @_; return <<"EOF"; { "uuid": "$uuid", "network_config": { "content_path": "/content/0000" } } EOF } sub generate_configdrive2 { my ($conf, $vmid, $drive, $volname, $storeid) = @_; my ($user_data, $network_data, $meta_data, $vendor_data) = get_custom_cloudinit_files($conf); if (PVE::QemuServer::Helpers::windows_version($conf->{ostype})) { $user_data = cloudinit_userdata($conf, $vmid) if !defined($user_data); $network_data = cloudbase_network_eni($conf) if !defined($network_data); $vendor_data = '' if !defined($vendor_data); if (!defined($meta_data)) { my $instance_id = cloudbase_gen_instance_id($user_data, $network_data); $meta_data = cloudbase_configdrive2_metadata($instance_id, $conf); } } else { $user_data = cloudinit_userdata($conf, $vmid) if !defined($user_data); $network_data = configdrive2_network($conf) if !defined($network_data); $vendor_data = '' if !defined($vendor_data); if (!defined($meta_data)) { $meta_data = configdrive2_gen_metadata($user_data, $network_data); } } # we always allocate a 4MiB disk for cloudinit and with the overhead of the ISO # make sure we always stay below it by keeping the sum of all files below 3 MiB my $sum = length($user_data) + length($network_data) + length($meta_data) + length($vendor_data); die "Cloud-Init sum of snippets too big (> 3 MiB)\n" if $sum > (3 * 1024 * 1024); my $files = { '/openstack/latest/user_data' => $user_data, '/openstack/content/0000' => $network_data, '/openstack/latest/meta_data.json' => $meta_data, '/openstack/latest/vendor_data.json' => $vendor_data, }; commit_cloudinit_disk($conf, $vmid, $drive, $volname, $storeid, $files, 'config-2'); } sub cloudbase_network_eni { my ($conf) = @_; my $content = ""; my ($searchdomains, $nameservers) = get_dns_conf($conf); if ($nameservers && @$nameservers) { $nameservers = join(' ', @$nameservers); } my @ifaces = grep { /^net(\d+)$/ } keys %$conf; foreach my $iface (sort @ifaces) { (my $id = $iface) =~ s/^net//; next if !$conf->{"ipconfig$id"}; my $net = PVE::QemuServer::Network::parse_ipconfig($conf->{"ipconfig$id"}); $id = "eth$id"; $content .= "auto $id\n"; if ($net->{ip}) { if ($net->{ip} eq 'dhcp') { $content .= "iface $id inet dhcp\n"; } else { my ($addr, $mask) = split_ip4($net->{ip}); $content .= "iface $id inet static\n"; $content .= " address $addr\n"; $content .= " netmask $mask\n"; $content .= " gateway $net->{gw}\n" if $net->{gw}; $content .= " dns-nameservers $nameservers\n" if $nameservers; } } if ($net->{ip6}) { if ($net->{ip6} =~ /^(auto|dhcp)$/) { $content .= "iface $id inet6 $1\n"; } else { my ($addr, $mask) = split('/', $net->{ip6}); $content .= "iface $id inet6 static\n"; $content .= " address $addr\n"; $content .= " netmask $mask\n"; $content .= " gateway $net->{gw6}\n" if $net->{gw6}; $content .= " dns-nameservers $nameservers\n" if $nameservers; } } } return $content; } sub cloudbase_configdrive2_metadata { my ($uuid, $conf) = @_; my $meta_data = { uuid => $uuid, 'network_config' => { 'content_path' => '/content/0000', }, }; $meta_data->{'admin_pass'} = $conf->{cipassword} if $conf->{cipassword}; if (defined(my $keys = $conf->{sshkeys})) { $keys = URI::Escape::uri_unescape($keys); $keys = [map { my $key = $_; chomp $key; $key } split(/\n/, $keys)]; $keys = [grep { /\S/ } @$keys]; my $i = 0; foreach my $k (@$keys) { $meta_data->{'public_keys'}->{"key-$i"} = $k; $i++; } } my $json = encode_json($meta_data); return $json; } sub cloudbase_gen_instance_id { my ($user, $network) = @_; my $uuid_str = Digest::SHA::sha1_hex($user . $network); return $uuid_str; } sub generate_opennebula { my ($conf, $vmid, $drive, $volname, $storeid) = @_; my $content = ""; my $username = $conf->{ciuser} || "root"; $content .= "USERNAME=$username\n" if defined($username); if (defined(my $password = $conf->{cipassword})) { $content .= "CRYPTED_PASSWORD_BASE64=" . encode_base64($password) . "\n"; } if (defined($conf->{sshkeys})) { my $keys = [split(/\s*\n\s*/, URI::Escape::uri_unescape($conf->{sshkeys}))]; $content .= "SSH_PUBLIC_KEY=\"" . join("\n", $keys->@*) . "\"\n"; } my ($hostname, $fqdn) = get_hostname_fqdn($conf, $vmid); $content .= "SET_HOSTNAME=$hostname\n"; my ($searchdomains, $nameservers) = get_dns_conf($conf); $content .= 'DNS="' . join(' ', @$nameservers) . "\"\n" if $nameservers && @$nameservers; $content .= 'SEARCH_DOMAIN="' . join(' ', @$searchdomains) . "\"\n" if $searchdomains && @$searchdomains; my $networkenabled = undef; my @ifaces = grep { /^net(\d+)$/ } keys %$conf; foreach my $iface (sort @ifaces) { (my $id = $iface) =~ s/^net//; my $net = PVE::QemuServer::Network::parse_net($conf->{$iface}); next if !$conf->{"ipconfig$id"}; my $ipconfig = PVE::QemuServer::Network::parse_ipconfig($conf->{"ipconfig$id"}); my $ethid = "ETH$id"; my $mac = lc $net->{hwaddr}; if ($ipconfig->{ip}) { $networkenabled = 1; if ($ipconfig->{ip} eq 'dhcp') { $content .= "${ethid}_DHCP=YES\n"; } else { my ($addr, $mask) = split_ip4($ipconfig->{ip}); $content .= "${ethid}_IP=$addr\n"; $content .= "${ethid}_MASK=$mask\n"; $content .= "${ethid}_MAC=$mac\n"; $content .= "${ethid}_GATEWAY=$ipconfig->{gw}\n" if $ipconfig->{gw}; } $content .= "${ethid}_MTU=$net->{mtu}\n" if $net->{mtu}; } if ($ipconfig->{ip6}) { $networkenabled = 1; if ($ipconfig->{ip6} eq 'dhcp') { $content .= "${ethid}_DHCP6=YES\n"; } elsif ($ipconfig->{ip6} eq 'auto') { $content .= "${ethid}_AUTO6=YES\n"; } else { my ($addr, $mask) = split('/', $ipconfig->{ip6}); $content .= "${ethid}_IP6=$addr\n"; $content .= "${ethid}_MASK6=$mask\n"; $content .= "${ethid}_MAC6=$mac\n"; $content .= "${ethid}_GATEWAY6=$ipconfig->{gw6}\n" if $ipconfig->{gw6}; } $content .= "${ethid}_MTU=$net->{mtu}\n" if $net->{mtu}; } } $content .= "NETWORK=YES\n" if $networkenabled; my $files = { '/context.sh' => $content }; commit_cloudinit_disk($conf, $vmid, $drive, $volname, $storeid, $files, 'CONTEXT'); } sub nocloud_network_v2 { my ($conf) = @_; my $content = ''; my $head = "version: 2\n" . "ethernets:\n"; my $dns_done; my @ifaces = grep { /^net(\d+)$/ } keys %$conf; foreach my $iface (sort @ifaces) { (my $id = $iface) =~ s/^net//; next if !$conf->{"ipconfig$id"}; # indentation - network interfaces are inside an 'ethernets' hash my $i = ' '; my $net = PVE::QemuServer::Network::parse_net($conf->{$iface}); my $ipconfig = PVE::QemuServer::Network::parse_ipconfig($conf->{"ipconfig$id"}); my $mac = $net->{macaddr} or die "network interface '$iface' has no mac address\n"; $content .= "${i}$iface:\n"; $i .= ' '; $content .= "${i}match:\n" . "${i} macaddress: \"$mac\"\n" . "${i}set-name: eth$id\n"; my @addresses; if (defined(my $ip = $ipconfig->{ip})) { if ($ip eq 'dhcp') { $content .= "${i}dhcp4: true\n"; } else { push @addresses, $ip; } } if (defined(my $ip = $ipconfig->{ip6})) { if ($ip eq 'dhcp') { $content .= "${i}dhcp6: true\n"; } else { push @addresses, $ip; } } if (@addresses) { $content .= "${i}addresses:\n"; $content .= "${i}- '$_'\n" foreach @addresses; } if (defined(my $gw = $ipconfig->{gw})) { $content .= "${i}gateway4: '$gw'\n"; } if (defined(my $gw = $ipconfig->{gw6})) { $content .= "${i}gateway6: '$gw'\n"; } next if $dns_done; $dns_done = 1; my ($searchdomains, $nameservers) = get_dns_conf($conf); if ($searchdomains || $nameservers) { $content .= "${i}nameservers:\n"; if (defined($nameservers) && @$nameservers) { $content .= "${i} addresses:\n"; $content .= "${i} - '$_'\n" foreach @$nameservers; } if (defined($searchdomains) && @$searchdomains) { $content .= "${i} search:\n"; $content .= "${i} - '$_'\n" foreach @$searchdomains; } } } return $head . $content; } sub nocloud_network { my ($conf) = @_; my $content = "version: 1\n" . "config:\n"; my @ifaces = grep { /^net(\d+)$/ } keys %$conf; foreach my $iface (sort @ifaces) { (my $id = $iface) =~ s/^net//; next if !$conf->{"ipconfig$id"}; # indentation - network interfaces are inside an 'ethernets' hash my $i = ' '; my $net = PVE::QemuServer::Network::parse_net($conf->{$iface}); my $ipconfig = PVE::QemuServer::Network::parse_ipconfig($conf->{"ipconfig$id"}); my $mac = lc($net->{macaddr}) or die "network interface '$iface' has no mac address\n"; $content .= "${i}- type: physical\n" . "${i} name: eth$id\n" . "${i} mac_address: '$mac'\n" . "${i} subnets:\n"; $i .= ' '; if (defined(my $ip = $ipconfig->{ip})) { if ($ip eq 'dhcp') { $content .= "${i}- type: dhcp4\n"; } else { my ($addr, $mask) = split_ip4($ip); $content .= "${i}- type: static\n" . "${i} address: '$addr'\n" . "${i} netmask: '$mask'\n"; if (defined(my $gw = $ipconfig->{gw})) { $content .= "${i} gateway: '$gw'\n"; } } } if (defined(my $ip = $ipconfig->{ip6})) { if ($ip eq 'dhcp') { $content .= "${i}- type: dhcp6\n"; } elsif ($ip eq 'auto') { # SLAAC is only supported by cloud-init since 19.4 $content .= "${i}- type: ipv6_slaac\n"; } else { $content .= "${i}- type: static6\n" . "${i} address: '$ip'\n"; if (defined(my $gw = $ipconfig->{gw6})) { $content .= "${i} gateway: '$gw'\n"; } } } } my $i = ' '; my ($searchdomains, $nameservers) = get_dns_conf($conf); if ($searchdomains || $nameservers) { $content .= "${i}- type: nameserver\n"; if (defined($nameservers) && @$nameservers) { $content .= "${i} address:\n"; $content .= "${i} - '$_'\n" foreach @$nameservers; } if (defined($searchdomains) && @$searchdomains) { $content .= "${i} search:\n"; $content .= "${i} - '$_'\n" foreach @$searchdomains; } } return $content; } sub nocloud_metadata { my ($uuid) = @_; return "instance-id: $uuid\n"; } sub nocloud_gen_metadata { my ($user, $network) = @_; my $uuid_str = Digest::SHA::sha1_hex($user . $network); return nocloud_metadata($uuid_str); } sub generate_nocloud { my ($conf, $vmid, $drive, $volname, $storeid) = @_; my ($user_data, $network_data, $meta_data, $vendor_data) = get_custom_cloudinit_files($conf); $user_data = cloudinit_userdata($conf, $vmid) if !defined($user_data); $network_data = nocloud_network($conf) if !defined($network_data); $vendor_data = '' if !defined($vendor_data); if (!defined($meta_data)) { $meta_data = nocloud_gen_metadata($user_data, $network_data); } # we always allocate a 4MiB disk for cloudinit and with the overhead of the ISO # make sure we always stay below it by keeping the sum of all files below 3 MiB my $sum = length($user_data) + length($network_data) + length($meta_data) + length($vendor_data); die "Cloud-Init sum of snippets too big (> 3 MiB)\n" if $sum > (3 * 1024 * 1024); my $files = { '/user-data' => $user_data, '/network-config' => $network_data, '/meta-data' => $meta_data, '/vendor-data' => $vendor_data, }; commit_cloudinit_disk($conf, $vmid, $drive, $volname, $storeid, $files, 'cidata'); } sub get_custom_cloudinit_files { my ($conf) = @_; my $cicustom = $conf->{cicustom}; my $files = $cicustom ? PVE::JSONSchema::parse_property_string('pve-qm-cicustom', $cicustom) : {}; my $network_volid = $files->{network}; my $user_volid = $files->{user}; my $meta_volid = $files->{meta}; my $vendor_volid = $files->{vendor}; my $storage_conf = PVE::Storage::config(); my $network_data; if ($network_volid) { $network_data = read_cloudinit_snippets_file($storage_conf, $network_volid); } my $user_data; if ($user_volid) { $user_data = read_cloudinit_snippets_file($storage_conf, $user_volid); } my $meta_data; if ($meta_volid) { $meta_data = read_cloudinit_snippets_file($storage_conf, $meta_volid); } my $vendor_data; if ($vendor_volid) { $vendor_data = read_cloudinit_snippets_file($storage_conf, $vendor_volid); } return ($user_data, $network_data, $meta_data, $vendor_data); } sub read_cloudinit_snippets_file { my ($storage_conf, $volid) = @_; my ($vtype, undef) = PVE::Storage::parse_volname($storage_conf, $volid); die "$volid is not in the snippets directory\n" if $vtype ne 'snippets'; my $full_path = PVE::Storage::abs_filesystem_path($storage_conf, $volid, 1); return PVE::Tools::file_get_contents($full_path, 1 * 1024 * 1024); } my $cloudinit_methods = { configdrive2 => \&generate_configdrive2, nocloud => \&generate_nocloud, opennebula => \&generate_opennebula, }; sub has_changes { my ($conf) = @_; if (my $cloudinit = $conf->{'special-sections'}->{cloudinit}) { return !!$cloudinit->%*; } return; } sub generate_cloudinit_config { my ($conf, $vmid) = @_; my $format = get_cloudinit_format($conf); my $has_changes = has_changes($conf); PVE::QemuConfig->foreach_volume( $conf, sub { my ($ds, $drive) = @_; my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1); return if !$volname || $volname !~ m/vm-$vmid-cloudinit/; my $generator = $cloudinit_methods->{$format} or die "missing cloudinit methods for format '$format'\n"; $generator->($conf, $vmid, $drive, $volname, $storeid); }, ); return $has_changes; } sub apply_cloudinit_config { my ($conf, $vmid) = @_; my $has_changes = generate_cloudinit_config($conf, $vmid); if ($has_changes) { delete $conf->{'special-sections'}->{cloudinit}; PVE::QemuConfig->write_config($vmid, $conf); return 1; } return $has_changes; } sub dump_cloudinit_config { my ($conf, $vmid, $type, $mask_password) = @_; my $format = get_cloudinit_format($conf); if ($type eq 'user') { return cloudinit_userdata($conf, $vmid, $mask_password); } elsif ($type eq 'network') { if ($format eq 'nocloud') { return nocloud_network($conf); } else { return configdrive2_network($conf); } } else { # metadata config # Don't mask password here to get correct uuid. my $user = cloudinit_userdata($conf, $vmid); if ($format eq 'nocloud') { my $network = nocloud_network($conf); return nocloud_gen_metadata($user, $network); } else { my $network = configdrive2_network($conf); return configdrive2_gen_metadata($user, $network); } } } 1; ================================================ FILE: src/PVE/QemuServer/DBusVMState.pm ================================================ package PVE::QemuServer::DBusVMState; use strict; use warnings; use Net::DBus; use Net::DBus::RemoteService; use PVE::SafeSyslog; use PVE::Systemd; use PVE::Tools; use PVE::QemuServer::Helpers; use constant { DBUS_VMSTATE_EXE => '/usr/libexec/qemu-server/dbus-vmstate', }; # Call a method for an object from a specific interface name. # In contrast to calling the method directly by using $obj->Method(), this # actually respects the owner of the object and thus can be used for interfaces # with might have multiple (queued) owners on the DBus. my sub dbus_call_method { my ($obj, $interface, $method, $params, $timeout) = @_; $timeout = 10 if !$timeout; my $con = $obj->{service}->get_bus()->get_connection(); my $call = $con->make_method_call_message( $obj->{service}->get_service_name(), $obj->{object_path}, $interface, $method, ); $call->set_destination($obj->get_service()->get_owner_name()); $call->append_args_list($params->@*) if $params; return $con->send_with_reply_and_block($call, $timeout * 1000)->get_args_list(); } # Retrieves a property from an object from a specific interface name. # In contrast to accessing the property directly by using $obj->Property, this # actually respects the owner of the object and thus can be used for interfaces # with might have multiple (queued) owners on the DBus. my sub dbus_get_property { my ($obj, $interface, $name) = @_; my @reply = dbus_call_method($obj, 'org.freedesktop.DBus.Properties', 'Get', [$interface, $name]); return $reply[0]; } # Starts the dbus-vmstate helper D-Bus service daemon and adds the needed # object to the appropriate QEMU instance for the specified VM. sub qemu_add_dbus_vmstate { my ($vmid) = @_; if (!PVE::QemuServer::Helpers::vm_running_locally($vmid)) { die "VM $vmid must be running locally\n"; } # In case some leftover, previous instance is running, stop it. Otherwise # we run into errors, as a systemd service instance is unique. if (defined(qemu_del_dbus_vmstate($vmid, quiet => 1))) { warn "stopped previously running dbus-vmstate helper for VM $vmid\n"; } # Start the actual service, which will then register itself with QEMU. eval { PVE::Tools::run_command(['systemctl', 'start', "pve-dbus-vmstate\@$vmid"]) }; if (my $err = $@) { die "failed to start DBus VMState service for VM $vmid: $err\n"; } } # Stops the dbus-vmstate helper D-Bus service daemon and removes the associated # object from QEMU for the specified VM. # # Returns the number of migrated conntrack entries, or undef in case of error. sub qemu_del_dbus_vmstate { my ($vmid, %params) = @_; my $num_entries = undef; my $dbus = eval { Net::DBus->system(); }; if (my $err = $@) { # log fundamental error even if $params{quiet} is set syslog('warn', "failed to connect to DBus system bus: $err"); return undef; } my $dbus_obj = eval { $dbus->get_bus_object(); }; if (my $err = $@) { # log fundamental error even if $params{quiet} is set syslog('warn', "failed to get DBus bus object: $err"); return undef; } my $owners = eval { $dbus_obj->ListQueuedOwners('org.qemu.VMState1') }; if (my $err = $@) { syslog('warn', "failed to retrieve org.qemu.VMState1 owners: $err\n") if !$params{quiet}; return undef; } # Iterate through all name owners for 'org.qemu.VMState1' and compare # the ID. If we found the corresponding one for $vmid, retrieve the # `NumMigratedEntries` property and call the `Quit()` method on it. # Any D-Bus interaction might die/croak, so try to be careful here and # swallow any hard errors. foreach my $owner (@$owners) { my $service = eval { Net::DBus::RemoteService->new($dbus, $owner, 'org.qemu.VMState1') }; if (my $err = $@) { syslog('warn', "failed to get org.qemu.VMState1 service from D-Bus $owner: $err\n") if !$params{quiet}; next; } my $object = eval { $service->get_object('/org/qemu/VMState1') }; if (my $err = $@) { syslog('warn', "failed to get /org/qemu/VMState1 object from D-Bus $owner: $err\n") if !$params{quiet}; next; } my $id = eval { dbus_get_property($object, 'org.qemu.VMState1', 'Id') }; if (defined($id) && $id eq "pve-vmstate-$vmid") { my $helperobj = eval { $service->get_object('/org/qemu/VMState1', 'com.proxmox.VMStateHelper') }; if (my $err = $@) { syslog( 'warn', "found dbus-vmstate helper, but does not implement com.proxmox.VMStateHelper? ($err)\n", ) if !$params{quiet}; last; } $num_entries = eval { dbus_get_property($object, 'com.proxmox.VMStateHelper', 'NumMigratedEntries'); }; # Quit() does QMP object-del which has a timeout of 60 seconds eval { dbus_call_method($object, 'com.proxmox.VMStateHelper', 'Quit', [], 70); }; if (my $err = $@) { syslog('warn', "failed to call quit on dbus-vmstate for VM $vmid: $err\n") if !$params{quiet}; } last; } } return $num_entries; } 1; ================================================ FILE: src/PVE/QemuServer/Drive.pm ================================================ package PVE::QemuServer::Drive; use strict; use warnings; use Storable qw(dclone); use IO::File; use List::Util qw(first); use PVE::RESTEnvironment qw(log_warn); use PVE::Storage; use PVE::Storage::Common; use PVE::JSONSchema qw(get_standard_option); use PVE::QemuServer::Monitor qw(qsd_qmp_peer vm_qmp_peer); use base qw(Exporter); our @EXPORT_OK = qw( is_valid_drivename checked_parse_volname checked_volume_format drive_is_cloudinit drive_is_cdrom parse_drive print_drive storage_allows_io_uring_default ); my $DROPPED_PROPERTIES = ['cyls', 'heads', 'secs', 'trans']; our $QEMU_FORMAT_RE = qr/raw|qcow|qcow2|qed|vmdk|cloop/; PVE::JSONSchema::register_standard_option( 'pve-qm-image-format', { type => 'string', enum => [qw(raw qcow qed qcow2 vmdk cloop)], description => "The drive's backing file's data format.", optional => 1, }, ); # Check that a volume can be used for image-related operations with QEMU, in # particular, attached as VM image or ISO, used for qemu-img, or (live-)imported. # NOTE Currently, this helper cannot be used for backups. # TODO allow configuring certain restrictions via $opts argument, e.g. expected vtype? sub checked_parse_volname { my ($storecfg, $volid) = @_; my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = PVE::Storage::parse_volname($storecfg, $volid); if ($vtype eq 'import') { die "unable to parse format for import volume '$volid'\n" if !$format; if ($format =~ m/^ova\+(.*)$/) { my $extracted_format = $1; die "volume '$volid' - unknown import format '$format'\n" if $extracted_format !~ m/^($QEMU_FORMAT_RE)$/; return ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format); } } # TODO PVE 9 - consider switching to die for an undefined format $format = 'raw' if !defined($format); die "volume '$volid' - not a QEMU image format '$format'\n" if $format !~ m/^($QEMU_FORMAT_RE)$/; # For iso content type, no format is returned yet. return ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format); } sub checked_volume_format { my ($storecfg, $volid) = @_; return (checked_parse_volname($storecfg, $volid))[6]; } my $cdrom_path; sub get_cdrom_path { return $cdrom_path if defined($cdrom_path); $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2'); if (!defined($cdrom_path)) { log_warn("no physical CD-ROM available, ignoring"); $cdrom_path = ''; } return $cdrom_path; } sub get_iso_path { my ($storecfg, $cdrom) = @_; if ($cdrom eq 'cdrom') { return get_cdrom_path(); } elsif ($cdrom eq 'none') { return ''; } elsif ($cdrom =~ m|^/|) { return $cdrom; } else { return PVE::Storage::path($storecfg, $cdrom); } } # Returns the path that can be used on the QEMU commandline and in QMP commands as well as the # checked format of the drive. sub get_path_and_format { my ($storecfg, $drive, $live_restore_name) = @_; my $path; my $volid = $drive->{file}; my $drive_id = get_drive_id($drive); my ($storeid) = PVE::Storage::parse_volume_id($volid, 1); if (drive_is_cdrom($drive)) { $path = get_iso_path($storecfg, $volid); die "$drive_id: cannot back cdrom drive with a live restore image\n" if $live_restore_name; } else { if ($storeid) { $path = PVE::Storage::path($storecfg, $volid); } else { $path = $volid; } } # For PVE-managed volumes, use the format from the storage layer and prevent overrides via the # drive's 'format' option. For unmanaged volumes, fallback to 'raw' to avoid auto-detection by # QEMU. For the special case 'none' (get_iso_path() returns an empty $path), there should be no # format or QEMU won't start. my $format; if (drive_is_cdrom($drive) && !$path) { # no format } elsif ($storeid) { $format = checked_volume_format($storecfg, $volid); if ($drive->{format} && $drive->{format} ne $format) { die "drive '$drive_id' - volume '$volid' - 'format=$drive->{format}' option different" . " from storage format '$format'\n"; } } else { $format = $drive->{format} // 'raw'; } return ($path, $format); } my $MAX_IDE_DISKS = 4; my $MAX_SCSI_DISKS = 31; my $MAX_VIRTIO_DISKS = 16; our $MAX_SATA_DISKS = 6; our $MAX_UNUSED_DISKS = 256; our $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!; our $drivedesc_hash; # Schema when disk allocation is possible. our $drivedesc_hash_with_alloc = {}; my %drivedesc_base = ( volume => { alias => 'file' }, file => { type => 'string', format => 'pve-volume-id-or-qm-path', default_key => 1, format_description => 'volume', description => "The drive's backing volume.", }, media => { type => 'string', enum => [qw(cdrom disk)], description => "The drive's media type.", default => 'disk', optional => 1, }, snapshot => { type => 'boolean', description => "Controls qemu's snapshot mode feature." . " If activated, changes made to the disk are temporary and will" . " be discarded when the VM is shutdown.", optional => 1, }, cache => { type => 'string', enum => [qw(none writethrough writeback unsafe directsync)], description => "The drive's cache mode", optional => 1, }, format => get_standard_option('pve-qm-image-format'), size => { type => 'string', format => 'disk-size', format_description => 'DiskSize', description => "Disk size. This is purely informational and has no effect.", optional => 1, }, backup => { type => 'boolean', description => "Whether the drive should be included when making backups.", optional => 1, }, replicate => { type => 'boolean', description => 'Whether the drive should considered for replication jobs.', optional => 1, default => 1, }, rerror => { type => 'string', enum => [qw(ignore report stop)], description => 'Read error action.', optional => 1, }, werror => { type => 'string', enum => [qw(enospc ignore report stop)], description => 'Write error action.', optional => 1, }, aio => { type => 'string', enum => [qw(native threads io_uring)], description => 'AIO type to use.', optional => 1, }, discard => { type => 'string', enum => [qw(ignore on)], description => 'Controls whether to pass discard/trim requests to the underlying storage.', optional => 1, }, detect_zeroes => { type => 'boolean', description => 'Controls whether to detect and try to optimize writes of zeroes.', optional => 1, }, serial => { type => 'string', format => 'urlencoded', format_description => 'serial', maxLength => 20 * 3, # *3 since it's %xx url enoded description => "The drive's reported serial number, url-encoded, up to 20 bytes long.", optional => 1, }, shared => { type => 'boolean', description => 'Mark this locally-managed volume as available on all nodes', verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!", optional => 1, default => 0, }, ); my %iothread_fmt = ( iothread => { type => 'boolean', description => "Whether to use iothreads for this drive", optional => 1, }, ); my %product_fmt = ( product => { type => 'string', pattern => '[A-Za-z0-9\-_\s]{,16}', # QEMU (8.1) will quietly only use 16 bytes format_description => 'product', description => "The drive's product name, up to 16 bytes long.", optional => 1, }, ); my %vendor_fmt = ( vendor => { type => 'string', pattern => '[A-Za-z0-9\-_\s]{,8}', # QEMU (8.1) will quietly only use 8 bytes format_description => 'vendor', description => "The drive's vendor name, up to 8 bytes long.", optional => 1, }, ); my %model_fmt = ( model => { type => 'string', format => 'urlencoded', format_description => 'model', maxLength => 40 * 3, # *3 since it's %xx url enoded description => "The drive's reported model name, url-encoded, up to 40 bytes long.", optional => 1, }, ); my %queues_fmt = ( queues => { type => 'integer', description => "Number of queues.", minimum => 2, optional => 1, }, ); my %readonly_fmt = ( ro => { type => 'boolean', description => "Whether the drive is read-only.", optional => 1, }, ); my %scsiblock_fmt = ( scsiblock => { type => 'boolean', description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host", optional => 1, default => 0, }, ); my %ssd_fmt = ( ssd => { type => 'boolean', description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.", optional => 1, }, ); my %wwn_fmt = ( wwn => { type => 'string', pattern => qr/^(0x)[0-9a-fA-F]{16}/, format_description => 'wwn', description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.", optional => 1, }, ); my $add_throttle_desc = sub { my ($key, $type, $what, $unit, $longunit, $minimum) = @_; my $d = { type => $type, format_description => $unit, description => "Maximum $what in $longunit.", optional => 1, }; $d->{minimum} = $minimum if defined($minimum); $drivedesc_base{$key} = $d; }; # throughput: (leaky bucket) $add_throttle_desc->('bps', 'integer', 'r/w speed', 'bps', 'bytes per second'); $add_throttle_desc->('bps_rd', 'integer', 'read speed', 'bps', 'bytes per second'); $add_throttle_desc->('bps_wr', 'integer', 'write speed', 'bps', 'bytes per second'); $add_throttle_desc->('mbps', 'number', 'r/w speed', 'mbps', 'megabytes per second'); $add_throttle_desc->('mbps_rd', 'number', 'read speed', 'mbps', 'megabytes per second'); $add_throttle_desc->('mbps_wr', 'number', 'write speed', 'mbps', 'megabytes per second'); $add_throttle_desc->('iops', 'integer', 'r/w I/O', 'iops', 'operations per second'); $add_throttle_desc->('iops_rd', 'integer', 'read I/O', 'iops', 'operations per second'); $add_throttle_desc->('iops_wr', 'integer', 'write I/O', 'iops', 'operations per second'); # pools: (pool of IO before throttling starts taking effect) $add_throttle_desc->('mbps_max', 'number', 'unthrottled r/w pool', 'mbps', 'megabytes per second'); $add_throttle_desc->( 'mbps_rd_max', 'number', 'unthrottled read pool', 'mbps', 'megabytes per second', ); $add_throttle_desc->( 'mbps_wr_max', 'number', 'unthrottled write pool', 'mbps', 'megabytes per second', ); $add_throttle_desc->( 'iops_max', 'integer', 'unthrottled r/w I/O pool', 'iops', 'operations per second', ); $add_throttle_desc->( 'iops_rd_max', 'integer', 'unthrottled read I/O pool', 'iops', 'operations per second', ); $add_throttle_desc->( 'iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second', ); # burst lengths $add_throttle_desc->('bps_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1); $add_throttle_desc->( 'bps_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1, ); $add_throttle_desc->( 'bps_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1, ); $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1); $add_throttle_desc->( 'iops_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1, ); $add_throttle_desc->( 'iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1, ); # legacy support $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' }; $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' }; $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' }; $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' }; my $ide_fmt = { %drivedesc_base, %model_fmt, %ssd_fmt, %wwn_fmt, }; PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt); my $idedesc = { optional => 1, type => 'string', format => $ide_fmt, description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " . ($MAX_IDE_DISKS - 1) . ").", }; PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc); my $scsi_fmt = { %drivedesc_base, %iothread_fmt, %product_fmt, %queues_fmt, %readonly_fmt, %scsiblock_fmt, %ssd_fmt, %vendor_fmt, %wwn_fmt, }; my $scsidesc = { optional => 1, type => 'string', format => $scsi_fmt, description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").", }; PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc); my $sata_fmt = { %drivedesc_base, %ssd_fmt, %wwn_fmt, }; my $satadesc = { optional => 1, type => 'string', format => $sata_fmt, description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1) . ").", }; PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc); my $virtio_fmt = { %drivedesc_base, %iothread_fmt, %readonly_fmt, }; my $virtiodesc = { optional => 1, type => 'string', format => $virtio_fmt, description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").", }; PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc); my %efitype_fmt = ( efitype => { type => 'string', enum => [qw(2m 4m)], description => "Size and type of the OVMF EFI vars. '4m' is newer and recommended," . " and required for Secure Boot. For backwards compatibility, '2m' is used" . " if not otherwise specified. Ignored for VMs with arch=aarch64 (ARM).", optional => 1, default => '2m', }, 'pre-enrolled-keys' => { type => 'boolean', description => "Use am EFI vars template with distribution-specific and Microsoft Standard" . " keys enrolled, if used with 'efitype=4m'. Note that this will enable Secure Boot by" . " default, though it can still be turned off from within the VM.", optional => 1, default => 0, }, 'ms-cert' => { type => 'string', enum => [qw(2011 2023 2023w 2023k)], description => "Informational marker indicating the version of the latest Microsoft UEFI certificates" . " that have been enrolled by Proxmox VE. The value '2023k' means that the 'Microsoft" . " UEFI CA 2023', the 'Windows UEFI CA 2023' and the 'Microsoft Corporation KEK 2K CA" . " 2023' certificates are included. The values '2023' and '2023w' are" . " deprecated and for compatibility only.", optional => 1, default => '2011', }, ); my $efidisk_fmt = { volume => { alias => 'file' }, file => { type => 'string', format => 'pve-volume-id-or-qm-path', default_key => 1, format_description => 'volume', description => "The drive's backing volume.", }, format => get_standard_option('pve-qm-image-format'), size => { type => 'string', format => 'disk-size', format_description => 'DiskSize', description => "Disk size. This is purely informational and has no effect.", optional => 1, }, %efitype_fmt, }; my $efidisk_desc = { optional => 1, type => 'string', format => $efidisk_fmt, description => "Configure a disk for storing EFI vars.", }; PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc); my %tpmversion_fmt = ( version => { type => 'string', enum => [qw(v1.2 v2.0)], description => "The TPM interface version. v2.0 is newer and should be preferred." . " Note that this cannot be changed later on.", optional => 1, default => 'v1.2', }, ); my $tpmstate_fmt = { volume => { alias => 'file' }, file => { type => 'string', format => 'pve-volume-id-or-qm-path', default_key => 1, format_description => 'volume', description => "The drive's backing volume.", }, format => get_standard_option('pve-vm-image-format', { optional => 1 }), size => { type => 'string', format => 'disk-size', format_description => 'DiskSize', description => "Disk size. This is purely informational and has no effect.", optional => 1, }, %tpmversion_fmt, }; my $tpmstate_desc = { optional => 1, type => 'string', format => $tpmstate_fmt, description => "Configure a Disk for storing TPM state. The format is fixed to 'raw'.", }; use constant TPMSTATE_DISK_SIZE => 4 * 1024 * 1024; my $alldrive_fmt = { %drivedesc_base, %iothread_fmt, %model_fmt, %product_fmt, %queues_fmt, %readonly_fmt, %scsiblock_fmt, %ssd_fmt, %vendor_fmt, %wwn_fmt, %tpmversion_fmt, %efitype_fmt, }; my %import_from_fmt = ( 'import-from' => { type => 'string', format => 'pve-volume-id-or-absolute-path', format_description => 'source volume', description => "Create a new disk, importing from this source (volume ID or absolute " . "path). When an absolute path is specified, it's up to you to ensure that the source " . "is not actively used by another process during the import!", optional => 1, }, ); my $alldrive_fmt_with_alloc = { %$alldrive_fmt, %import_from_fmt, }; my $unused_fmt = { volume => { alias => 'file' }, file => { type => 'string', format => 'pve-volume-id', default_key => 1, format_description => 'volume', description => "The drive's backing volume.", }, }; my $unuseddesc = { optional => 1, type => 'string', format => $unused_fmt, description => "Reference to unused volumes. This is used internally, and should not be modified manually.", }; my $with_alloc_desc_cache = { unused => $unuseddesc, # Allocation for unused is not supported currently. }; my $desc_with_alloc = sub { my ($type, $desc) = @_; return $with_alloc_desc_cache->{$type} if $with_alloc_desc_cache->{$type}; my $new_desc = dclone($desc); $new_desc->{format}->{'import-from'} = $import_from_fmt{'import-from'}; my $extra_note = ''; if ($type eq 'efidisk') { $extra_note = " Note that SIZE_IN_GiB is ignored here and that the default EFI vars are " . "copied to the volume instead."; } elsif ($type eq 'tpmstate') { $extra_note = " Note that SIZE_IN_GiB is ignored here and 4 MiB will be used instead."; } $new_desc->{description} .= " Use the special syntax STORAGE_ID:SIZE_IN_GiB to allocate a new " . "volume.${extra_note} Use STORAGE_ID:0 and the 'import-from' parameter to import from an " . "existing volume."; $with_alloc_desc_cache->{$type} = $new_desc; return $new_desc; }; for (my $i = 0; $i < $MAX_IDE_DISKS; $i++) { $drivedesc_hash->{"ide$i"} = $idedesc; $drivedesc_hash_with_alloc->{"ide$i"} = $desc_with_alloc->('ide', $idedesc); } for (my $i = 0; $i < $MAX_SATA_DISKS; $i++) { $drivedesc_hash->{"sata$i"} = $satadesc; $drivedesc_hash_with_alloc->{"sata$i"} = $desc_with_alloc->('sata', $satadesc); } for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++) { $drivedesc_hash->{"scsi$i"} = $scsidesc; $drivedesc_hash_with_alloc->{"scsi$i"} = $desc_with_alloc->('scsi', $scsidesc); } for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++) { $drivedesc_hash->{"virtio$i"} = $virtiodesc; $drivedesc_hash_with_alloc->{"virtio$i"} = $desc_with_alloc->('virtio', $virtiodesc); } $drivedesc_hash->{efidisk0} = $efidisk_desc; $drivedesc_hash_with_alloc->{efidisk0} = $desc_with_alloc->('efidisk', $efidisk_desc); $drivedesc_hash->{tpmstate0} = $tpmstate_desc; $drivedesc_hash_with_alloc->{tpmstate0} = $desc_with_alloc->('tpmstate', $tpmstate_desc); for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) { $drivedesc_hash->{"unused$i"} = $unuseddesc; $drivedesc_hash_with_alloc->{"unused$i"} = $desc_with_alloc->('unused', $unuseddesc); } sub valid_drive_names_for_boot { return grep { $_ ne 'efidisk0' && $_ ne 'tpmstate0' } valid_drive_names(); } sub valid_drive_names { # order is important - used to autoselect boot disk return ( (map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))), (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))), (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))), (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))), 'efidisk0', 'tpmstate0', ); } sub valid_drive_names_with_unused { return (valid_drive_names(), map { "unused$_" } (0 .. ($MAX_UNUSED_DISKS - 1))); } sub is_valid_drivename { my $dev = shift; return defined($drivedesc_hash->{$dev}) && $dev !~ /^unused\d+$/; } PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk); sub verify_bootdisk { my ($value, $noerr) = @_; return $value if is_valid_drivename($value); return if $noerr; die "invalid boot disk '$value'\n"; } sub drive_is_cloudinit { my ($drive) = @_; return $drive->{file} =~ m@[:/](?:vm-\d+-)?cloudinit(?:\.$QEMU_FORMAT_RE)?$@; } sub drive_is_cdrom { my ($drive, $exclude_cloudinit) = @_; return 0 if $exclude_cloudinit && drive_is_cloudinit($drive); return $drive && $drive->{media} && ($drive->{media} eq 'cdrom'); } sub parse_drive_interface { my ($key) = @_; if ($key =~ m/^([^\d]+)(\d+)$/) { return ($1, $2); } die "unable to parse drive interface $key\n"; } # ideX = [volume=]volume-id[,media=d] # [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no] # [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop] # [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off] # [,iothread=on][,serial=serial][,model=model] sub parse_drive { my ($key, $data, $with_alloc) = @_; my ($interface, $index) = eval { parse_drive_interface($key) }; return if $@; my $desc_hash = $with_alloc ? $drivedesc_hash_with_alloc : $drivedesc_hash; if (!defined($desc_hash->{$key})) { warn "invalid drive key: $key\n"; return; } my $desc = $desc_hash->{$key}->{format}; my $res = eval { my $pps_opts = { skip => $DROPPED_PROPERTIES }; PVE::JSONSchema::parse_property_string($desc, $data, undef, undef, $pps_opts); }; return if !$res; $res->{interface} = $interface; $res->{index} = $index; my $error = 0; foreach my $opt (qw(bps bps_rd bps_wr)) { if (my $bps = defined(delete $res->{$opt})) { if (defined($res->{"m$opt"})) { warn "both $opt and m$opt specified\n"; ++$error; next; } $res->{"m$opt"} = sprintf("%.3f", $bps / (1024 * 1024.0)); } } # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases" for my $requirement ( [mbps_max => 'mbps'], [mbps_rd_max => 'mbps_rd'], [mbps_wr_max => 'mbps_wr'], [miops_max => 'miops'], [miops_rd_max => 'miops_rd'], [miops_wr_max => 'miops_wr'], [bps_max_length => 'mbps_max'], [bps_rd_max_length => 'mbps_rd_max'], [bps_wr_max_length => 'mbps_wr_max'], [iops_max_length => 'iops_max'], [iops_rd_max_length => 'iops_rd_max'], [iops_wr_max_length => 'iops_wr_max'], ) { my ($option, $requires) = @$requirement; if ($res->{$option} && !$res->{$requires}) { warn "$option requires $requires\n"; ++$error; } } return if $error; return if $res->{mbps_rd} && $res->{mbps}; return if $res->{mbps_wr} && $res->{mbps}; return if $res->{iops_rd} && $res->{iops}; return if $res->{iops_wr} && $res->{iops}; if ($res->{media} && ($res->{media} eq 'cdrom')) { return if $res->{snapshot} || $res->{format}; return if $res->{interface} eq 'virtio'; } if (my $size = $res->{size}) { return if !defined($res->{size} = PVE::JSONSchema::parse_size($size)); } return $res; } sub print_drive { my ($drive, $with_alloc) = @_; my $skip = ['index', 'interface']; my $fmt = $with_alloc ? $alldrive_fmt_with_alloc : $alldrive_fmt; return PVE::JSONSchema::print_property_string($drive, $fmt, $skip); } sub get_drive_id { my ($drive) = @_; die "get_drive_id: no interface\n" if !defined($drive->{interface}); die "get_drive_id: no index\n" if !defined($drive->{index}); return "$drive->{interface}$drive->{index}"; } sub get_bootdisks { my ($conf) = @_; my $bootcfg; $bootcfg = PVE::JSONSchema::parse_property_string('pve-qm-boot', $conf->{boot}) if $conf->{boot}; if (!defined($bootcfg) || $bootcfg->{legacy}) { return [$conf->{bootdisk}] if $conf->{bootdisk}; return []; } my @list = PVE::Tools::split_list($bootcfg->{order}); @list = grep { is_valid_drivename($_) } @list; return \@list; } sub bootdisk_size { my ($storecfg, $conf) = @_; my $bootdisks = get_bootdisks($conf); return if !@$bootdisks; for my $bootdisk (@$bootdisks) { next if !is_valid_drivename($bootdisk); next if !$conf->{$bootdisk}; my $drive = parse_drive($bootdisk, $conf->{$bootdisk}); next if !defined($drive); next if drive_is_cdrom($drive); my $volid = $drive->{file}; next if !$volid; return $drive->{size}; } return; } sub update_disksize { my ($drive, $newsize) = @_; return if !defined($newsize); my $oldsize = $drive->{size} // 0; if ($newsize != $oldsize) { $drive->{size} = $newsize; my $old_fmt = PVE::JSONSchema::format_size($oldsize); my $new_fmt = PVE::JSONSchema::format_size($newsize); my $msg = "size of disk '$drive->{file}' updated from $old_fmt to $new_fmt"; return ($drive, $msg); } return; } sub is_volume_in_use { my ($storecfg, $conf, $skip_drive, $volid) = @_; my $path = PVE::Storage::path($storecfg, $volid); my $scan_config = sub { my ($cref) = @_; foreach my $key (keys %$cref) { my $value = $cref->{$key}; if (is_valid_drivename($key)) { next if $skip_drive && $key eq $skip_drive; my $drive = parse_drive($key, $value); next if !$drive || !$drive->{file} || drive_is_cdrom($drive); return 1 if $volid eq $drive->{file}; if ($drive->{file} =~ m!^/!) { return 1 if $drive->{file} eq $path; } else { my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1); next if !$storeid; my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1); next if !$scfg; return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}); } } } return 0; }; return 1 if &$scan_config($conf); undef $skip_drive; for my $snap (values %{ $conf->{snapshots} }) { return 1 if $scan_config->($snap); } return 0; } sub resolve_first_disk { my ($conf, $cdrom) = @_; my @disks = valid_drive_names_for_boot(); foreach my $ds (@disks) { next if !$conf->{$ds}; my $disk = parse_drive($ds, $conf->{$ds}); next if drive_is_cdrom($disk) xor $cdrom; return $ds; } return; } sub scsi_inquiry { my ($fh, $noerr) = @_; my $SG_IO = 0x2285; my $SG_GET_VERSION_NUM = 0x2282; my $versionbuf = "\x00" x 8; my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf); if (!$ret) { die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr; return; } my $version = unpack("I", $versionbuf); if ($version < 30000) { die "scsi generic interface too old\n" if !$noerr; return; } my $buf = "\x00" x 36; my $sensebuf = "\x00" x 8; my $cmd = pack("C x3 C x1", 0x12, 36); # see /usr/include/scsi/sg.h my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I"; my $packet = pack( $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000, ); $ret = ioctl($fh, $SG_IO, $packet); if (!$ret) { die "scsi ioctl SG_IO failed - $!\n" if !$noerr; return; } my @res = unpack($sg_io_hdr_t, $packet); if ($res[17] || $res[18]) { die "scsi ioctl SG_IO status error - $!\n" if !$noerr; return; } my $res = {}; $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf); $res->{removable} = $res->{removable} & 128 ? 1 : 0; $res->{type} &= 0x1F; return $res; } sub path_is_scsi { my ($path) = @_; my $fh = IO::File->new("+<$path") || return; my $res = scsi_inquiry($fh, 1); close($fh); return $res; } sub get_scsi_device_type { my ($drive, $storecfg, $machine_version) = @_; my $devicetype = 'hd'; my $path = ''; if (drive_is_cdrom($drive) || drive_is_cloudinit($drive)) { $devicetype = 'cd'; } else { if ($drive->{file} =~ m|^/|) { $path = $drive->{file}; if (my $info = path_is_scsi($path)) { if ($info->{type} == 0 && $drive->{scsiblock}) { $devicetype = 'block'; } elsif ($info->{type} == 1) { # tape $devicetype = 'generic'; } } } elsif ($drive->{file} =~ $NEW_DISK_RE) { # special syntax cannot be parsed to path return $devicetype; } else { $path = PVE::Storage::path($storecfg, $drive->{file}); } # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380) if ( $path =~ m/^iscsi\:\/\// && !PVE::QemuServer::Helpers::min_version($machine_version, 4, 1) ) { $devicetype = 'generic'; } } return $devicetype; } sub storage_allows_io_uring_default { my ($scfg, $cache_direct) = @_; # io_uring with cache mode writeback or writethrough on krbd will hang... return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct; # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only # sometimes, just plain disable... return if $scfg && $scfg->{type} eq 'lvm'; # io_uring causes problems when used with CIFS since kernel 5.15 # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html return if $scfg && $scfg->{type} eq 'cifs'; return 1; } sub drive_uses_cache_direct { my ($drive, $scfg) = @_; my $cache_direct = 0; if (my $cache = $drive->{cache}) { $cache_direct = $cache =~ /^(?:off|none|directsync)$/; } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) { $cache_direct = 1; } return $cache_direct; } sub aio_cmdline_option { my ($scfg, $drive, $cache_direct) = @_; return $drive->{aio} if $drive->{aio}; if (storage_allows_io_uring_default($scfg, $cache_direct)) { # io_uring supports all cache modes return 'io_uring'; } else { # aio native works only with O_DIRECT if ($cache_direct) { return 'native'; } else { return 'threads'; } } } # must not be called for CD-ROMs sub detect_zeroes_cmdline_option { my ($drive) = @_; die "cannot use detect-zeroes for CD-ROM\n" if drive_is_cdrom($drive); if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) { return 'off'; } elsif ($drive->{discard}) { return $drive->{discard} eq 'on' ? 'unmap' : 'on'; } # This used to be our default with discard not being specified: return 'on'; } sub drive_uses_qsd_fuse { my ($storecfg, $drive) = @_; if ($drive->{interface} eq 'tpmstate') { my ($storeid) = PVE::Storage::parse_volume_id($drive->{file}, 1); my $format = checked_volume_format($storecfg, $drive->{file}); return $storeid && $format ne 'raw'; } return; } sub drive_qmp_peer { my ($storecfg, $vmid, $drive) = @_; return drive_uses_qsd_fuse($storecfg, $drive) ? qsd_qmp_peer($vmid) : vm_qmp_peer($vmid); } 1; ================================================ FILE: src/PVE/QemuServer/DriveDevice.pm ================================================ package PVE::QemuServer::DriveDevice; use strict; use warnings; use URI::Escape; use PVE::QemuServer::Drive qw (drive_is_cdrom); use PVE::QemuServer::Helpers qw(kvm_user_version min_version); use PVE::QemuServer::Machine; use PVE::QemuServer::PCI qw(print_pci_addr); use base qw(Exporter); our @EXPORT_OK = qw( print_drivedevice_full scsihw_infos ); sub scsihw_infos { my ($scsihw, $drive_index) = @_; my $maxdev = 0; if (!$scsihw || ($scsihw =~ m/^lsi/)) { $maxdev = 7; } elsif ($scsihw && ($scsihw eq 'virtio-scsi-single')) { $maxdev = 1; } else { $maxdev = 256; } my $controller = int($drive_index / $maxdev); my $controller_prefix = ($scsihw && $scsihw eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw"; return ($maxdev, $controller, $controller_prefix); } sub print_drivedevice_full { my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_; my $device = ''; my $maxdev = 0; my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version()); my $has_write_cache = 1; # whether the device has a 'write-cache' option my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive); if ($drive->{interface} eq 'virtio') { my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch); $device = 'virtio-blk-pci'; # for the switch to -blockdev, there is no blockdev for 'none' if (!min_version($machine_version, 10, 0) || $drive->{file} ne 'none') { $device .= ",drive=drive-$drive_id"; } $device .= ",id=${drive_id}${pciaddr}"; $device .= ",iothread=iothread-$drive_id" if $drive->{iothread}; } elsif ($drive->{interface} eq 'scsi') { my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf->{scsihw}, $drive->{index}); my $unit = $drive->{index} % $maxdev; my $device_type = PVE::QemuServer::Drive::get_scsi_device_type($drive, $storecfg, $machine_version); if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') { $device = "scsi-$device_type,bus=$controller_prefix$controller.0,scsi-id=$unit"; } else { $device = "scsi-$device_type,bus=$controller_prefix$controller.0,channel=0,scsi-id=0" . ",lun=$drive->{index}"; } # for the switch to -blockdev, there is no blockdev for 'none' if (!min_version($machine_version, 10, 0) || $drive->{file} ne 'none') { $device .= ",drive=drive-$drive_id"; } $device .= ",id=$drive_id"; # For the switch to -blockdev, the SCSI device ID needs to be explicitly specified. Note # that only ide-cd and ide-hd have a 'device_id' option. if ( min_version($machine_version, 10, 0) && ($device_type eq 'cd' || $device_type eq 'hd') ) { $device .= ",device_id=drive-${drive_id}"; } if ($drive->{ssd} && ($device_type eq 'block' || $device_type eq 'hd')) { $device .= ",rotation_rate=1"; } $device .= ",wwn=$drive->{wwn}" if $drive->{wwn}; # only scsi-hd and scsi-cd support passing vendor and product information and have a # 'write-cache' option if ($device_type eq 'hd' || $device_type eq 'cd') { if (my $vendor = $drive->{vendor}) { $device .= ",vendor=$vendor"; } if (my $product = $drive->{product}) { $device .= ",product=$product"; } $has_write_cache = 1; } else { $has_write_cache = 0; } } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') { my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2; my $controller = int($drive->{index} / $maxdev); my $unit = $drive->{index} % $maxdev; # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they # were before. Move odd ones up by 2 where they don't clash. if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') { $controller += 2 * ($unit % 2); $unit = 0; } my $device_type = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd"; # With ide-hd, the inserted block node needs to be marked as writable too, but -blockdev # will complain if it's marked as writable but the actual backing device is read-only (e.g. # read-only base LV). IDE/SATA do not support being configured as read-only, the most # similar is using ide-cd instead of ide-hd, with most of the code and configuration shared # in QEMU. Since a template is never actually started, the front-end device is never # accessed. The backup only accesses the inserted block node, so it does not matter for the # backup if the type is 'ide-cd' instead. $device_type = 'cd' if $conf->{template}; $device = "ide-$device_type"; if ($drive->{interface} eq 'ide') { $device .= ",bus=ide.$controller,unit=$unit"; } else { $device .= ",bus=ahci$controller.$unit"; } if (!min_version($machine_version, 10, 0) || $drive->{file} ne 'none') { $device .= ",drive=drive-$drive_id"; } $device .= ",id=$drive_id"; if ($device_type eq 'hd') { if (my $model = $drive->{model}) { $model = URI::Escape::uri_unescape($model); $device .= ",model=$model"; } if ($drive->{ssd}) { $device .= ",rotation_rate=1"; } } $device .= ",wwn=$drive->{wwn}" if $drive->{wwn}; } elsif ($drive->{interface} eq 'usb') { die "implement me"; # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0 } else { die "unsupported interface type"; } $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex}; if (my $serial = $drive->{serial}) { $serial = URI::Escape::uri_unescape($serial); $device .= ",serial=$serial"; } if (min_version($machine_version, 10, 0)) { # for the switch to -blockdev if (!drive_is_cdrom($drive) && $has_write_cache) { my $write_cache = 'on'; if (my $cache = $drive->{cache}) { $write_cache = 'off' if $cache eq 'writethrough' || $cache eq 'directsync'; } $device .= ",write-cache=$write_cache"; } for my $o (qw(rerror werror)) { $device .= ",$o=$drive->{$o}" if defined($drive->{$o}); } } return $device; } 1; ================================================ FILE: src/PVE/QemuServer/Helpers.pm ================================================ package PVE::QemuServer::Helpers; use strict; use warnings; use File::stat; use IO::File; use JSON; use PVE::Cluster; use PVE::INotify; use PVE::ProcFSTools; use PVE::Tools; use base 'Exporter'; our @EXPORT_OK = qw( min_version config_aware_timeout get_iscsi_initiator_name kvm_user_version parse_number_sets windows_version get_host_arch ); my $nodename = PVE::INotify::nodename(); my $arch_to_qemu_binary = { aarch64 => '/usr/bin/qemu-system-aarch64', x86_64 => '/usr/bin/qemu-system-x86_64', }; # wrapper around the Tools helper, having it here makes it easier to mock for testing sub get_host_arch { return PVE::Tools::get_host_arch(); } sub get_command_for_arch($) { my ($arch) = @_; return '/usr/bin/kvm' if get_host_arch() eq $arch; # i.e. native arch my $cmd = $arch_to_qemu_binary->{$arch} or die "don't know how to emulate architecture '$arch'\n"; return $cmd; } sub get_vm_arch { my ($conf) = @_; return $conf->{arch} // get_host_arch(); } my $kvm_user_version = {}; my $kvm_mtime = {}; sub kvm_user_version { my ($binary) = @_; $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default my $st = stat($binary); my $cachedmtime = $kvm_mtime->{$binary} // -1; return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} && $cachedmtime == $st->mtime; $kvm_user_version->{$binary} = 'unknown'; $kvm_mtime->{$binary} = $st->mtime; my $code = sub { my $line = shift; if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) { $kvm_user_version->{$binary} = $2; } }; eval { PVE::Tools::run_command([$binary, '--version'], outfunc => $code); }; warn $@ if $@; return $kvm_user_version->{$binary}; } # Paths and directories our $var_run_tmpdir = "/var/run/qemu-server"; mkdir $var_run_tmpdir; sub qmp_socket { my ($peer) = @_; my ($id, $type) = $peer->@{qw(id type)}; return "${var_run_tmpdir}/${id}.${type}"; } sub qsd_pidfile_name { my ($id) = @_; return "${var_run_tmpdir}/qsd-${id}.pid"; } sub qsd_fuse_export_cleanup_files { my ($id) = @_; # Usually, /var/run is a symlink to /run. It needs to be the exact path for checking if mounted # below. Note that Cwd::realpath() needs to be done on the directory already. Doing it on the # file does not work if the storage daemon is not running and the FUSE is still mounted. my ($real_dir) = Cwd::realpath($var_run_tmpdir) =~ m/^(.*)$/; # untaint if (!$real_dir) { warn "error resolving $var_run_tmpdir - not checking for left-over QSD files\n"; return; } my $mounts = PVE::ProcFSTools::parse_proc_mounts(); PVE::Tools::dir_glob_foreach( $real_dir, "qsd-${id}-.*\.fuse", sub { my ($file) = @_; my $path = "${real_dir}/${file}"; if (grep { $_->[1] eq $path } $mounts->@*) { PVE::Tools::run_command(['umount', $path]); } unlink $path; }, ); } sub qsd_fuse_export_path { my ($id, $export_name) = @_; return "${var_run_tmpdir}/qsd-${id}-${export_name}.fuse"; } sub vm_pidfile_name { my ($vmid) = @_; return "${var_run_tmpdir}/$vmid.pid"; } sub vnc_socket { my ($vmid) = @_; return "${var_run_tmpdir}/$vmid.vnc"; } # Parse the cmdline of a running kvm/qemu-* process and return arguments as hash sub parse_cmdline { my ($pid) = @_; my $fh = IO::File->new("/proc/$pid/cmdline", "r"); if (defined($fh)) { my $line = <$fh>; $fh->close; return if !$line; my @param = split(/\0/, $line); my $cmd = $param[0]; return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-[^/]+$@); my $phash = {}; my $pending_cmd; for (my $i = 0; $i < scalar(@param); $i++) { my $p = $param[$i]; next if !$p; if ($p =~ m/^--?(.*)$/) { if ($pending_cmd) { $phash->{$pending_cmd} = {}; } $pending_cmd = $1; } elsif ($pending_cmd) { $phash->{$pending_cmd} = { value => $p }; $pending_cmd = undef; } } return $phash; } return; } my sub instance_running_locally { my ($pidfile) = @_; if (my $fd = IO::File->new("<$pidfile")) { my $st = stat($fd); my $line = <$fd>; close($fd); my $mtime = $st->mtime; if ($mtime > time()) { warn "file '$pidfile' modified in future\n"; } if ($line =~ m/^(\d+)$/) { my $pid = $1; my $cmdline = parse_cmdline($pid); if ( $cmdline && defined($cmdline->{pidfile}) && $cmdline->{pidfile}->{value} && $cmdline->{pidfile}->{value} eq $pidfile ) { if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) { return $pid; } } } } return; } sub qsd_running_locally { my ($id) = @_; my $pidfile = qsd_pidfile_name($id); return instance_running_locally($pidfile); } sub vm_running_locally { my ($vmid) = @_; my $pidfile = vm_pidfile_name($vmid); return instance_running_locally($pidfile); } sub min_version { my ($verstr, $major, $minor, $pve) = @_; if ($verstr =~ m/^(\d+)\.(\d+)(?:\.(\d+))?(?:\+pve(\d+))?/) { return 1 if version_cmp($1, $major, $2, $minor, $4, $pve) >= 0; return 0; } die "internal error: cannot check version of invalid string '$verstr'"; } # gets in pairs the versions you want to compares, i.e.: # ($a-major, $b-major, $a-minor, $b-minor, $a-extra, $b-extra, ...) # returns 0 if same, -1 if $a is older than $b, +1 if $a is newer than $b sub version_cmp { my @versions = @_; my $size = scalar(@versions); return 0 if $size == 0; if ($size & 1) { my (undef, $fn, $line) = caller(0); die "cannot compare odd count of versions, called from $fn:$line\n"; } for (my $i = 0; $i < $size; $i += 2) { my ($left, $right) = splice(@versions, 0, 2); $left //= 0; $right //= 0; return 1 if $left > $right; return -1 if $left < $right; } return 0; } sub config_aware_timeout { my ($config, $memory, $is_suspended) = @_; my $timeout = 30; # Based on user reported startup time for vm with 512GiB @ 4-5 minutes if (defined($memory) && $memory > 30720) { $timeout = int($memory / 1024); } # When using PCI passthrough, users reported much higher startup times, # growing with the amount of memory configured. Constant factor chosen # based on user reports. if (grep(/^hostpci[0-9]+$/, keys %$config)) { $timeout *= 4; } if ($is_suspended && $timeout < 300) { $timeout = 300; } if ($config->{hugepages} && $timeout < 150) { $timeout = 150; } # Some testing showed that adding a NIC increased the start time by ~450ms # consistently across different NIC models, options and already existing # number of NICs. # So 10x that to account for any potential system differences seemed # reasonable. User reports with real-life values (20+: ~50s, 25: 45s, 17: 42s) # also make this seem a good value. my $nic_count = scalar(grep { /^net\d+/ } keys %{$config}); $timeout += $nic_count * 5; return $timeout; } sub get_node_pvecfg_version { my ($node) = @_; my $nodes_version_info = PVE::Cluster::get_node_kv('version-info', $node); return if !$nodes_version_info->{$node}; my $version_info = decode_json($nodes_version_info->{$node}); return $version_info->{version}; } sub pvecfg_min_version { my ($verstr, $major, $minor, $release) = @_; return 0 if !$verstr; if ($verstr =~ m/^(\d+)\.(\d+)(?:[.-](\d+))?/) { return 1 if version_cmp($1, $major, $2, $minor, $3 // 0, $release) >= 0; return 0; } die "internal error: cannot check version of invalid string '$verstr'"; } sub parse_number_sets { my ($set) = @_; my $res = []; foreach my $part (split(/;/, $set)) { if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) { die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1; push @$res, [$1, $2]; } else { die "invalid range: $part\n"; } } return $res; } sub windows_version { my ($ostype) = @_; return 0 if !$ostype; my $winversion = 0; if ($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') { $winversion = 5; } elsif ($ostype eq 'w2k8' || $ostype eq 'wvista') { $winversion = 6; } elsif ($ostype =~ m/^win(\d+)$/) { $winversion = $1; } return $winversion; } sub needs_extraction { my ($vtype, $fmt) = @_; return $vtype eq 'import' && $fmt =~ m/^ova\+(.*)$/; } sub get_iscsi_initiator_name { my $initiator; my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return; while (defined(my $line = <$fh>)) { next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/; $initiator = $1; last; } $fh->close(); return $initiator; } my $_host_bits; sub get_host_phys_address_bits { return $_host_bits if defined($_host_bits); my $fh = IO::File->new('/proc/cpuinfo', "r") or return; while (defined(my $line = <$fh>)) { # hopefully we never need to care about mixed (big.LITTLE) archs if ($line =~ m/^address sizes\s*:\s*(\d+)\s*bits physical/i) { $_host_bits = int($1); $fh->close(); return $_host_bits; } } $fh->close(); return; # undef, cannot really do anything.. } 1; ================================================ FILE: src/PVE/QemuServer/ImportDisk.pm ================================================ package PVE::QemuServer::ImportDisk; use strict; use warnings; use PVE::Storage; use PVE::Tools qw(run_command extract_param); use PVE::QemuConfig; use PVE::QemuServer; use PVE::QemuServer::QemuImage; # imports an external disk image to an existing VM # and creates by default a drive entry unused[n] pointing to the created volume # $params->{drive_name} may be used to specify ide0, scsi1, etc ... # $params->{format} may be used to specify qcow2, raw, etc ... # $params->{skiplock} may be used to skip checking for a lock in the VM config # $params->{'skip-config-update'} may be used to import the disk without updating the VM config sub do_import { my ($src_path, $src_size, $vmid, $storage_id, $params) = @_; my $drive_name = extract_param($params, 'drive_name'); my $format = extract_param($params, 'format'); if ($drive_name && !(PVE::QemuServer::is_valid_drivename($drive_name))) { die "invalid drive name: $drive_name\n"; } # get target format, target image's path, and whether it's possible to sparseinit my $storecfg = PVE::Storage::config(); my $dst_format = PVE::QemuServer::resolve_dst_disk_format($storecfg, $storage_id, undef, $format); warn "format '$format' is not supported by the target storage - using '$dst_format' instead\n" if $format && $format ne $dst_format; my $dst_volid = PVE::Storage::vdisk_alloc( $storecfg, $storage_id, $vmid, $dst_format, undef, $src_size / 1024, ); my $zeroinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $dst_volid); my $create_drive = sub { my $vm_conf = PVE::QemuConfig->load_config($vmid); if (!$params->{skiplock}) { PVE::QemuConfig->check_lock($vm_conf); } if ($drive_name) { # should never happen as setting $drive_name is not exposed to public interface die "cowardly refusing to overwrite existing entry: $drive_name\n" if $vm_conf->{$drive_name}; my $modified = {}; # record what $option we modify $modified->{$drive_name} = 1; $vm_conf->{pending}->{$drive_name} = $dst_volid; PVE::QemuConfig->write_config($vmid, $vm_conf); my $running = PVE::QemuServer::check_running($vmid); if ($running) { my $errors = {}; PVE::QemuServer::vmconfig_hotplug_pending( $vmid, $vm_conf, $storecfg, $modified, $errors, ); warn "hotplugging imported disk '$_' failed: $errors->{$_}\n" for keys %$errors; } else { PVE::QemuServer::vmconfig_apply_pending($vmid, $vm_conf, $storecfg); } } else { $drive_name = PVE::QemuConfig->add_unused_volume($vm_conf, $dst_volid); PVE::QemuConfig->write_config($vmid, $vm_conf); } }; eval { # trap interrupts so we have a chance to clean up local $SIG{INT} = local $SIG{TERM} = local $SIG{QUIT} = local $SIG{HUP} = local $SIG{PIPE} = sub { die "interrupted by signal $!\n"; }; PVE::Storage::activate_volumes($storecfg, [$dst_volid]); PVE::QemuServer::QemuImage::convert( $src_path, $dst_volid, $src_size, { 'is-zero-initialized' => $zeroinit }, ); PVE::Storage::deactivate_volumes($storecfg, [$dst_volid]); PVE::QemuConfig->lock_config($vmid, $create_drive) if !$params->{'skip-config-update'}; }; if (my $err = $@) { eval { PVE::Storage::vdisk_free($storecfg, $dst_volid) }; warn "cleanup of $dst_volid failed: $@\n" if $@; die $err; } return ($drive_name, $dst_volid); } 1; ================================================ FILE: src/PVE/QemuServer/Machine.pm ================================================ package PVE::QemuServer::Machine; use strict; use warnings; use PVE::QemuServer::Helpers; use PVE::QemuServer::MetaInfo; use PVE::QemuServer::Monitor; use PVE::JSONSchema qw(get_standard_option parse_property_string print_property_string); # The PVE machine versions allow rolling out (incompatibel) changes to the hardware layout and/or # the QEMU command of a VM without requiring a newer QEMU upstream machine version. # # To use this find the newest available QEMU machine version, add and entry in this hash if it does # not already exists and then bump the higherst version, or introduce a new one starting at 1, as # the upstream version is counted as having a PVE revision of 0. # Additionally you must describe in short what the basic changes done with such a new PVE machine # revision in the respective subhash, use the full version including the pve one as key there. # # NOTE: Do not overuse this. one or two changes per upstream machine can be fine, if needed. But # most of the time it's better to batch more together and if there is no time pressure then wait a # few weeks/months until the next QEMU machine revision is ready. As it will get confusing otherwise # and we lazily use some simple ascii sort when processing these, so more than 10 per entry require # changes but should be avoided in the first place. # TODO: add basic test to ensure the keys are correct and there's a change entry for each version. our $PVE_MACHINE_VERSION = { '4.1' => { highest => 2, revisions => { '+pve1' => 'Introduction of pveX versioning, no changes.', '+pve2' => 'Increases the number of SCSI drives supported.', }, }, '9.2' => { highest => 1, revisions => { '+pve1' => 'Disables S3/S4 power states by default.', }, }, '10.0' => { highest => 1, revisions => { '+pve1' => 'Set host_mtu vNIC option even with default value for migration compat.', }, }, }; my $machine_fmt = { type => { default_key => 1, description => "Specifies the QEMU machine type.", type => 'string', pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)', maxLength => 40, format_description => 'machine type', optional => 1, }, viommu => { type => 'string', description => "Enable and set guest vIOMMU variant (Intel vIOMMU needs q35 to be set as" . " machine type).", enum => ['intel', 'virtio'], optional => 1, }, 'aw-bits' => { type => 'number', description => "Specifies the vIOMMU address space bit width.", verbose_description => "Specifies the vIOMMU address space bit width.\n\n" . "Intel vIOMMU supports a bit width of either 39 or 48 bits and" . " VirtIO vIOMMU supports any bit width between 32 and 64 bits.", minimum => 32, maximum => 64, optional => 1, }, 'enable-s3' => { type => 'boolean', description => "Enables S3 power state. Defaults to false beginning with machine types 9.2+pve1, true before.", optional => 1, }, 'enable-s4' => { type => 'boolean', description => "Enables S4 power state. Defaults to false beginning with machine types 9.2+pve1, true before.", optional => 1, }, }; PVE::JSONSchema::register_format('pve-qemu-machine-fmt', $machine_fmt); PVE::JSONSchema::register_standard_option( 'pve-qemu-machine', { description => "Specify the QEMU machine.", type => 'string', optional => 1, format => PVE::JSONSchema::get_format('pve-qemu-machine-fmt'), }, ); sub parse_machine { my ($value) = @_; return if !$value; my $res = parse_property_string($machine_fmt, $value); return $res; } sub print_machine { my ($machine_conf) = @_; return print_property_string($machine_conf, $machine_fmt); } my $default_machines = { x86_64 => 'pc', aarch64 => 'virt', }; sub default_machine_for_arch { my ($arch) = @_; my $machine = $default_machines->{$arch} or die "unsupported architecture '$arch'\n"; return $machine; } sub assert_valid_machine_property { my ($machine_conf) = @_; if ($machine_conf->{viommu} && $machine_conf->{viommu} eq "intel") { my $q35 = $machine_conf->{type} && ($machine_conf->{type} =~ m/q35/) ? 1 : 0; die "to use Intel vIOMMU please set the machine type to q35\n" if !$q35; die "Intel vIOMMU supports only 39 or 48 bits as address width\n" if $machine_conf->{'aw-bits'} && $machine_conf->{'aw-bits'} != 39 && $machine_conf->{'aw-bits'} != 48; } die "cannot set aw-bits if no vIOMMU is configured\n" if $machine_conf->{'aw-bits'} && !$machine_conf->{viommu}; } =head3 machine_base_type my $base_type = machine_base_type($machine_type); Returns the base type of the machine, currently either C, C or C. A value must be passed in. Dies if the machine type cannot be determined, but should not happen if it is valid for the C<$machine_fmt> schema. =cut my sub machine_base_type { my ($machine_type) = @_; die "unable to determine machine base type - no value\n" if !$machine_type; return 'q35' if $machine_type =~ m/q35/; return 'i440fx' if $machine_type =~ m/^pc/; return 'virt' if $machine_type =~ m/^virt/; die "unable to determine machine base type '$machine_type'\n"; } sub machine_type_is_q35 { my ($conf) = @_; my $machine_conf = parse_machine($conf->{machine}); return 0 if !$machine_conf || !$machine_conf->{type}; return machine_base_type($machine_conf->{type}) eq 'q35' ? 1 : 0; } # When you need to check a new flag, extend here and the POD for machine_supports_flag(). my $supported_machine_flags = { i440fx => { hpet => 1, }, q35 => { hpet => 1, }, virt => {}, }; =head3 machine_supports_flag if (machine_supports_flag($machine_type, $flag)) { push $machine_flags->@*, $flag; } Check whether the machine type C<$machine_type> supports the machine flag C<$flag>. Both arguments must have a value. Flags which can be checked currently: C. =cut sub machine_supports_flag { my ($machine_type, $flag) = @_; die "cannot check machine flag support - no machine type provided\n" if !$machine_type; die "cannot check machine flag support - no flag provided\n" if !$flag; my $base_type = machine_base_type($machine_type); return $supported_machine_flags->{$base_type}->{$flag}; } # In list context, also returns whether the current machine is deprecated or not. sub current_from_query_machines { my ($machines) = @_; my ($current, $default); for my $machine ($machines->@*) { $default = $machine->{name} if $machine->{'is-default'}; if ($machine->{'is-current'}) { $current = $machine->{name}; # pve-version only exists for the current machine $current .= "+$machine->{'pve-version'}" if $machine->{'pve-version'}; return wantarray ? ($current, $machine->{deprecated} ? 1 : 0) : $current; } } # fallback to the default machine if current is not supported by qemu - assume never deprecated my $fallback = $default || 'pc'; return wantarray ? ($fallback, 0) : $fallback; } # This only works if VM is running. # In list context, also returns whether the current machine is deprecated or not. sub get_current_qemu_machine { my ($vmid) = @_; my $res = PVE::QemuServer::Monitor::mon_cmd($vmid, 'query-machines'); return current_from_query_machines($res); } =head3 extract_version_parts my ($major, $minor, $pve) = extract_version_parts($machine_type); Returns the major, minor and pve versions from the given C<$machine_type> string. Returns nothing if the string did not contain any version or if parsing failed. =cut my sub extract_version_parts { my ($machine_type) = @_; if ($machine_type =~ m/^(?:pc(?:-i440fx|-q35)?|virt)-(\d+)\.(\d+)(?:\.(\d+))?(?:\+pve(\d+))?(?:\.pxe)?/ ) { return ($1, $2, $4); } return; } # returns a string with major.minor+pve, patch version-part is ignored # as it's seldom resembling a real QEMU machine type, so it would be '0' 99% of # the time anyway.. This explicitly separates pveversion from the machine. sub extract_version { my ($machine_type, $kvmversion) = @_; my ($major, $minor, $pve); ($major, $minor, $pve) = extract_version_parts($machine_type) if defined($machine_type); if (defined($major) && defined($minor)) { my $versionstr = "${major}.${minor}"; $versionstr .= "+pve${pve}" if $pve; return $versionstr; } elsif (defined($kvmversion)) { if ($kvmversion =~ m/^(\d+)\.(\d+)/) { my $pvever = get_pve_version($kvmversion); return "$1.$2+pve$pvever"; } } return; } =head3 machine_version_cmp sort { machine_version_cmp($a, $b) } @machine_types Comparision function for sorting machine types by version. =cut sub machine_version_cmp { my ($machine_type_a, $machine_type_b) = @_; my ($major_a, $minor_a, $pve_a) = extract_version_parts($machine_type_a); my ($major_b, $minor_b, $pve_b) = extract_version_parts($machine_type_b); return PVE::QemuServer::Helpers::version_cmp( $major_a, $major_b, $minor_a, $minor_b, $pve_a, $pve_b, ); } sub is_machine_version_at_least { my ($machine_type, $major, $minor, $pve) = @_; return PVE::QemuServer::Helpers::min_version(extract_version($machine_type), $major, $minor, $pve); } sub get_machine_pve_revisions { my ($machine_version_str) = @_; if ($machine_version_str =~ m/^(\d+\.\d+)/) { return $PVE_MACHINE_VERSION->{$1}; } die "internal error: cannot get pve version for invalid string '$machine_version_str'"; } sub get_pve_version { my ($verstr) = @_; if (my $pve_machine = get_machine_pve_revisions($verstr)) { return $pve_machine->{highest} || die "internal error - machine version '$verstr' missing 'highest'"; } return 0; } sub can_run_pve_machine_version { my ($machine_version, $kvmversion) = @_; $machine_version =~ m/^(\d+)\.(\d+)(?:\+pve(\d+))?(?:\.pxe)?$/; my $major = $1; my $minor = $2; my $pvever = $3; $kvmversion =~ m/(\d+)\.(\d+)/; return 0 if PVE::QemuServer::Helpers::version_cmp($1, $major, $2, $minor) < 0; # if $pvever is missing or 0, we definitely support it as long as we didn't # fail the QEMU version check above return 1 if !$pvever; my $max_supported = get_pve_version("$major.$minor"); return 1 if $max_supported >= $pvever; return 0; } sub qemu_machine_pxe { my ($vmid, $conf) = @_; my $machine = get_current_qemu_machine($vmid); my $machine_conf = parse_machine($conf->{machine}); if ($machine_conf->{type} && $machine_conf->{type} =~ m/\.pxe$/) { $machine .= '.pxe'; } return $machine; } sub latest_installed_machine_version { my ($kvmversion) = @_; $kvmversion = PVE::QemuServer::Helpers::kvm_user_version() if !defined($kvmversion); my ($version) = ($kvmversion =~ m/^(\d+\.\d+)/); my $pvever = get_pve_version($version); $version .= "+pve$pvever" if $pvever > 0; return $version; } sub windows_get_pinned_machine_version { my ($machine, $base_version, $kvmversion) = @_; die "internal error - no machine provided" if !$machine; my $pin_version = $base_version; if (!defined($base_version) || !can_run_pve_machine_version($base_version, $kvmversion)) { $pin_version = latest_installed_machine_version($kvmversion); } if ($machine eq 'pc') { $machine = "pc-i440fx-$pin_version"; } elsif ($machine eq 'q35') { $machine = "pc-q35-$pin_version"; } elsif ($machine eq 'virt') { $machine = "virt-$pin_version"; } else { warn "unknown machine type '$machine', not touching that!\n"; } return $machine; } sub get_vm_machine { my ($conf, $forcemachine) = @_; my $machine_conf = parse_machine($conf->{machine}); my $machine = $forcemachine || $machine_conf->{type}; if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) { my $kvmversion = PVE::QemuServer::Helpers::kvm_user_version(); my $arch = PVE::QemuServer::Helpers::get_vm_arch($conf); $machine ||= default_machine_for_arch($arch); # we must pin Windows VMs without a specific version and no meta info about creation QEMU to # 5.1, as 5.2 fixed a bug in ACPI layout which confuses windows quite a bit and may result # in various regressions.. # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html # Starting from QEMU 9.1, pin to the creation version instead. Support for 5.1 is expected # to drop with QEMU 11.1 and it would still be good to handle Windows VMs that do not have # an explicit machine version for whatever reason. if (PVE::QemuServer::Helpers::windows_version($conf->{ostype})) { my $base_version = '5.1'; # TODO PVE 10 - die early if there is a Windows VM both without explicit machine version # and without meta info. if (my $meta = PVE::QemuServer::MetaInfo::parse_meta_info($conf->{meta})) { if (PVE::QemuServer::Helpers::min_version($meta->{'creation-qemu'}, 9, 1)) { # need only major.minor ($base_version) = ($meta->{'creation-qemu'} =~ m/^(\d+.\d+)/); } } $machine = windows_get_pinned_machine_version($machine, $base_version, $kvmversion); } else { my $pvever = get_pve_version($kvmversion); $machine .= "+pve$pvever"; } } if ($machine !~ m/\+pve\d+?(?:\.pxe)?$/) { my $is_pxe = $machine =~ m/^(.*?)\.pxe$/; $machine = $1 if $is_pxe; # for version-pinned machines that do not include a pve-version (e.g. # pc-q35-4.1), we assume 0 to keep them stable in case we bump $machine .= '+pve0'; $machine .= '.pxe' if $is_pxe; } return $machine; } sub check_and_pin_machine_string { my ($machine_string, $ostype, $arch) = @_; my $machine_conf = parse_machine($machine_string); my $machine = $machine_conf->{type}; if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) { # always pin Windows' machine version on create, they get confused too easily if (PVE::QemuServer::Helpers::windows_version($ostype)) { $machine = default_machine_for_arch($arch) if !$machine; $machine_conf->{type} = windows_get_pinned_machine_version($machine); print "pinning machine type to '$machine_conf->{type}' for Windows guest OS\n"; } } assert_valid_machine_property($machine_conf); return print_machine($machine_conf); } # disable s3/s4 by default for 9.2+pve1 machine types # returns an arrayref of cmdline options for qemu or undef sub get_power_state_flags { my ($machine_conf, $arch, $version_guard) = @_; my $machine = $machine_conf->{type} || default_machine_for_arch($arch); if ($machine =~ /^virt/) { return; # virt machines are normally ARM64 ones which have no concept of s3/s4 } my $object = ($machine =~ m/q35/) ? "ICH9-LPC" : "PIIX4_PM"; my $default = 1; if ($version_guard->(9, 2, 1)) { $default = 0; } my $s3 = $machine_conf->{'enable-s3'} // $default; my $s4 = $machine_conf->{'enable-s4'} // $default; my $options = []; # they're enabled by default in QEMU, so only add the flags to disable them if (!$s3) { push $options->@*, '-global', "${object}.disable_s3=1"; } if (!$s4) { push $options->@*, '-global', "${object}.disable_s4=1"; } if (scalar($options->@*)) { return $options; } return; } 1; ================================================ FILE: src/PVE/QemuServer/Makefile ================================================ DESTDIR= PREFIX=/usr PERLDIR=$(PREFIX)/share/perl5 SOURCES=Agent.pm \ Blockdev.pm \ BlockJob.pm \ Cfg2Cmd.pm \ CGroup.pm \ Cloudinit.pm \ CPUConfig.pm \ DBusVMState.pm \ Drive.pm \ DriveDevice.pm \ Helpers.pm \ ImportDisk.pm \ Machine.pm \ Memory.pm \ MetaInfo.pm \ Monitor.pm \ Network.pm \ OVMF.pm \ PCI.pm \ QemuImage.pm \ QMPHelpers.pm \ QSD.pm \ RNG.pm \ RunState.pm \ StateFile.pm \ USB.pm \ Virtiofs.pm \ VolumeChain.pm .PHONY: install install: $(SOURCES) for i in $(SOURCES); do install -D -m 0644 $$i $(DESTDIR)$(PERLDIR)/PVE/QemuServer/$$i; done $(MAKE) -C Cfg2Cmd install ================================================ FILE: src/PVE/QemuServer/Memory.pm ================================================ package PVE::QemuServer::Memory; use strict; use warnings; use PVE::JSONSchema qw(parse_property_string); use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach); use PVE::Exception qw(raise raise_param_exc); use PVE::QemuServer::Helpers qw(parse_number_sets); use PVE::QemuServer::Monitor qw(mon_cmd); use PVE::QemuServer::QMPHelpers qw(qemu_devicedel qemu_objectdel); use base qw(Exporter); our @EXPORT_OK = qw( get_current_memory ); our $MAX_NUMA = 8; my $numa_fmt = { cpus => { type => "string", pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/, description => "CPUs accessing this NUMA node.", format_description => "id[-id];...", }, memory => { type => "number", description => "Amount of memory this NUMA node provides.", optional => 1, }, hostnodes => { type => "string", pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/, description => "Host NUMA nodes to use.", format_description => "id[-id];...", optional => 1, }, policy => { type => 'string', enum => [qw(preferred bind interleave)], description => "NUMA allocation policy.", optional => 1, }, }; PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt); our $numadesc = { optional => 1, type => 'string', format => $numa_fmt, description => "NUMA topology.", }; PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc); sub parse_numa { my ($data) = @_; my $res = parse_property_string($numa_fmt, $data); $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus}); $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes}); return $res; } my $STATICMEM = 1024; our $memory_fmt = { current => { description => "Current amount of online RAM for the VM in MiB. This is the maximum available memory when" . " you use the balloon device.", type => 'integer', default_key => 1, minimum => 16, default => 512, }, }; sub print_memory { my $memory = shift; return PVE::JSONSchema::print_property_string($memory, $memory_fmt); } sub parse_memory { my ($value) = @_; return { current => $memory_fmt->{current}->{default} } if !defined($value); my $res = PVE::JSONSchema::parse_property_string($memory_fmt, $value); return $res; } my sub get_max_mem { my ($conf) = @_; my $cpu = {}; if (my $cpu_prop_str = $conf->{cpu}) { $cpu = PVE::JSONSchema::parse_property_string('pve-vm-cpu-conf', $cpu_prop_str) or die "Cannot parse cpu description: $cpu_prop_str\n"; } my $bits; if (my $phys_bits = $cpu->{'phys-bits'}) { if ($phys_bits eq 'host') { $bits = PVE::QemuServer::Helpers::get_host_phys_address_bits(); } elsif ($phys_bits =~ /^(\d+)$/) { $bits = int($phys_bits); } } if (!defined($bits)) { # fixme: what fallback? my $host_bits = PVE::QemuServer::Helpers::get_host_phys_address_bits() // 36; if ($cpu->{cputype} && $cpu->{cputype} =~ /^(host|max)$/) { $bits = $host_bits; } else { $bits = $host_bits > 40 ? 40 : $host_bits; # take the smaller one } } $bits = $bits & ~1; # round down to nearest even as limit is lower with odd bit sizes # heuristic: remove 20 bits to get MB and half that as QEMU needs some overhead my $bits_to_max_mem = int(1 << ($bits - 21)); return $bits_to_max_mem > 4 * 1024 * 1024 ? 4 * 1024 * 1024 : $bits_to_max_mem; } sub get_current_memory { my ($value) = @_; my $memory = parse_memory($value); return $memory->{current}; } sub get_numa_node_list { my ($conf) = @_; my @numa_map; for (my $i = 0; $i < $MAX_NUMA; $i++) { my $entry = $conf->{"numa$i"} or next; my $numa = parse_numa($entry) or next; push @numa_map, $i; } return @numa_map if @numa_map; my $sockets = $conf->{sockets} || 1; return (0 .. ($sockets - 1)); } sub host_numanode_exists { my ($id) = @_; return -d "/sys/devices/system/node/node$id/"; } # only valid when numa nodes map to a single host node sub get_numa_guest_to_host_map { my ($conf) = @_; my $map = {}; for (my $i = 0; $i < $MAX_NUMA; $i++) { my $entry = $conf->{"numa$i"} or next; my $numa = parse_numa($entry) or next; $map->{$i} = print_numa_hostnodes($numa->{hostnodes}); } return $map if %$map; my $sockets = $conf->{sockets} || 1; return { map { $_ => $_ } (0 .. ($sockets - 1)) }; } sub foreach_dimm { my ($conf, $vmid, $memory, $static_memory, $func) = @_; my $dimm_id = 0; my $current_size = $static_memory; my $dimm_size = 0; if ($conf->{hugepages} && $conf->{hugepages} == 1024) { $dimm_size = 1024; } else { $dimm_size = 512; } return if $current_size == $memory; my @numa_map = get_numa_node_list($conf); for (my $j = 0; $j < 8; $j++) { for (my $i = 0; $i < 32; $i++) { my $name = "dimm${dimm_id}"; $dimm_id++; my $numanode = $numa_map[$i % @numa_map]; $current_size += $dimm_size; &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory); return $current_size if $current_size >= $memory; } $dimm_size *= 2; } } sub qemu_memory_hotplug { my ($vmid, $conf, $value) = @_; return $value if !PVE::QemuServer::Helpers::vm_running_locally($vmid); my $oldmem = parse_memory($conf->{memory}); my $newmem = parse_memory($value); return $value if $newmem->{current} == $oldmem->{current}; my $memory = $oldmem->{current}; $value = $newmem->{current}; my $sockets = $conf->{sockets} || 1; my $static_memory = $STATICMEM; $static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024); die "memory can't be lower than $static_memory MB" if $value < $static_memory; my $MAX_MEM = get_max_mem($conf); die "you cannot add more memory than max mem $MAX_MEM MB!\n" if $value > $MAX_MEM; if ($value > $memory) { my $numa_hostmap; foreach_dimm( $conf, $vmid, $value, $static_memory, sub { my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_; return if $current_size <= get_current_memory($conf->{memory}); if ($conf->{hugepages}) { $numa_hostmap = get_numa_guest_to_host_map($conf) if !$numa_hostmap; my $hugepages_size = hugepages_size($conf, $dimm_size); my $path = hugepages_mount_path($hugepages_size); my $host_numanode = $numa_hostmap->{$numanode}; my $hugepages_topology->{$hugepages_size}->{$host_numanode} = hugepages_nr($dimm_size, $hugepages_size); my $code = sub { my $hugepages_host_topology = hugepages_host_topology(); hugepages_allocate($hugepages_topology, $hugepages_host_topology); eval { mon_cmd( $vmid, "object-add", 'qom-type' => "memory-backend-file", id => "mem-$name", size => int($dimm_size * 1024 * 1024), 'mem-path' => $path, share => JSON::true, prealloc => JSON::true, ); }; if (my $err = $@) { hugepages_reset($hugepages_host_topology); die $err; } hugepages_pre_deallocate($hugepages_topology); }; eval { hugepages_update_locked($code); }; } else { eval { mon_cmd( $vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", size => int($dimm_size * 1024 * 1024), ); }; } if (my $err = $@) { eval { qemu_objectdel($vmid, "mem-$name"); }; die $err; } eval { mon_cmd( $vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode, ); }; if (my $err = $@) { eval { qemu_objectdel($vmid, "mem-$name"); }; die $err; } # update conf after each successful module hotplug $newmem->{current} = $current_size; $conf->{memory} = print_memory($newmem); PVE::QemuConfig->write_config($vmid, $conf); }, ); } else { my $dimms = qemu_memdevices_list($vmid, 'dimm'); my $current_size = $memory; for my $name (sort { ($b =~ /^dimm(\d+)$/)[0] <=> ($a =~ /^dimm(\d+)$/)[0] } keys %$dimms) { my $dimm_size = $dimms->{$name}->{size} / 1024 / 1024; last if $current_size <= $value; print "try to unplug memory dimm $name\n"; my $retry = 0; while (1) { eval { qemu_devicedel($vmid, $name) }; sleep 3; my $dimm_list = qemu_memdevices_list($vmid, 'dimm'); last if !$dimm_list->{$name}; raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5; $retry++; } $current_size -= $dimm_size; # update conf after each successful module unplug $newmem->{current} = $current_size; $conf->{memory} = print_memory($newmem); eval { qemu_objectdel($vmid, "mem-$name"); }; PVE::QemuConfig->write_config($vmid, $conf); } } return $conf->{memory}; } sub qemu_memdevices_list { my ($vmid, $type) = @_; my $dimmarray = mon_cmd($vmid, "query-memory-devices"); my $dimms = {}; foreach my $dimm (@$dimmarray) { next if $type && $dimm->{data}->{id} !~ /^$type(\d+)$/; $dimms->{ $dimm->{data}->{id} }->{id} = $dimm->{data}->{id}; $dimms->{ $dimm->{data}->{id} }->{node} = $dimm->{data}->{node}; $dimms->{ $dimm->{data}->{id} }->{addr} = $dimm->{data}->{addr}; $dimms->{ $dimm->{data}->{id} }->{size} = $dimm->{data}->{size}; $dimms->{ $dimm->{data}->{id} }->{slot} = $dimm->{data}->{slot}; } return $dimms; } sub config { my ($conf, $vmid, $sockets, $cores, $hotplug, $virtiofs_enabled, $cmd, $machine_flags) = @_; my $memory = get_current_memory($conf->{memory}); my $static_memory = 0; if ($hotplug) { die "NUMA needs to be enabled for memory hotplug\n" if !$conf->{numa}; my $MAX_MEM = get_max_mem($conf); die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM; for (my $i = 0; $i < $MAX_NUMA; $i++) { die "cannot enable memory hotplugging with custom NUMA topology\n" if $conf->{"numa$i"}; } my $sockets = $conf->{sockets} || 1; $static_memory = $STATICMEM; $static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024); die "minimum memory must be ${static_memory}MB\n" if ($memory < $static_memory); push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M"; } else { $static_memory = $memory; push @$cmd, '-m', $static_memory; } die "numa needs to be enabled to use hugepages" if $conf->{hugepages} && !$conf->{numa}; die "Memory hotplug does not work in combination with virtio-fs.\n" if $hotplug && $virtiofs_enabled; if ($conf->{numa}) { my $numa_totalmemory = undef; for (my $i = 0; $i < $MAX_NUMA; $i++) { next if !$conf->{"numa$i"}; my $numa = parse_numa($conf->{"numa$i"}); next if !$numa; # memory die "missing NUMA node$i memory value\n" if !$numa->{memory}; my $numa_memory = $numa->{memory}; $numa_totalmemory += $numa_memory; my $memdev = $virtiofs_enabled ? "virtiofs-mem$i" : "ram-node$i"; my $mem_object = print_mem_object($conf, $memdev, $numa_memory); # cpus my $cpulists = $numa->{cpus}; die "missing NUMA node$i cpus\n" if !defined($cpulists); my $cpus = join( ',cpus=', map { my ($start, $end) = @$_; defined($end) ? "$start-$end" : $start } @$cpulists, ); # hostnodes my $hostnodelists = $numa->{hostnodes}; if (defined($hostnodelists)) { my $hostnodes = print_numa_hostnodes($hostnodelists); # policy my $policy = $numa->{policy}; die "you need to define a policy for hostnode $hostnodes\n" if !$policy; $mem_object .= ",host-nodes=$hostnodes,policy=$policy"; } else { die "numa hostnodes need to be defined to use hugepages" if $conf->{hugepages}; } push @$cmd, '-object', $mem_object; push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=$memdev"; } die "total memory for NUMA nodes must be equal to vm static memory\n" if $numa_totalmemory && $numa_totalmemory != $static_memory; #if no custom tology, we split memory and cores across numa nodes if (!$numa_totalmemory) { my $numa_memory = ($static_memory / $sockets); for (my $i = 0; $i < $sockets; $i++) { die "host NUMA node$i doesn't exist\n" if !host_numanode_exists($i) && $conf->{hugepages}; my $cpus = ($cores * $i); $cpus .= "-" . ($cpus + $cores - 1) if $cores > 1; my $memdev = $virtiofs_enabled ? "virtiofs-mem$i" : "ram-node$i"; my $mem_object = print_mem_object($conf, $memdev, $numa_memory); push @$cmd, '-object', $mem_object; push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=$memdev"; } } } elsif ($virtiofs_enabled) { # kvm: '-machine memory-backend' and '-numa memdev' properties are mutually exclusive push @$cmd, '-object', 'memory-backend-memfd,id=virtiofs-mem' . ",size=$conf->{memory}M,share=on"; push @$machine_flags, 'memory-backend=virtiofs-mem'; } if ($hotplug) { foreach_dimm( $conf, $vmid, $memory, $static_memory, sub { my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_; my $mem_object = print_mem_object($conf, "mem-$name", $dimm_size); push @$cmd, "-object", $mem_object; push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode"; die "memory size ($memory) must be aligned to $dimm_size for hotplugging\n" if $current_size > $memory; }, ); } } sub print_mem_object { my ($conf, $id, $size) = @_; if ($conf->{hugepages}) { my $hugepages_size = hugepages_size($conf, $size); my $path = hugepages_mount_path($hugepages_size); return "memory-backend-file,id=$id,size=${size}M,mem-path=$path,share=on,prealloc=yes"; } elsif ($id =~ m/^virtiofs-mem/) { return "memory-backend-memfd,id=$id,size=${size}M,share=on"; } else { return "memory-backend-ram,id=$id,size=${size}M"; } } sub print_numa_hostnodes { my ($hostnodelists) = @_; my $hostnodes; foreach my $hostnoderange (@$hostnodelists) { my ($start, $end) = @$hostnoderange; $hostnodes .= ',' if $hostnodes; $hostnodes .= $start; $hostnodes .= "-$end" if defined($end); $end //= $start; for (my $i = $start; $i <= $end; ++$i) { die "host NUMA node$i doesn't exist\n" if !host_numanode_exists($i); } } return $hostnodes; } sub hugepages_mount { my $mountdata = PVE::ProcFSTools::parse_proc_mounts(); foreach my $size (qw(2048 1048576)) { next if (!-d "/sys/kernel/mm/hugepages/hugepages-${size}kB"); my $path = "/run/hugepages/kvm/${size}kB"; my $found = grep { $_->[2] =~ /^hugetlbfs/ && $_->[1] eq $path } @$mountdata; if (!$found) { File::Path::make_path($path) if (!-d $path); my $cmd = ['/bin/mount', '-t', 'hugetlbfs', '-o', "pagesize=${size}k", 'hugetlbfs', $path]; run_command($cmd, errmsg => "hugepage mount error"); } } } sub hugepages_mount_path { my ($size) = @_; $size = $size * 1024; return "/run/hugepages/kvm/${size}kB"; } sub hugepages_nr { my ($size, $hugepages_size) = @_; return $size / $hugepages_size; } sub hugepages_chunk_size_supported { my ($size) = @_; return -d "/sys/kernel/mm/hugepages/hugepages-" . ($size * 1024) . "kB"; } sub hugepages_size { my ($conf, $size) = @_; die "hugepages option is not enabled" if !$conf->{hugepages}; die "memory size '$size' is not a positive even integer; cannot use for hugepages\n" if $size <= 0 || $size & 1; die "your system doesn't support hugepages\n" if !hugepages_chunk_size_supported(2) && !hugepages_chunk_size_supported(1024); if ($conf->{hugepages} eq 'any') { # try to use 1GB if available && memory size is matching if (hugepages_chunk_size_supported(1024) && ($size & 1023) == 0) { return 1024; } elsif (hugepages_chunk_size_supported(2)) { return 2; } else { die "host only supports 1024 GB hugepages, but requested size '$size' is not a multiple of 1024 MB\n"; } } else { my $hugepagesize = $conf->{hugepages}; if (!hugepages_chunk_size_supported($hugepagesize)) { die "your system doesn't support hugepages of $hugepagesize MB\n"; } elsif (($size % $hugepagesize) != 0) { die "Memory size $size is not a multiple of the requested hugepages size $hugepagesize\n"; } return $hugepagesize; } } sub hugepages_topology { my ($conf, $hotplug) = @_; my $hugepages_topology = {}; return if !$conf->{numa}; my $memory = get_current_memory($conf->{memory}); my $static_memory = 0; my $sockets = $conf->{sockets} || 1; my $numa_custom_topology = undef; if ($hotplug) { $static_memory = $STATICMEM; $static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024); } else { $static_memory = $memory; } #custom numa topology for (my $i = 0; $i < $MAX_NUMA; $i++) { next if !$conf->{"numa$i"}; my $numa = parse_numa($conf->{"numa$i"}); next if !$numa; $numa_custom_topology = 1; my $numa_memory = $numa->{memory}; my $hostnodelists = $numa->{hostnodes}; my $hostnodes = print_numa_hostnodes($hostnodelists); die "more than 1 hostnode value in numa node is not supported when hugepages are enabled" if $hostnodes !~ m/^(\d)$/; my $hugepages_size = hugepages_size($conf, $numa_memory); $hugepages_topology->{$hugepages_size}->{$hostnodes} += hugepages_nr($numa_memory, $hugepages_size); } #if no custom numa tology, we split memory and cores across numa nodes if (!$numa_custom_topology) { my $numa_memory = ($static_memory / $sockets); for (my $i = 0; $i < $sockets; $i++) { my $hugepages_size = hugepages_size($conf, $numa_memory); $hugepages_topology->{$hugepages_size}->{$i} += hugepages_nr($numa_memory, $hugepages_size); } } if ($hotplug) { my $numa_hostmap = get_numa_guest_to_host_map($conf); foreach_dimm( $conf, undef, $memory, $static_memory, sub { my ($conf, undef, $name, $dimm_size, $numanode, $current_size, $memory) = @_; $numanode = $numa_hostmap->{$numanode}; my $hugepages_size = hugepages_size($conf, $dimm_size); $hugepages_topology->{$hugepages_size}->{$numanode} += hugepages_nr($dimm_size, $hugepages_size); }, ); } return $hugepages_topology; } sub hugepages_host_topology { #read host hugepages my $hugepages_host_topology = {}; dir_glob_foreach( "/sys/devices/system/node/", 'node(\d+)', sub { my ($nodepath, $numanode) = @_; dir_glob_foreach( "/sys/devices/system/node/$nodepath/hugepages/", 'hugepages\-(\d+)kB', sub { my ($hugepages_path, $hugepages_size) = @_; $hugepages_size = $hugepages_size / 1024; my $hugepages_nr = PVE::Tools::file_read_firstline( "/sys/devices/system/node/$nodepath/hugepages/$hugepages_path/nr_hugepages" ); $hugepages_host_topology->{$hugepages_size}->{$numanode} = $hugepages_nr; }, ); }, ); return $hugepages_host_topology; } sub hugepages_allocate { my ($hugepages_topology, $hugepages_host_topology) = @_; #allocate new hupages if needed foreach my $size (sort keys %$hugepages_topology) { my $nodes = $hugepages_topology->{$size}; foreach my $numanode (keys %$nodes) { my $hugepages_size = $size * 1024; my $hugepages_requested = $hugepages_topology->{$size}->{$numanode}; my $path = "/sys/devices/system/node/node${numanode}/hugepages/hugepages-${hugepages_size}kB/"; my $hugepages_free = PVE::Tools::file_read_firstline($path . "free_hugepages"); my $hugepages_nr = PVE::Tools::file_read_firstline($path . "nr_hugepages"); if ($hugepages_requested > $hugepages_free) { my $hugepages_needed = $hugepages_requested - $hugepages_free; PVE::ProcFSTools::write_proc_entry( $path . "nr_hugepages", $hugepages_nr + $hugepages_needed, ); #verify that is correctly allocated $hugepages_free = PVE::Tools::file_read_firstline($path . "free_hugepages"); if ($hugepages_free < $hugepages_requested) { #rollback to initial host config hugepages_reset($hugepages_host_topology); die "hugepage allocation failed"; } } } } } sub hugepages_default_nr_hugepages { my ($size) = @_; my $cmdline = PVE::Tools::file_read_firstline("/proc/cmdline"); my $args = PVE::Tools::split_args($cmdline); my $parsed_size = 2; # default is 2M foreach my $arg (@$args) { if ($arg eq "hugepagesz=2M") { $parsed_size = 2; } elsif ($arg eq "hugepagesz=1G") { $parsed_size = 1024; } elsif ($arg =~ m/^hugepages=(\d+)?$/) { if ($parsed_size == $size) { return $1; } } } return 0; } sub hugepages_pre_deallocate { my ($hugepages_topology) = @_; foreach my $size (sort keys %$hugepages_topology) { my $hugepages_size = $size * 1024; my $path = "/sys/kernel/mm/hugepages/hugepages-${hugepages_size}kB/"; my $hugepages_nr = hugepages_default_nr_hugepages($size); PVE::ProcFSTools::write_proc_entry($path . "nr_hugepages", $hugepages_nr); } } sub hugepages_reset { my ($hugepages_topology) = @_; foreach my $size (sort keys %$hugepages_topology) { my $nodes = $hugepages_topology->{$size}; foreach my $numanode (keys %$nodes) { my $hugepages_nr = $hugepages_topology->{$size}->{$numanode}; my $hugepages_size = $size * 1024; my $path = "/sys/devices/system/node/node${numanode}/hugepages/hugepages-${hugepages_size}kB/"; PVE::ProcFSTools::write_proc_entry($path . "nr_hugepages", $hugepages_nr); } } } sub hugepages_update_locked { my ($code, @param) = @_; my $timeout = 60; #could be long if a lot of hugepages need to be allocated my $lock_filename = "/var/lock/hugepages.lck"; my $res = lock_file($lock_filename, $timeout, $code, @param); die $@ if $@; return $res; } 1; ================================================ FILE: src/PVE/QemuServer/MetaInfo.pm ================================================ package PVE::QemuServer::MetaInfo; use strict; use warnings; use PVE::JSONSchema; use PVE::QemuServer::Helpers; our $meta_info_fmt = { 'ctime' => { type => 'integer', description => "The guest creation timestamp as UNIX epoch time", minimum => 0, optional => 1, }, 'creation-qemu' => { type => 'string', description => "The QEMU (machine) version from the time this VM was created.", pattern => '\d+(\.\d+)+', optional => 1, }, }; sub parse_meta_info { my ($value) = @_; return if !$value; my $res = eval { PVE::JSONSchema::parse_property_string($meta_info_fmt, $value) }; warn $@ if $@; return $res; } sub new_meta_info_string { my () = @_; # for now do not allow to override any value return PVE::JSONSchema::print_property_string( { 'creation-qemu' => PVE::QemuServer::Helpers::kvm_user_version(), ctime => "" . int(time()), }, $meta_info_fmt, ); } 1; ================================================ FILE: src/PVE/QemuServer/Monitor.pm ================================================ package PVE::QemuServer::Monitor; use strict; use warnings; use PVE::SafeSyslog; use PVE::QemuServer::Helpers; use PVE::QMPClient; use base 'Exporter'; our @EXPORT_OK = qw( mon_cmd qmp_cmd qsd_qmp_peer vm_qmp_peer ); =head3 qmp_cmd my $peer = { name => $name, id => $id, type => $type }; my $result = qmp_cmd($peer, $execute, %arguments); Execute the C<$qmp_command_name> with arguments C<%params> for the peer C<$peer>. The type C<$type> of the peer can be C for the QEMU instance of the VM, C for the guest agent of the VM or C for the QEMU storage daemon associated to the VM. Dies if the VM is not running or the monitor socket cannot be reached, even if the C argument is used. Returns the structured result from the QMP side converted from JSON to structured Perl data. In case the C argument is used and the QMP command failed or timed out, the result is a hash reference with an C key containing the error message. Parameters: =over =item C<$peer>: The peer to communicate with. A hash reference with: =over =item C<$name>: Name of the peer used in error messages. =item C<$id>: Identifier for the peer. The pair C<($id, $type)> uniquely identifies a peer. =item C<$type>: Type of the peer to communicate with. This can be C for the VM's QEMU instance, C for the VM's guest agent or C for the QEMU storage daemon associated to the VM. =back =item C<$execute>: The QMP command name. =item C<%arguments>: Additional arguments for the QMP command. The following custom arguments are not part of the QMP schema and supported for all commands: =over =item C: wait at most for this amount of time. If there was no actual error, the QMP/QGA command will still continue to be executed even after the timeout reached. =item C: do not die when the command gets an error or the timeout is hit. The caller needs to handle the error that is returned as a structured result. =back =back =cut sub qmp_cmd { my ($peer, $execute, %arguments) = @_; my $cmd = { execute => $execute, arguments => \%arguments }; my $res; my ($noerr, $timeout); if ($cmd->{arguments}) { ($noerr, $timeout) = delete($cmd->{arguments}->@{qw(noerr timeout)}); } eval { if ($peer->{type} eq 'qmp' || $peer->{type} eq 'qga') { die "$peer->{name} not running\n" if !PVE::QemuServer::Helpers::vm_running_locally($peer->{id}); } elsif ($peer->{type} eq 'qsd') { die "$peer->{name} not running\n" if !PVE::QemuServer::Helpers::qsd_running_locally($peer->{id}); } else { die "qmp_cmd - unknown peer type $peer->{type}\n"; } my $sname = PVE::QemuServer::Helpers::qmp_socket($peer); if (-e $sname) { # test if VM is reasonably new and supports qmp/qga my $qmpclient = PVE::QMPClient->new(); $res = $qmpclient->cmd($peer, $cmd, $timeout, $noerr); } else { die "unable to open monitor socket\n"; } }; if (my $err = $@) { syslog("err", "$peer->{name} $peer->{type} command failed - $err"); die $err; } return $res; } sub vm_qmp_peer { my ($vmid) = @_; return { name => "VM $vmid", id => $vmid, type => 'qmp' }; } sub qsd_qmp_peer { my ($id) = @_; return { name => "QEMU storage daemon $id", id => $id, type => 'qsd' }; } sub qsd_cmd { my ($id, $execute, %params) = @_; return qmp_cmd(qsd_qmp_peer($id), $execute, %params); } sub mon_cmd { my ($vmid, $execute, %params) = @_; my $type = ($execute =~ /^guest\-+/) ? 'qga' : 'qmp'; return qmp_cmd({ name => "VM $vmid", id => $vmid, type => $type }, $execute, %params); } sub hmp_cmd { my ($vmid, $cmdline, $timeout) = @_; return qmp_cmd( vm_qmp_peer($vmid), 'human-monitor-command', 'command-line' => $cmdline, timeout => $timeout, ); } 1; ================================================ FILE: src/PVE/QemuServer/Network.pm ================================================ package PVE::QemuServer::Network; use strict; use warnings; use PVE::Cluster; use PVE::Firewall::Helpers; use PVE::JSONSchema qw(get_standard_option parse_property_string); use PVE::Network::SDN::Vnets; use PVE::Network::SDN::Zones; use PVE::RESTEnvironment qw(log_warn); use PVE::Tools qw($IPV6RE file_read_firstline); use PVE::QemuServer::Monitor qw(mon_cmd); my $nic_model_list = [ 'e1000', 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em', 'e1000e', 'i82551', 'i82557b', 'i82559er', 'ne2k_isa', 'ne2k_pci', 'pcnet', 'rtl8139', 'virtio', 'vmxnet3', ]; my $net_fmt_bridge_descr = <<__EOD__; Bridge to attach the network device to. The Proxmox VE standard bridge is called 'vmbr0'. If you do not specify a bridge, we create a kvm user (NATed) network device, which provides DHCP and DNS services. The following addresses are used: 10.0.2.2 Gateway 10.0.2.3 DNS Server 10.0.2.4 SMB Server The DHCP server assign addresses to the guest starting from 10.0.2.15. __EOD__ my $net_fmt = { macaddr => get_standard_option( 'mac-addr', { description => "MAC address. That address must be unique within your network. This is" . " automatically generated if not specified.", }, ), model => { type => 'string', description => "Network Card Model. The 'virtio' model provides the best performance with" . " very low CPU overhead. If your guest does not support this driver, it is usually" . " best to use 'e1000'.", enum => $nic_model_list, default_key => 1, }, (map { $_ => { keyAlias => 'model', alias => 'macaddr' } } @$nic_model_list), bridge => get_standard_option( 'pve-bridge-id', { description => $net_fmt_bridge_descr, optional => 1, }, ), queues => { type => 'integer', minimum => 0, maximum => 64, description => 'Number of packet queues to be used on the device.', optional => 1, }, rate => { type => 'number', minimum => 0, description => "Rate limit in mbps (megabytes per second) as floating point number.", optional => 1, }, tag => { type => 'integer', minimum => 1, maximum => 4094, description => 'VLAN tag to apply to packets on this interface.', optional => 1, }, trunks => { type => 'string', pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/, description => 'VLAN trunks to pass through this interface.', format_description => 'vlanid[;vlanid...]', optional => 1, }, firewall => { type => 'boolean', description => 'Whether this interface should be protected by the firewall.', optional => 1, }, link_down => { type => 'boolean', description => 'Whether this interface should be disconnected (like pulling the plug).', optional => 1, }, mtu => { type => 'integer', minimum => 1, maximum => 65520, description => "Force MTU of network device (VirtIO only). Setting to '1' or empty will use the bridge MTU", optional => 1, }, }; our $netdesc = { optional => 1, type => 'string', format => $net_fmt, description => "Specify network devices.", }; PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc); my $ipconfig_fmt = { ip => { type => 'string', format => 'pve-ipv4-config', format_description => 'IPv4Format/CIDR', description => 'IPv4 address in CIDR format.', optional => 1, default => 'dhcp', }, gw => { type => 'string', format => 'ipv4', format_description => 'GatewayIPv4', description => 'Default gateway for IPv4 traffic.', optional => 1, requires => 'ip', }, ip6 => { type => 'string', format => 'pve-ipv6-config', format_description => 'IPv6Format/CIDR', description => 'IPv6 address in CIDR format.', optional => 1, default => 'dhcp', }, gw6 => { type => 'string', format => 'ipv6', format_description => 'GatewayIPv6', description => 'Default gateway for IPv6 traffic.', optional => 1, requires => 'ip6', }, }; PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt); our $ipconfigdesc = { optional => 1, type => 'string', format => 'pve-qm-ipconfig', description => <<'EODESCR', cloud-init: Specify IP addresses and gateways for the corresponding interface. IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified. The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided. For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires cloud-init 19.4 or newer. If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4. EODESCR }; # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate= sub parse_net { my ($data, $disable_mac_autogen) = @_; my $res = eval { parse_property_string($net_fmt, $data) }; if ($@) { warn $@; return; } if (!defined($res->{macaddr}) && !$disable_mac_autogen) { my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg'); $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}); } return $res; } # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip sub parse_ipconfig { my ($data) = @_; my $res = eval { parse_property_string($ipconfig_fmt, $data) }; if ($@) { warn $@; return; } if ($res->{gw} && !$res->{ip}) { warn 'gateway specified without specifying an IP address'; return; } if ($res->{gw6} && !$res->{ip6}) { warn 'IPv6 gateway specified without specifying an IPv6 address'; return; } if ($res->{gw} && $res->{ip} eq 'dhcp') { warn 'gateway specified together with DHCP'; return; } if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) { # gw6 + auto/dhcp warn "IPv6 gateway specified together with $res->{ip6} address"; return; } if (!$res->{ip} && !$res->{ip6}) { return { ip => 'dhcp', ip6 => 'dhcp' }; } return $res; } sub print_net { my $net = shift; return PVE::JSONSchema::print_property_string($net, $net_fmt); } sub add_random_macs { my ($settings) = @_; foreach my $opt (keys %$settings) { next if $opt !~ m/^net(\d+)$/; my $net = parse_net($settings->{$opt}); next if !$net; $settings->{$opt} = print_net($net); } } sub add_nets_bridge_fdb { my ($conf, $vmid) = @_; for my $opt (keys %$conf) { next if $opt !~ m/^net(\d+)$/; my $iface = "tap${vmid}i$1"; # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start my $net = parse_net($conf->{$opt}, 1) or next; my $mac = $net->{macaddr}; if (!$mac) { log_warn( "MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!" ) if !file_read_firstline("/sys/class/net/$iface/brport/learning"); next; } my $bridge = $net->{bridge}; if (!$bridge) { log_warn("Interface '$iface' not attached to any bridge."); next; } PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge); } } sub del_nets_bridge_fdb { my ($conf, $vmid) = @_; for my $opt (keys %$conf) { next if $opt !~ m/^net(\d+)$/; my $iface = "tap${vmid}i$1"; my $net = parse_net($conf->{$opt}) or next; my $mac = $net->{macaddr} or next; my $bridge = $net->{bridge}; PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge); } } sub create_ifaces_ipams_ips { my ($conf, $vmid) = @_; foreach my $opt (keys %$conf) { if ($opt =~ m/^net(\d+)$/) { my $value = $conf->{$opt}; my $net = parse_net($value); eval { PVE::Network::SDN::Vnets::add_next_free_cidr( $net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1, ); }; warn $@ if $@; } } } sub delete_ifaces_ipams_ips { my ($conf, $vmid) = @_; foreach my $opt (keys %$conf) { if ($opt =~ m/^net(\d+)$/) { my $net = parse_net($conf->{$opt}); eval { PVE::Network::SDN::Vnets::del_ips_from_mac( $net->{bridge}, $net->{macaddr}, $conf->{name}, ); }; warn $@ if $@; } } } sub tap_plug { my ($iface, $bridge, $tag, $firewall, $trunks, $rate) = @_; $firewall = $firewall && PVE::Firewall::Helpers::needs_fwbr($bridge); PVE::Network::SDN::Zones::tap_plug($iface, $bridge, $tag, $firewall, $trunks, $rate); } sub get_nets_host_mtu { my ($vmid, $conf) = @_; my $nets_host_mtu = []; for my $opt (sort keys $conf->%*) { next if $opt !~ m/^net(\d+)$/; my $net = parse_net($conf->{$opt}); next if $net->{model} ne 'virtio'; my $host_mtu = eval { mon_cmd( $vmid, 'qom-get', path => "/machine/peripheral/$opt", property => 'host_mtu', ); }; if (my $err = $@) { log_warn("$opt: could not query host_mtu - $err"); } elsif (defined($host_mtu)) { push $nets_host_mtu->@*, "${opt}=${host_mtu}"; } else { log_warn("$opt: got undefined value when querying host_mtu"); } } return join(',', $nets_host_mtu->@*); } 1; ================================================ FILE: src/PVE/QemuServer/OVMF.pm ================================================ package PVE::QemuServer::OVMF; use strict; use warnings; use JSON qw(to_json); use PVE::File qw(file_exists file_get_size); use PVE::GuestHelpers qw(safe_string_ne); use PVE::RESTEnvironment qw(log_warn); use PVE::Storage; use PVE::Tools; use PVE::QemuServer::Blockdev; use PVE::QemuServer::Drive qw(checked_volume_format parse_drive print_drive); use PVE::QemuServer::Helpers; use PVE::QemuServer::QemuImage; use PVE::QemuServer::QSD; my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/'; my $OVMF = { x86_64 => { '4m-no-smm' => [ "$EDK2_FW_BASE/OVMF_CODE_4M.fd", "$EDK2_FW_BASE/OVMF_VARS_4M.fd", ], '4m-no-smm-ms' => [ "$EDK2_FW_BASE/OVMF_CODE_4M.fd", "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd", ], '4m' => [ "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd", "$EDK2_FW_BASE/OVMF_VARS_4M.fd", ], '4m-ms' => [ "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd", "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd", ], '4m-sev' => [ "$EDK2_FW_BASE/OVMF_SEV_CODE_4M.fd", "$EDK2_FW_BASE/OVMF_SEV_VARS_4M.fd", ], '4m-snp' => [ "$EDK2_FW_BASE/OVMF_SEV_4M.fd", ], '4m-tdx' => [ "$EDK2_FW_BASE/OVMF_TDX_4M.ms.fd", ], # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build # anymore. how can we deperacate this sanely without breaking existing instances, or using # older backups and snapshot? default => [ "$EDK2_FW_BASE/OVMF_CODE.fd", "$EDK2_FW_BASE/OVMF_VARS.fd", ], }, aarch64 => { default => [ "$EDK2_FW_BASE/AAVMF_CODE.fd", "$EDK2_FW_BASE/AAVMF_VARS.fd", ], }, }; my sub get_ovmf_files($$$$) { my ($arch, $efidisk, $smm, $cvm_type) = @_; my $types = $OVMF->{$arch} or die "no OVMF images known for architecture '$arch'\n"; my $type = 'default'; if ($arch eq 'x86_64') { if ($cvm_type && $cvm_type eq 'snp') { $type = "4m-snp"; my ($ovmf) = $types->{$type}->@*; die "EFI base image '$ovmf' not found\n" if !file_exists($ovmf); return ($ovmf); } elsif ($cvm_type && ($cvm_type eq 'std' || $cvm_type eq 'es')) { $type = "4m-sev"; } elsif ($cvm_type && $cvm_type eq 'tdx') { $type = "4m-tdx"; my ($ovmf) = $types->{$type}->@*; die "EFI base image '$ovmf' not found\n" if !file_exists($ovmf); return ($ovmf); } elsif (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') { $type = $smm ? "4m" : "4m-no-smm"; $type .= '-ms' if $efidisk->{'pre-enrolled-keys'}; } else { # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9 } } my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*; die "EFI base image '$ovmf_code' not found\n" if !file_exists($ovmf_code); die "EFI vars image '$ovmf_vars' not found\n" if !file_exists($ovmf_vars); return ($ovmf_code, $ovmf_vars); } my sub print_ovmf_drive_commandlines { my ($conf, $storecfg, $vmid, $hw_info, $version_guard, $readonly) = @_; my ($cvm_type, $arch, $q35) = $hw_info->@{qw(cvm-type arch q35)}; my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef; die "Attempting to configure SEV-SNP with pflash devices instead of using `-bios`\n" if $cvm_type && $cvm_type eq 'snp'; die "Attempting to configure TDX with pflash devices instead of using `-bios`\n" if $cvm_type && $cvm_type eq 'tdx'; my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35, $cvm_type); my $ovmf_vars_size = file_get_size($ovmf_vars); my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0"; if ($d) { my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1); my ($path, $format) = $d->@{ 'file', 'format' }; if ($storeid) { $path = PVE::Storage::path($storecfg, $d->{file}); $format //= checked_volume_format($storecfg, $d->{file}); } elsif (!defined($format)) { die "efidisk format must be specified\n"; } # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329 if ($path =~ m/^rbd:/) { $var_drive_str .= ',cache=writeback'; $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too } $var_drive_str .= ",format=$format,file=$path"; $var_drive_str .= ",size=" . $ovmf_vars_size if $format eq 'raw' && $version_guard->(4, 1, 2); $var_drive_str .= ',readonly=on' if $readonly; } else { log_warn("no efidisk configured! Using temporary efivars disk."); my $path = "/tmp/$vmid-ovmf.fd"; PVE::Tools::file_copy($ovmf_vars, $path, $ovmf_vars_size); $var_drive_str .= ",format=raw,file=$path"; $var_drive_str .= ",size=" . $ovmf_vars_size if $version_guard->(4, 1, 2); } return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str); } sub get_efivars_size { my ($arch, $efidisk, $smm, $cvm_type) = @_; my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm, $cvm_type); return file_get_size($ovmf_vars); } my sub is_ms_2023_cert_enrolled { my ($path) = @_; my $inside_db_section; my $found_ms_2023_cert; my $detect_ms_2023_cert = sub { my ($line) = @_; return if $found_ms_2023_cert; $inside_db_section = undef if !$line; $found_ms_2023_cert = 1 if $inside_db_section && $line =~ m/CN=Microsoft UEFI CA 2023/; $inside_db_section = 1 if $line =~ m/^name=db guid=guid:EfiImageSecurityDatabase/; return; }; PVE::Tools::run_command( ['virt-fw-vars', '--input', $path, '--print', '--verbose'], outfunc => $detect_ms_2023_cert, ); return $found_ms_2023_cert; } sub create_efidisk($$$$$$$$) { my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm, $cvm_type) = @_; my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm, $cvm_type); my $vars_size_b = file_get_size($ovmf_vars); my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb'); my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size); PVE::Storage::activate_volumes($storecfg, [$volid]); PVE::QemuServer::QemuImage::convert($ovmf_vars, $volid, $vars_size_b); my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3); if ($efidisk->{'pre-enrolled-keys'} && is_ms_2023_cert_enrolled($ovmf_vars)) { $efidisk->{'ms-cert'} = '2023k'; } return ($volid, $size / 1024); } my sub generate_ovmf_blockdev { my ($conf, $storecfg, $vmid, $hw_info, $readonly) = @_; my ($cvm_type, $arch, $machine_version, $q35) = $hw_info->@{qw(cvm-type arch machine-version q35)}; my $drive = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef; die "Attempting to configure SEV-SNP with pflash devices instead of using `-bios`\n" if $cvm_type && $cvm_type eq 'snp'; my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $drive, $q35, $cvm_type); my $ovmf_code_blockdev = { driver => 'raw', file => { driver => 'file', filename => "$ovmf_code" }, 'node-name' => 'pflash0', 'read-only' => JSON::true, }; my $format; if ($drive) { my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1); $format = $drive->{format}; if ($storeid) { $format //= checked_volume_format($storecfg, $drive->{file}); } elsif (!defined($format)) { die "efidisk format must be specified\n"; } } else { log_warn("no efidisk configured! Using temporary efivars disk."); my $path = "/tmp/$vmid-ovmf.fd"; PVE::Tools::file_copy($ovmf_vars, $path, file_get_size($ovmf_vars)); $drive = { file => $path, interface => 'efidisk', index => 0 }; $format = 'raw'; } # Prior to -blockdev, QEMU's default 'writeback' cache mode was used for EFI disks, rather than # the Proxmox VE default 'none'. Use that for -blockdev too, to avoid bug #3329. $drive->{cache} = 'writeback' if !$drive->{cache}; my $extra_blockdev_options = {}; $extra_blockdev_options->{'read-only'} = 1 if $readonly; $extra_blockdev_options->{size} = file_get_size($ovmf_vars) if $format eq 'raw'; my $throttle_group = PVE::QemuServer::Blockdev::generate_throttle_group($drive); my $ovmf_vars_blockdev = PVE::QemuServer::Blockdev::generate_drive_blockdev( $storecfg, $drive, $machine_version, $extra_blockdev_options, ); return ($ovmf_code_blockdev, $ovmf_vars_blockdev, $throttle_group); } sub print_ovmf_commandline { my ($conf, $storecfg, $vmid, $hw_info, $version_guard, $readonly) = @_; my $cvm_type = $hw_info->{'cvm-type'}; my $cmd = []; my $machine_flags = []; if ($cvm_type && ($cvm_type eq 'snp' || $cvm_type eq 'tdx')) { if (defined($conf->{efidisk0})) { log_warn( "EFI disks are not supported with Confidential Virtual Machines and will be ignored" ); } push $cmd->@*, '-bios', get_ovmf_files($hw_info->{arch}, undef, undef, $cvm_type); } else { if ($version_guard->(10, 0, 0)) { # for the switch to -blockdev my ($code_blockdev, $vars_blockdev, $throttle_group) = generate_ovmf_blockdev($conf, $storecfg, $vmid, $hw_info, $readonly); push $cmd->@*, '-object', to_json($throttle_group, { canonical => 1 }); push $cmd->@*, '-blockdev', to_json($code_blockdev, { canonical => 1 }); push $cmd->@*, '-blockdev', to_json($vars_blockdev, { canonical => 1 }); push $machine_flags->@*, "pflash0=$code_blockdev->{'node-name'}", "pflash1=$vars_blockdev->{'node-name'}"; } else { my ($code_drive_str, $var_drive_str) = print_ovmf_drive_commandlines( $conf, $storecfg, $vmid, $hw_info, $version_guard, $readonly, ); push $cmd->@*, '-drive', $code_drive_str; push $cmd->@*, '-drive', $var_drive_str; } } return ($cmd, $machine_flags); } sub should_enroll_ms_2023_cert { my ($efidisk) = @_; return if !$efidisk->{'pre-enrolled-keys'}; return if $efidisk->{'ms-cert'} && $efidisk->{'ms-cert'} eq '2023k'; return 1; } sub ensure_ms_2023_cert_enrolled { my ($storecfg, $vmid, $efidisk) = @_; return if !should_enroll_ms_2023_cert($efidisk); print "efidisk0: enrolling Microsoft UEFI CA 2023\n"; my $qsd_id = "vm-$vmid-efi-enroll"; if (my $qsd_pid = PVE::QemuServer::Helpers::qsd_running_locally($qsd_id)) { die "QEMU storage daemon $qsd_id already running with PID $qsd_pid (left over process?)\n"; } PVE::QemuServer::QSD::start($qsd_id); eval { # virt-fw-vars will only apply the --microsoft-kek option when combined with # --enroll-{cert,generate,redhat}. That requires also specifying a platform key, so instead # use the --add-kek option. my $ms_2023_kek_path = '/usr/lib/python3/dist-packages/virt/firmware/certs/' . 'MicrosoftCorporationKEK2KCA2023.pem'; # Taken from guids.py in the virt-fw-vars sources. my $ms_vendor_guid = '77fa9abd-0359-4d32-bd60-28f4e78f784b'; my $efi_vars_path = PVE::QemuServer::QSD::add_fuse_export($qsd_id, $efidisk, 'efidisk0-enroll'); PVE::Tools::run_command( [ 'virt-fw-vars', '--inplace', $efi_vars_path, '--distro-keys', 'ms-uefi', '--distro-keys', 'windows', '--add-kek', $ms_vendor_guid, $ms_2023_kek_path, ], ); PVE::QemuServer::QSD::remove_fuse_export($qsd_id, 'efidisk0-enroll'); }; my $err = $@; PVE::QemuServer::QSD::quit($qsd_id); die "efidisk0: enrolling Microsoft UEFI CA 2023 failed - $err" if $err; $efidisk->{'ms-cert'} = '2023k'; return $efidisk; } sub drive_change { my ($storecfg, $vmid, $old_drive, $new_drive) = @_; if ( $old_drive->{file} eq $new_drive->{file} # change affecting the same volume && safe_string_ne($old_drive->{'ms-cert'}, $new_drive->{'ms-cert'}) # ms-cert changed && $new_drive->{'ms-cert'} && $new_drive->{'ms-cert'} =~ m/^2023/ ) { # The ms-cert marker was newly changed to 2023, ensure it's enrolled. Clear it first to # avoid detecting as already enrolled. delete $new_drive->{'ms-cert'}; ensure_ms_2023_cert_enrolled($storecfg, $vmid, $new_drive); } # Otherwise, there is nothing special to do. Note that changing away from ms-cert=2023 is # allowed too, the marker is not the source of truth. } 1; ================================================ FILE: src/PVE/QemuServer/PCI.pm ================================================ package PVE::QemuServer::PCI; use warnings; use strict; use IO::File; use PVE::JSONSchema; use PVE::Mapping::PCI; use PVE::SysFSTools; use PVE::Tools; use PVE::QemuServer::Helpers; use PVE::QemuServer::Machine; use base 'Exporter'; our @EXPORT_OK = qw( print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci ); our $MAX_HOSTPCI_DEVICES = 16; my $PCIRE = qr/(?:[a-f0-9]{4,}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/; my $hostpci_fmt = { host => { default_key => 1, optional => 1, type => 'string', pattern => qr/$PCIRE(;$PCIRE)*/, format_description => 'HOSTPCIID[;HOSTPCIID2...]', description => < { optional => 1, type => 'string', format_description => 'mapping-id', format => 'pve-configid', description => "The ID of a cluster wide mapping. Either this or the default-key 'host'" . " must be set.", }, rombar => { type => 'boolean', description => "Specify whether or not the device's ROM will be visible in the" . " guest's memory map.", optional => 1, default => 1, }, romfile => { type => 'string', pattern => '[^,;]+', format_description => 'string', description => "Custom pci device rom filename (must be located in /usr/share/kvm/).", optional => 1, }, pcie => { type => 'boolean', description => "Choose the PCI-express bus (needs the 'q35' machine model).", optional => 1, default => 0, }, 'x-vga' => { type => 'boolean', description => "Enable vfio-vga device support.", optional => 1, default => 0, }, 'legacy-igd' => { type => 'boolean', description => "Pass this device in legacy IGD mode, making it the primary and exclusive" . " graphics device in the VM. Requires 'pc-i440fx' machine type and VGA set to 'none'.", optional => 1, default => 0, }, 'mdev' => { type => 'string', format_description => 'string', pattern => '[^/\.:]+', optional => 1, description => < { type => 'string', pattern => qr/^0x[0-9a-fA-F]{4}$/, format_description => 'hex id', optional => 1, description => "Override PCI vendor ID visible to guest", }, 'device-id' => { type => 'string', pattern => qr/^0x[0-9a-fA-F]{4}$/, format_description => 'hex id', optional => 1, description => "Override PCI device ID visible to guest", }, 'sub-vendor-id' => { type => 'string', pattern => qr/^0x[0-9a-fA-F]{4}$/, format_description => 'hex id', optional => 1, description => "Override PCI subsystem vendor ID visible to guest", }, 'sub-device-id' => { type => 'string', pattern => qr/^0x[0-9a-fA-F]{4}$/, format_description => 'hex id', optional => 1, description => "Override PCI subsystem device ID visible to guest", }, 'driver' => { type => 'string', optional => 1, default => 'vfio', enum => [qw(vfio keep)], description => "If set to 'keep' the device will neither be reset nor bound to the " . "'vfio-pci' driver. Useful for devices that already have the correct driver loaded.", }, }; PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt); our $hostpcidesc = { optional => 1, type => 'string', format => 'pve-qm-hostpci', description => "Map host PCI devices into guest.", verbose_description => < { bus => 0, addr => 1, conflict_ok => qw(ehci) }, ehci => { bus => 0, addr => 1, conflict_ok => qw(piix3) }, # instead of piix3 on arm vga => { bus => 0, addr => 2, conflict_ok => qw(legacy-igd) }, 'legacy-igd' => { bus => 0, addr => 2, conflict_ok => qw(vga) }, # legacy-igd requires vga=none balloon0 => { bus => 0, addr => 3 }, watchdog => { bus => 0, addr => 4 }, scsihw0 => { bus => 0, addr => 5, conflict_ok => qw(pci.3) }, 'pci.3' => { bus => 0, addr => 5, conflict_ok => qw(scsihw0) }, # also used for virtio-scsi-single bridge scsihw1 => { bus => 0, addr => 6 }, ahci0 => { bus => 0, addr => 7 }, qga0 => { bus => 0, addr => 8 }, spice => { bus => 0, addr => 9 }, virtio0 => { bus => 0, addr => 10 }, virtio1 => { bus => 0, addr => 11 }, virtio2 => { bus => 0, addr => 12 }, virtio3 => { bus => 0, addr => 13 }, virtio4 => { bus => 0, addr => 14 }, virtio5 => { bus => 0, addr => 15 }, hostpci0 => { bus => 0, addr => 16 }, hostpci1 => { bus => 0, addr => 17 }, net0 => { bus => 0, addr => 18 }, net1 => { bus => 0, addr => 19 }, net2 => { bus => 0, addr => 20 }, net3 => { bus => 0, addr => 21 }, net4 => { bus => 0, addr => 22 }, net5 => { bus => 0, addr => 23 }, vga1 => { bus => 0, addr => 24 }, vga2 => { bus => 0, addr => 25 }, vga3 => { bus => 0, addr => 26 }, hostpci2 => { bus => 0, addr => 27 }, hostpci3 => { bus => 0, addr => 28 }, #addr29 : usb-host (pve-usb.cfg) 'pci.1' => { bus => 0, addr => 30 }, 'pci.2' => { bus => 0, addr => 31 }, 'net6' => { bus => 1, addr => 1 }, 'net7' => { bus => 1, addr => 2 }, 'net8' => { bus => 1, addr => 3 }, 'net9' => { bus => 1, addr => 4 }, 'net10' => { bus => 1, addr => 5 }, 'net11' => { bus => 1, addr => 6 }, 'net12' => { bus => 1, addr => 7 }, 'net13' => { bus => 1, addr => 8 }, 'net14' => { bus => 1, addr => 9 }, 'net15' => { bus => 1, addr => 10 }, 'net16' => { bus => 1, addr => 11 }, 'net17' => { bus => 1, addr => 12 }, 'net18' => { bus => 1, addr => 13 }, 'net19' => { bus => 1, addr => 14 }, 'net20' => { bus => 1, addr => 15 }, 'net21' => { bus => 1, addr => 16 }, 'net22' => { bus => 1, addr => 17 }, 'net23' => { bus => 1, addr => 18 }, 'net24' => { bus => 1, addr => 19 }, 'net25' => { bus => 1, addr => 20 }, 'net26' => { bus => 1, addr => 21 }, 'net27' => { bus => 1, addr => 22 }, 'net28' => { bus => 1, addr => 23 }, 'net29' => { bus => 1, addr => 24 }, 'net30' => { bus => 1, addr => 25 }, 'net31' => { bus => 1, addr => 26 }, 'xhci' => { bus => 1, addr => 27 }, 'pci.4' => { bus => 1, addr => 28 }, 'rng0' => { bus => 1, addr => 29 }, 'pci.2-igd' => { bus => 1, addr => 30 }, # replaces pci.2 in case a legacy IGD device is passed through 'virtio6' => { bus => 2, addr => 1 }, 'virtio7' => { bus => 2, addr => 2 }, 'virtio8' => { bus => 2, addr => 3 }, 'virtio9' => { bus => 2, addr => 4 }, 'virtio10' => { bus => 2, addr => 5 }, 'virtio11' => { bus => 2, addr => 6 }, 'virtio12' => { bus => 2, addr => 7 }, 'virtio13' => { bus => 2, addr => 8 }, 'virtio14' => { bus => 2, addr => 9 }, 'virtio15' => { bus => 2, addr => 10 }, 'ivshmem' => { bus => 2, addr => 11 }, 'audio0' => { bus => 2, addr => 12 }, hostpci4 => { bus => 2, addr => 13 }, hostpci5 => { bus => 2, addr => 14 }, hostpci6 => { bus => 2, addr => 15 }, hostpci7 => { bus => 2, addr => 16 }, hostpci8 => { bus => 2, addr => 17 }, hostpci9 => { bus => 2, addr => 18 }, hostpci10 => { bus => 2, addr => 19 }, hostpci11 => { bus => 2, addr => 20 }, hostpci12 => { bus => 2, addr => 21 }, hostpci13 => { bus => 2, addr => 22 }, hostpci14 => { bus => 2, addr => 23 }, hostpci15 => { bus => 2, addr => 24 }, 'virtioscsi0' => { bus => 3, addr => 1 }, 'virtioscsi1' => { bus => 3, addr => 2 }, 'virtioscsi2' => { bus => 3, addr => 3 }, 'virtioscsi3' => { bus => 3, addr => 4 }, 'virtioscsi4' => { bus => 3, addr => 5 }, 'virtioscsi5' => { bus => 3, addr => 6 }, 'virtioscsi6' => { bus => 3, addr => 7 }, 'virtioscsi7' => { bus => 3, addr => 8 }, 'virtioscsi8' => { bus => 3, addr => 9 }, 'virtioscsi9' => { bus => 3, addr => 10 }, 'virtioscsi10' => { bus => 3, addr => 11 }, 'virtioscsi11' => { bus => 3, addr => 12 }, 'virtioscsi12' => { bus => 3, addr => 13 }, 'virtioscsi13' => { bus => 3, addr => 14 }, 'virtioscsi14' => { bus => 3, addr => 15 }, 'virtioscsi15' => { bus => 3, addr => 16 }, 'virtioscsi16' => { bus => 3, addr => 17 }, 'virtioscsi17' => { bus => 3, addr => 18 }, 'virtioscsi18' => { bus => 3, addr => 19 }, 'virtioscsi19' => { bus => 3, addr => 20 }, 'virtioscsi20' => { bus => 3, addr => 21 }, 'virtioscsi21' => { bus => 3, addr => 22 }, 'virtioscsi22' => { bus => 3, addr => 23 }, 'virtioscsi23' => { bus => 3, addr => 24 }, 'virtioscsi24' => { bus => 3, addr => 25 }, 'virtioscsi25' => { bus => 3, addr => 26 }, 'virtioscsi26' => { bus => 3, addr => 27 }, 'virtioscsi27' => { bus => 3, addr => 28 }, 'virtioscsi28' => { bus => 3, addr => 29 }, 'virtioscsi29' => { bus => 3, addr => 30 }, 'virtioscsi30' => { bus => 3, addr => 31 }, 'scsihw2' => { bus => 4, addr => 1 }, 'scsihw3' => { bus => 4, addr => 2 }, 'scsihw4' => { bus => 4, addr => 3 }, } if !defined($pci_addr_map); return $pci_addr_map; } sub generate_mdev_uuid { my ($vmid, $index) = @_; return sprintf("%08d-0000-0000-0000-%012d", $index, $vmid); } my $get_addr_mapping_from_id = sub { my ($map, $id) = @_; my $d = $map->{$id}; return if !defined($d) || !defined($d->{bus}) || !defined($d->{addr}); return { bus => $d->{bus}, addr => sprintf("0x%x", $d->{addr}) }; }; sub print_pci_addr { my ($id, $bridges, $arch) = @_; die "aarch64 cannot use IDE devices\n" if $arch eq 'aarch64' && $id =~ /^ide/; my $res = ''; my $map = get_pci_addr_map(); if (my $d = $get_addr_mapping_from_id->($map, $id)) { # Using same bus slots on all HW, so we need to check special cases here. For aarch64, the # virt machine has an initial pcie.0. The other pci bridges that get added are called pci.N. my $busname = $arch eq 'aarch64' && $d->{bus} eq 0 ? 'pcie' : 'pci'; $res = ",bus=$busname.$d->{bus},addr=$d->{addr}"; $bridges->{ $d->{bus} } = 1 if $bridges; } return $res; } my $pcie_addr_map; sub get_pcie_addr_map { $pcie_addr_map = { vga => { bus => 'pcie.0', addr => 1 }, hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 }, hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 }, hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 }, hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 }, hostpci4 => { bus => "ich9-pcie-port-5", addr => 0 }, hostpci5 => { bus => "ich9-pcie-port-6", addr => 0 }, hostpci6 => { bus => "ich9-pcie-port-7", addr => 0 }, hostpci7 => { bus => "ich9-pcie-port-8", addr => 0 }, hostpci8 => { bus => "ich9-pcie-port-9", addr => 0 }, hostpci9 => { bus => "ich9-pcie-port-10", addr => 0 }, hostpci10 => { bus => "ich9-pcie-port-11", addr => 0 }, hostpci11 => { bus => "ich9-pcie-port-12", addr => 0 }, hostpci12 => { bus => "ich9-pcie-port-13", addr => 0 }, hostpci13 => { bus => "ich9-pcie-port-14", addr => 0 }, hostpci14 => { bus => "ich9-pcie-port-15", addr => 0 }, hostpci15 => { bus => "ich9-pcie-port-16", addr => 0 }, # win7 is picky about pcie assignments hostpci0bus0 => { bus => "pcie.0", addr => 16 }, hostpci1bus0 => { bus => "pcie.0", addr => 17 }, hostpci2bus0 => { bus => "pcie.0", addr => 18 }, hostpci3bus0 => { bus => "pcie.0", addr => 19 }, ivshmem => { bus => 'pcie.0', addr => 20 }, hostpci4bus0 => { bus => "pcie.0", addr => 9 }, hostpci5bus0 => { bus => "pcie.0", addr => 10 }, hostpci6bus0 => { bus => "pcie.0", addr => 11 }, hostpci7bus0 => { bus => "pcie.0", addr => 12 }, hostpci8bus0 => { bus => "pcie.0", addr => 13 }, hostpci9bus0 => { bus => "pcie.0", addr => 14 }, hostpci10bus0 => { bus => "pcie.0", addr => 15 }, hostpci11bus0 => { bus => "pcie.0", addr => 21 }, hostpci12bus0 => { bus => "pcie.0", addr => 22 }, hostpci13bus0 => { bus => "pcie.0", addr => 23 }, hostpci14bus0 => { bus => "pcie.0", addr => 24 }, hostpci15bus0 => { bus => "pcie.0", addr => 25 }, } if !defined($pcie_addr_map); return $pcie_addr_map; } sub print_pcie_addr { my ($id) = @_; my $res = ''; my $map = get_pcie_addr_map($id); if (my $d = $get_addr_mapping_from_id->($map, $id)) { $res = ",bus=$d->{bus},addr=$d->{addr}"; } return $res; } # Generates the device strings for additional pcie root ports. The first 4 pcie # root ports are defined in the pve-q35*.cfg files. sub print_pcie_root_port { my ($i) = @_; my $res = ''; my $root_port_addresses = { 4 => "10.0", 5 => "10.1", 6 => "10.2", 7 => "10.3", 8 => "10.4", 9 => "10.5", 10 => "10.6", 11 => "10.7", 12 => "11.0", 13 => "11.1", 14 => "11.2", 15 => "11.3", }; if (defined($root_port_addresses->{$i})) { my $id = $i + 1; $res = "pcie-root-port,id=ich9-pcie-port-${id}"; $res .= ",addr=$root_port_addresses->{$i}"; $res .= ",x-speed=16,x-width=32,multifunction=on,bus=pcie.0"; $res .= ",port=${id},chassis=${id}"; } return $res; } # returns the parsed pci config but parses the 'host' part into # a list if lists into the 'id' property like this: # # { # mdev => 1, # rombar => ... # ... # ids => [ # # this contains a list of alternative devices, # [ # # which are itself lists of ids for one multifunction device # { # id => "0000:00:00.0", # vendor => "...", # }, # { # id => "0000:00:00.1", # vendor => "...", # }, # ], # [ # ... # ], # ... # ], # } sub parse_hostpci { my ($value) = @_; return if !$value; my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value); my $alternatives = []; my $host = delete $res->{host}; my $mapping = delete $res->{mapping}; die "Cannot set both 'host' and 'mapping'.\n" if defined($host) && defined($mapping); if ($mapping) { # we have no ordinary pci id, must be a mapping my $devices = PVE::Mapping::PCI::find_on_current_node($mapping); die "PCI device mapping not found for '$mapping'\n" if !$devices || !scalar($devices->@*); my $config = PVE::Mapping::PCI::config(); my $mapping_cfg = $config->{ids}->{$mapping}; $res->{'live-migration-capable'} = 1 if $mapping_cfg->{'live-migration-capable'}; for my $device ($devices->@*) { eval { PVE::Mapping::PCI::assert_valid($mapping, $device, $mapping_cfg) }; die "PCI device mapping invalid (hardware probably changed): $@\n" if $@; push $alternatives->@*, [split(/;/, $device->{path})]; } } elsif ($host) { push $alternatives->@*, [split(/;/, $host)]; } else { die "Either 'host' or 'mapping' must be set.\n"; } $res->{ids} = []; for my $alternative ($alternatives->@*) { my $ids = []; foreach my $id ($alternative->@*) { my $devs = PVE::SysFSTools::lspci($id); die "no PCI device found for '$id'\n" if !scalar($devs->@*); push $ids->@*, @$devs; } if (scalar($ids->@*) > 1) { $res->{'has-multifunction'} = 1; die "cannot use mediated device with multifunction device\n" if $res->{mdev} || $res->{nvidia}; } elsif ($res->{mdev}) { if ($ids->[0]->{nvidia} && $res->{mdev} =~ m/^nvidia-(\d+)$/) { $res->{nvidia} = $1; delete $res->{mdev}; } } push $res->{ids}->@*, $ids; } return $res; } # parses all hostpci devices from a config and does some sanity checks # returns a hash like this: # { # hostpci0 => { # # hash from parse_hostpci function # }, # hostpci1 => { ... }, # ... # } sub parse_hostpci_devices { my ($conf) = @_; my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf); my $legacy_igd = 0; my $parsed_devices = {}; for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) { my $id = "hostpci$i"; my $d = parse_hostpci($conf->{$id}); next if !$d; # check syntax die "q35 machine model is not enabled" if !$q35 && $d->{pcie}; if ($d->{'legacy-igd'}) { die "only one device can be assigned in legacy-igd mode\n" if $legacy_igd; $legacy_igd = 1; die "legacy IGD assignment requires VGA mode to be 'none'\n" if !defined($conf->{'vga'}) || $conf->{'vga'} ne 'none'; die "legacy IGD assignment requires rombar to be enabled\n" if defined($d->{rombar}) && !$d->{rombar}; die "legacy IGD assignment is not compatible with x-vga\n" if $d->{'x-vga'}; die "legacy IGD assignment is not compatible with mdev\n" if $d->{mdev} || $d->{nvidia}; die "legacy IGD assignment is not compatible with q35\n" if $q35; die "legacy IGD assignment is not compatible with multifunction devices\n" if $d->{'has-multifunction'}; die "legacy IGD assignment is not compatible with alternate devices\n" if scalar($d->{ids}->@*) > 1; # check first device for valid id die "legacy IGD assignment only works for devices on host bus 00:02.0\n" if $d->{ids}->[0]->[0]->{id} !~ m/02\.0$/; } $parsed_devices->{$id} = $d; } return $parsed_devices; } # set vgpu type of a vf of an nvidia gpu with kernel 6.8 or newer my sub create_nvidia_device { my ($id, $model) = @_; $id = PVE::SysFSTools::normalize_pci_id($id); my $creation = "/sys/bus/pci/devices/$id/nvidia/current_vgpu_type"; die "no nvidia sysfs api for '$id'\n" if !-f $creation; my $current = PVE::Tools::file_read_firstline($creation); if ($current ne "0") { return 1 if $current eq $model; # reset vgpu type so we can see all available and set the real device die "unable to reset vgpu type for '$id'\n" if !PVE::SysFSTools::file_write($creation, "0"); } my $types = PVE::SysFSTools::get_mdev_types($id); my $selected; for my $type_definition ($types->@*) { next if $type_definition->{type} ne "nvidia-$model"; $selected = $type_definition; } if (!defined($selected) || $selected->{available} < 1) { die "vgpu type '$model' not available for '$id'\n"; } if (!PVE::SysFSTools::file_write($creation, $model)) { die "could not set vgpu type to '$model' for '$id'\n"; } return 1; } # takes the hash returned by parse_hostpci_devices and for all non mdev gpus, # selects one of the given alternatives by trying to reserve it # # mdev devices must be chosen later when we actually allocate it, but we # flatten the inner list since there can only be one device per alternative anyway sub choose_hostpci_devices { my ($devices, $vmid, $dry_run) = @_; my $used = {}; my $add_used_device = sub { my ($devices) = @_; for my $used_device ($devices->@*) { my $used_id = $used_device->{id}; die "device '$used_id' assigned more than once\n" if $used->{$used_id}; $used->{$used_id} = 1; } }; for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) { my $device = $devices->{"hostpci$i"}; next if !$device; if ($device->{mdev} && !$device->{nvidia}) { $device->{ids} = [map { $_->[0] } $device->{ids}->@*]; next; } if (scalar($device->{ids}->@* == 1)) { # we only have one alternative, use that $device->{ids} = $device->{ids}->[0]; $add_used_device->($device->{ids}); if ($device->{nvidia} && !$dry_run) { reserve_pci_usage($device->{ids}->[0]->{id}, $vmid, 10, undef); create_nvidia_device($device->{ids}->[0]->{id}, $device->{nvidia}); } next; } my $found = 0; for my $alternative ($device->{ids}->@*) { my $ids = [map { $_->{id} } @$alternative]; next if grep { defined($used->{$_}) } @$ids; # already used if (!$dry_run) { eval { reserve_pci_usage($ids, $vmid, 10, undef) }; next if $@; } if ($device->{nvidia} && !$dry_run) { eval { create_nvidia_device($ids->[0], $device->{nvidia}) }; if (my $err = $@) { warn $err; remove_pci_reservation($vmid, $ids); next; } } # found one that is not used or reserved $add_used_device->($alternative); $device->{ids} = $alternative; $found = 1; last; } die "could not find a free device for 'hostpci$i'\n" if !$found; } return $devices; } sub print_hostpci_devices { my ($vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $bootorder, $dry_run) = @_; my $kvm_off = 0; my $gpu_passthrough = 0; my $legacy_igd = 0; my $pciaddr; my $pci_devices = choose_hostpci_devices(parse_hostpci_devices($conf), $vmid, $dry_run); for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) { my $id = "hostpci$i"; my $d = $pci_devices->{$id}; next if !$d; $legacy_igd = 1 if $d->{'legacy-igd'}; if (my $pcie = $d->{pcie}) { # win7 wants to have the pcie devices directly on the pcie bus # instead of in the root port if ($winversion == 7) { $pciaddr = print_pcie_addr("${id}bus0"); } else { # add more root ports if needed, 4 are present by default # by pve-q35 cfgs, rest added here on demand. if ($i > 3) { push @$devices, '-device', print_pcie_root_port($i); } $pciaddr = print_pcie_addr($id); } } else { my $pci_name = $d->{'legacy-igd'} ? 'legacy-igd' : $id; $pciaddr = print_pci_addr($pci_name, $bridges, $arch); } my $num_devices = scalar($d->{ids}->@*); my $multifunction = $num_devices > 1 && !$d->{mdev}; my $xvga = ''; if ($d->{'x-vga'}) { $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf'); $kvm_off = 1; $vga->{type} = 'none' if !defined($conf->{vga}); $gpu_passthrough = 1; } my $sysfspath; if ($d->{mdev}) { my $uuid = generate_mdev_uuid($vmid, $i); $sysfspath = "/sys/bus/mdev/devices/$uuid"; } for (my $j = 0; $j < $num_devices; $j++) { my $pcidevice = $d->{ids}->[$j]; my $devicestr = "vfio-pci"; if ($sysfspath) { $devicestr .= ",sysfsdev=$sysfspath"; } else { $devicestr .= ",host=$pcidevice->{id}"; } if ($d->{'live-migration-capable'}) { $devicestr .= ",enable-migration=on"; } my $mf_addr = $multifunction ? ".$j" : ''; $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}"; if ($j == 0) { $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar}; $devicestr .= "$xvga"; $devicestr .= ",multifunction=on" if $multifunction; $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile}; $devicestr .= ",bootindex=$bootorder->{$id}" if $bootorder->{$id}; for my $option (qw(vendor-id device-id sub-vendor-id sub-device-id)) { $devicestr .= ",x-pci-$option=$d->{$option}" if $d->{$option}; } } push @$devices, '-device', $devicestr; last if $d->{mdev}; } } return ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices); } sub prepare_pci_device { my ($vmid, $pciid, $index, $device) = @_; my $info = PVE::SysFSTools::pci_device_info("$pciid"); die "cannot prepare PCI pass-through, IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support(); die "no pci device info for device '$pciid'\n" if !$info; my $driver = $device->{driver} // 'vfio'; if ($device->{nvidia} || $driver eq "keep") { # nothing to do } elsif (my $mdev = $device->{mdev}) { my $uuid = generate_mdev_uuid($vmid, $index); PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $mdev); } else { die "can't unbind/bind PCI group to VFIO '$pciid'\n" if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid); warn "failed to reset PCI device '$pciid', but trying to continue as not all devices need a reset\n" if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info); } return $info; } my $RUNDIR = '/run/qemu-server'; my $PCIID_RESERVATION_FILE = "${RUNDIR}/pci-id-reservations"; my $PCIID_RESERVATION_LOCK = "${PCIID_RESERVATION_FILE}.lock"; # a list of PCI ID to VMID reservations, the validity is protected against leakage by either a PID, # for successfully started VM processes, or a expiration time for the initial time window between # reservation and actual VM process start-up. my $parse_pci_reservation_unlocked = sub { my $pci_ids = {}; if (my $fh = IO::File->new($PCIID_RESERVATION_FILE, "r")) { while (my $line = <$fh>) { if ($line =~ m/^($PCIRE)\s(\d+)\s(time|pid)\:(\d+)$/) { $pci_ids->{$1} = { vmid => $2, "$3" => $4, }; } } } return $pci_ids; }; my $write_pci_reservation_unlocked = sub { my ($reservations) = @_; my $data = ""; for my $pci_id (sort keys $reservations->%*) { my ($vmid, $pid, $time) = $reservations->{$pci_id}->@{ 'vmid', 'pid', 'time' }; if (defined($pid)) { $data .= "$pci_id $vmid pid:$pid\n"; } else { $data .= "$pci_id $vmid time:$time\n"; } } PVE::Tools::file_set_contents($PCIID_RESERVATION_FILE, $data); }; # removes all PCI device reservations held by the `vmid` sub remove_pci_reservation { my ($vmid, $pci_ids) = @_; PVE::Tools::lock_file( $PCIID_RESERVATION_LOCK, 2, sub { my $reservation_list = $parse_pci_reservation_unlocked->(); for my $id (keys %$reservation_list) { next if defined($pci_ids) && !grep { $_ eq $id } $pci_ids->@*; my $reservation = $reservation_list->{$id}; next if $reservation->{vmid} != $vmid; delete $reservation_list->{$id}; } $write_pci_reservation_unlocked->($reservation_list); }, ); die $@ if $@; } # return all currently reserved ids from the given vmid sub get_reservations { my ($vmid) = @_; my $reservations = $parse_pci_reservation_unlocked->(); my $list = []; for my $pci_id (sort keys $reservations->%*) { push $list->@*, $pci_id if $reservations->{$pci_id}->{vmid} == $vmid; } return $list; } sub reserve_pci_usage { my ($requested_ids, $vmid, $timeout, $pid) = @_; $requested_ids = [$requested_ids] if !ref($requested_ids); return if !scalar(@$requested_ids); # do nothing for empty list PVE::Tools::lock_file( $PCIID_RESERVATION_LOCK, 5, sub { my $reservation_list = $parse_pci_reservation_unlocked->(); my $ctime = time(); for my $id ($requested_ids->@*) { my $reservation = $reservation_list->{$id}; if ($reservation && $reservation->{vmid} != $vmid) { # check time based reservation die "PCI device '$id' is currently reserved for use by VMID '$reservation->{vmid}'\n" if defined($reservation->{time}) && $reservation->{time} > $ctime; if (my $reserved_pid = $reservation->{pid}) { # check running vm my $running_pid = PVE::QemuServer::Helpers::vm_running_locally($reservation->{vmid}); if (defined($running_pid) && $running_pid == $reserved_pid) { die "PCI device '$id' already in use by VMID '$reservation->{vmid}'\n"; } else { warn "leftover PCI reservation found for $id, lets take it...\n"; } } } elsif ($reservation) { # already reserved by the same vmid if (my $reserved_time = $reservation->{time}) { if (defined($timeout)) { # use the longer timeout my $old_timeout = $reservation->{time} - 5 - $ctime; $timeout = $old_timeout if $old_timeout > $timeout; } } elsif (my $reserved_pid = $reservation->{pid}) { my $running_pid = PVE::QemuServer::Helpers::vm_running_locally($reservation->{vmid}); if (defined($running_pid) && $running_pid == $reservation->{pid}) { if (defined($pid)) { die "PCI device '$id' already in use by running VMID '$reservation->{vmid}'\n"; } elsif (defined($timeout)) { # ignore timeout reservation for running vms, can happen with e.g. # qm showcmd return; } } } } $reservation_list->{$id} = { vmid => $vmid }; if (defined($pid)) { # VM started up, we can reserve now with the actual PID $reservation_list->{$id}->{pid} = $pid; } elsif (defined($timeout)) { # temporary reserve as we don't now the PID yet $reservation_list->{$id}->{time} = $ctime + $timeout + 5; } } $write_pci_reservation_unlocked->($reservation_list); }, ); die $@ if $@; } 1; ================================================ FILE: src/PVE/QemuServer/QMPHelpers.pm ================================================ package PVE::QemuServer::QMPHelpers; use warnings; use strict; use PVE::QemuServer::Helpers; use PVE::QemuServer::Monitor qw(mon_cmd); use base 'Exporter'; our @EXPORT_OK = qw( qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel ); sub nbd_stop { my ($vmid) = @_; mon_cmd($vmid, 'nbd-server-stop', timeout => 25); } sub qemu_deviceadd { my ($vmid, $devicefull) = @_; $devicefull = "driver=" . $devicefull; PVE::QemuServer::Monitor::hmp_cmd($vmid, "device_add $devicefull", 25); } sub qemu_devicedel { my ($vmid, $deviceid) = @_; PVE::QemuServer::Monitor::hmp_cmd($vmid, "device_del $deviceid", 25); } sub qemu_objectadd { my ($vmid, $objectid, $qomtype, %args) = @_; mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype, %args); return 1; } sub qemu_objectdel { my ($vmid, $objectid) = @_; mon_cmd($vmid, "object-del", id => $objectid); return 1; } # dies if a) VM not running or not existing b) Version query failed # So, any defined return value is valid, any invalid state can be caught by eval sub runs_at_least_qemu_version { my ($vmid, $major, $minor, $extra) = @_; my $v = PVE::QemuServer::Monitor::mon_cmd($vmid, 'query-version'); die "could not query currently running version for VM $vmid\n" if !defined($v); $v = $v->{qemu}; return PVE::QemuServer::Helpers::version_cmp( $v->{major}, $major, $v->{minor}, $minor, $v->{micro}, $extra, ) >= 0; } 1; ================================================ FILE: src/PVE/QemuServer/QSD.pm ================================================ package PVE::QemuServer::QSD; use v5.36; use JSON qw(to_json); use PVE::JSONSchema qw(json_bool); use PVE::SafeSyslog qw(syslog); use PVE::Storage; use PVE::Tools; use PVE::QemuServer::Blockdev; use PVE::QemuServer::Helpers; use PVE::QemuServer::Monitor qw(qsd_qmp_peer); =head3 start PVE::QemuServer::QSD::start($id); Start a QEMU storage daemon instance with ID C<$id>. =cut sub start($id) { # If something is still mounted, that could block the new instance, try to clean up first. PVE::QemuServer::Helpers::qsd_fuse_export_cleanup_files($id); my $qmp_socket_path = PVE::QemuServer::Helpers::qmp_socket(qsd_qmp_peer($id)); my $pidfile = PVE::QemuServer::Helpers::qsd_pidfile_name($id); my $cmd = [ 'qemu-storage-daemon', '--daemonize', '--chardev', "socket,id=qmp,path=$qmp_socket_path,server=on,wait=off", '--monitor', 'chardev=qmp,mode=control', '--pidfile', $pidfile, ]; PVE::Tools::run_command($cmd); my $pid = PVE::QemuServer::Helpers::qsd_running_locally($id); syslog("info", "QEMU storage daemon $id started with PID $pid."); return; } =head3 add_fuse_export my $path = PVE::QemuServer::QSD::add_fuse_export($id, $drive, $name); Attach drive C<$drive> to the storage daemon with ID C<$id> and export it with name C<$name> via FUSE. Returns the path to the file representing the export. =cut sub add_fuse_export($id, $drive, $name) { my $storage_config = PVE::Storage::config(); PVE::Storage::activate_volumes($storage_config, [$drive->{file}]); my ($node_name, $read_only) = PVE::QemuServer::Blockdev::attach($storage_config, $id, $drive, { qsd => 1 }); my $fuse_path = PVE::QemuServer::Helpers::qsd_fuse_export_path($id, $name); PVE::Tools::file_set_contents($fuse_path, '', 0600); # mountpoint file needs to exist up-front my $export = { type => 'fuse', id => "$name", mountpoint => $fuse_path, 'node-name' => "$node_name", writable => json_bool(!$read_only), growable => JSON::false, 'allow-other' => 'off', }; PVE::QemuServer::Monitor::qsd_cmd($id, 'block-export-add', $export->%*); return $fuse_path; } =head3 remove_fuse_export PVE::QemuServer::QSD::remove_fuse_export($id, $name); Remove the export with name C<$name> from the storage daemon with ID C<$id>. =cut sub remove_fuse_export($id, $name) { PVE::QemuServer::Monitor::qsd_cmd($id, 'block-export-del', id => "$name"); return; } =head3 quit PVE::QemuServer::QSD::quit($id); Shut down the QEMU storage daemon with ID C<$id> and cleans up its PID file and socket. Waits for 60 seconds for clean shutdown, then sends SIGTERM and waits an additional 10 seconds before sending SIGKILL. =cut sub quit($id) { my $name = "QEMU storage daemon $id"; eval { PVE::QemuServer::Monitor::qsd_cmd($id, 'quit'); }; my $qmp_err = $@; warn "$name failed to handle 'quit' - $qmp_err" if $qmp_err; my $count = $qmp_err ? 60 : 0; # can't wait for QMP 'quit' to terminate the process if it failed my $pid = PVE::QemuServer::Helpers::qsd_running_locally($id); while ($pid) { if ($count == 60) { warn "$name still running with PID $pid - terminating now with SIGTERM\n"; kill 15, $pid; } elsif ($count == 70) { warn "$name still running with PID $pid - terminating now with SIGKILL\n"; kill 9, $pid; last; } sleep 1; $count++; $pid = PVE::QemuServer::Helpers::qsd_running_locally($id); } unlink PVE::QemuServer::Helpers::qsd_pidfile_name($id); unlink PVE::QemuServer::Helpers::qmp_socket(qsd_qmp_peer($id)); PVE::QemuServer::Helpers::qsd_fuse_export_cleanup_files($id); return; } 1; ================================================ FILE: src/PVE/QemuServer/QemuImage.pm ================================================ package PVE::QemuServer::QemuImage; use strict; use warnings; use Fcntl qw(S_ISBLK); use File::stat; use JSON; use PVE::Format qw(render_bytes); use PVE::Storage; use PVE::Tools; use PVE::QemuServer::Blockdev; use PVE::QemuServer::Drive qw(checked_volume_format); use PVE::QemuServer::Helpers; sub convert_iscsi_path { my ($path) = @_; if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) { my $portal = $1; my $target = $2; my $lun = $3; my $initiator_name = PVE::QemuServer::Helpers::get_iscsi_initiator_name(); return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name," . "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw"; } die "cannot convert iscsi path '$path', unknown format\n"; } my sub qcow2_target_image_opts { my ($storecfg, $drive, $qcow2_opts, $zeroinit) = @_; # There is no machine version, the qemu-img binary version is what's important. my $version = PVE::QemuServer::Helpers::kvm_user_version(); my $blockdev_opts = { 'no-throttle' => 1 }; $blockdev_opts->{'zero-initialized'} = 1 if $zeroinit; my $blockdev = PVE::QemuServer::Blockdev::generate_drive_blockdev( $storecfg, $drive, $version, $blockdev_opts, ); my $opts = []; my $opt_prefix = ''; my $next_child = $blockdev; while ($next_child) { my $current = $next_child; $next_child = delete($current->{file}); # TODO should cache settings be configured here (via appropriate drive configuration) rather # than via dedicated qemu-img options? delete($current->{cache}); # TODO e.g. can't use aio 'native' without cache.direct, just use QEMU default like for # other targets for now delete($current->{aio}); # no need for node names delete($current->{'node-name'}); # it's the write target, while the flag should be 'false' anyways, remove to be sure delete($current->{'read-only'}); # TODO should those be set (via appropriate drive configuration)? delete($current->{'detect-zeroes'}); delete($current->{'discard'}); for my $key (sort keys $current->%*) { my $value; if (ref($current->{$key})) { if ($current->{$key} eq JSON::false) { $value = 'false'; } elsif ($current->{$key} eq JSON::true) { $value = 'true'; } else { die "target image options: unhandled structured key: $key\n"; } } else { $value = $current->{$key}; } push $opts->@*, "$opt_prefix$key=$value"; } $opt_prefix .= 'file.'; } return join(',', $opts->@*); } # The possible options are: # bwlimit - The bandwidth limit in KiB/s. # is-zero-initialized - If the destination image is zero-initialized. # snapname - Use this snapshot of the source image. # source-path-format - Indicate the format of the source when the source is a path. For PVE-managed # volumes, the format from the storage layer is always used. sub convert { my ($src_volid, $dst_volid, $size, $opts) = @_; my ($bwlimit, $snapname) = $opts->@{qw(bwlimit snapname)}; my $storecfg = PVE::Storage::config(); my ($src_storeid) = PVE::Storage::parse_volume_id($src_volid, 1); my ($dst_storeid) = PVE::Storage::parse_volume_id($dst_volid, 1); die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid; my $cachemode; my $src_path; my $src_is_iscsi = 0; my $src_format; if ($src_storeid) { PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname); my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid); $src_format = checked_volume_format($storecfg, $src_volid); $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname); $src_is_iscsi = ($src_path =~ m|^iscsi://|); $cachemode = 'none' if $src_scfg->{type} eq 'zfspool'; } elsif (-f $src_volid || -b $src_volid) { $src_path = $src_volid; if ($opts->{'source-path-format'}) { $src_format = $opts->{'source-path-format'}; } elsif ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) { $src_format = $1; } } die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path; my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid); my $dst_format = checked_volume_format($storecfg, $dst_volid); my $dst_path = PVE::Storage::path($storecfg, $dst_volid); my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|); my $dst_needs_discard_no_unref = $dst_scfg->{'snapshot-as-volume-chain'} && $dst_format eq 'qcow2'; my $support_qemu_snapshots = PVE::Storage::volume_qemu_snapshot_method($storecfg, $src_volid); my $cmd = []; push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n'; push @$cmd, '-l', "snapshot.name=$snapname" if $snapname && $src_format eq 'qcow2' && $support_qemu_snapshots && $support_qemu_snapshots eq 'qemu'; push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool'; push @$cmd, '-T', $cachemode if defined($cachemode); push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit); if ($src_is_iscsi) { push @$cmd, '--image-opts'; $src_path = convert_iscsi_path($src_path); } elsif ($src_format) { push @$cmd, '-f', $src_format; } my $dst_uses_target_image_opts = $dst_is_iscsi || $dst_needs_discard_no_unref; push @$cmd, '--target-image-opts' if $dst_uses_target_image_opts; if ($dst_is_iscsi) { $dst_path = convert_iscsi_path($dst_path); } elsif ($dst_needs_discard_no_unref) { # don't use any other drive options, those are intended for use with a running VM and just # use scsi0 as a dummy interface+index for now my $dst_drive = { file => $dst_volid, interface => 'scsi', index => 0 }; $dst_path = qcow2_target_image_opts( $storecfg, $dst_drive, ['discard-no-unref=true'], $opts->{'is-zero-initialized'}, ); } else { push @$cmd, '-O', $dst_format; } push @$cmd, $src_path; if (!$dst_uses_target_image_opts && $opts->{'is-zero-initialized'}) { push @$cmd, "zeroinit:$dst_path"; } else { push @$cmd, $dst_path; } my $parser = sub { my $line = shift; if ($line =~ m/\((\S+)\/100\%\)/) { my $percent = $1; my $transferred = int($size * $percent / 100); my $total_h = render_bytes($size, 1); my $transferred_h = render_bytes($transferred, 1); print "transferred $transferred_h of $total_h ($percent%)\n"; } }; eval { PVE::Tools::run_command($cmd, timeout => undef, outfunc => $parser); }; my $err = $@; die "copy failed: $err" if $err; } 1; ================================================ FILE: src/PVE/QemuServer/RNG.pm ================================================ package PVE::QemuServer::RNG; use strict; use warnings; use PVE::JSONSchema; use PVE::Tools qw(file_read_firstline); use PVE::QemuServer::PCI qw(print_pci_addr); use base 'Exporter'; our @EXPORT_OK = qw( parse_rng check_rng_source print_rng_device_commandline print_rng_object_commandline ); my $rng_fmt = { source => { type => 'string', enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'], default_key => 1, description => "The file on the host to gather entropy from. Using urandom does *not*" . " decrease security in any meaningful way, as it's still seeded from real entropy, and" . " the bytes provided will most likely be mixed with real entropy on the guest as well." . " '/dev/hwrng' can be used to pass through a hardware RNG from the host.", }, max_bytes => { type => 'integer', description => "Maximum bytes of entropy allowed to get injected into the guest every" . " 'period' milliseconds. Use `0` to disable limiting (potentially dangerous!).", optional => 1, # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're # reading from /dev/urandom default => 1024, }, period => { type => 'integer', description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing" . " the guest to retrieve another 'max_bytes' of entropy.", optional => 1, default => 1000, }, }; PVE::JSONSchema::register_format('pve-qm-rng', $rng_fmt); our $rngdesc = { type => 'string', format => $rng_fmt, optional => 1, description => "Configure a VirtIO-based Random Number Generator.", }; PVE::JSONSchema::register_standard_option('pve-qm-rng', $rngdesc); sub parse_rng { my ($value) = @_; return if !$value; my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) }; warn $@ if $@; return $res; } sub check_rng_source { my ($source) = @_; # mostly relevant for /dev/hwrng, but doesn't hurt to check others too die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n" if !-e $source; my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current'; if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') { # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue. die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but" . " '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached" . " to the host.\n"; } } sub print_rng_device_commandline { my ($id, $rng, $bridges, $arch) = @_; die "no rng device specified\n" if !$rng; my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default}; my $period = $rng->{period} // $rng_fmt->{period}->{default}; my $limiter_str = ""; if ($max_bytes) { $limiter_str = ",max-bytes=$max_bytes,period=$period"; } my $rng_addr = print_pci_addr($id, $bridges, $arch); return "virtio-rng-pci,rng=$id$limiter_str$rng_addr"; } sub print_rng_object_commandline { my ($id, $rng) = @_; die "no rng device specified\n" if !$rng; my $source_path = $rng->{source}; check_rng_source($source_path); return "rng-random,filename=$source_path,id=$id"; } 1; ================================================ FILE: src/PVE/QemuServer/RunState.pm ================================================ package PVE::QemuServer::RunState; use strict; use warnings; use POSIX qw(strftime); use PVE::Cluster; use PVE::RPCEnvironment; use PVE::Storage; use PVE::QemuConfig; use PVE::QemuMigrate::Helpers; use PVE::QemuServer::Monitor qw(mon_cmd); use PVE::QemuServer::Network; # note: if using the statestorage parameter, the caller has to check privileges sub vm_suspend { my ($vmid, $skiplock, $includestate, $statestorage) = @_; my $conf; my $path; my $storecfg; my $vmstate; PVE::QemuConfig->lock_config( $vmid, sub { $conf = PVE::QemuConfig->load_config($vmid); my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup'); PVE::QemuConfig->check_lock($conf) if !($skiplock || $is_backing_up); die "cannot suspend to disk during backup\n" if $is_backing_up && $includestate; PVE::QemuMigrate::Helpers::check_non_migratable_resources($conf, $includestate, 0); if ($includestate) { $conf->{lock} = 'suspending'; my $date = strftime("%Y-%m-%d", localtime(time())); $storecfg = PVE::Storage::config(); if (!$statestorage) { $statestorage = PVE::QemuConfig::find_vmstate_storage($conf, $storecfg); # check permissions for the storage my $rpcenv = PVE::RPCEnvironment::get(); if ($rpcenv->{type} ne 'cli') { my $authuser = $rpcenv->get_user(); $rpcenv->check( $authuser, "/storage/$statestorage", ['Datastore.AllocateSpace'], ); } } $vmstate = PVE::QemuConfig->__snapshot_save_vmstate( $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1, ); $path = PVE::Storage::path($storecfg, $vmstate); PVE::QemuConfig->write_config($vmid, $conf); } else { mon_cmd($vmid, "stop"); } }, ); if ($includestate) { # save vm state PVE::Storage::activate_volumes($storecfg, [$vmstate]); eval { PVE::QemuMigrate::Helpers::set_migration_caps($vmid, 1); mon_cmd($vmid, "savevm-start", statefile => $path); for (;;) { my $state = mon_cmd($vmid, "query-savevm"); if (!$state->{status}) { die "savevm not active\n"; } elsif ($state->{status} eq 'active') { sleep(1); next; } elsif ($state->{status} eq 'completed') { print "State saved, quitting\n"; last; } elsif ($state->{status} eq 'failed' && $state->{error}) { die "query-savevm failed with error '$state->{error}'\n"; } else { die "query-savevm returned status '$state->{status}'\n"; } } }; my $err = $@; PVE::QemuConfig->lock_config( $vmid, sub { $conf = PVE::QemuConfig->load_config($vmid); if ($err) { # cleanup, but leave suspending lock, to indicate something went wrong eval { eval { mon_cmd($vmid, "savevm-end"); }; warn $@ if $@; PVE::Storage::deactivate_volumes($storecfg, [$vmstate]); PVE::Storage::vdisk_free($storecfg, $vmstate); delete $conf->@{ qw(vmstate runningmachine runningcpu running-nets-host-mtu)}; PVE::QemuConfig->write_config($vmid, $conf); }; warn $@ if $@; die $err; } die "lock changed unexpectedly\n" if !PVE::QemuConfig->has_lock($conf, 'suspending'); mon_cmd($vmid, "quit"); $conf->{lock} = 'suspended'; PVE::QemuConfig->write_config($vmid, $conf); }, ); } } # $nocheck is set when called as part of a migration - in this context the # location of the config file (source or target node) is not deterministic, # since migration cannot wait for pmxcfs to process the rename sub vm_resume { my ($vmid, $skiplock, $nocheck) = @_; PVE::QemuConfig->lock_config( $vmid, sub { # After migration, the VM might not immediately be able to respond to QMP commands, because # activating the block devices might take a bit of time. my $res = mon_cmd($vmid, 'query-status', timeout => 60); my $resume_cmd = 'cont'; my $reset = 0; my $conf; if ($nocheck) { $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node if ($@) { my $vmlist = PVE::Cluster::get_vmlist(); if (exists($vmlist->{ids}->{$vmid})) { my $node = $vmlist->{ids}->{$vmid}->{node}; $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node } if (!$conf) { PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again } } } else { $conf = PVE::QemuConfig->load_config($vmid); } die "VM $vmid is a template and cannot be resumed!\n" if PVE::QemuConfig->is_template($conf); if ($res->{status}) { return if $res->{status} eq 'running'; # job done, go home $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended'; $reset = 1 if $res->{status} eq 'shutdown'; } if (!$nocheck) { PVE::QemuConfig->check_lock($conf) if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup')); } if ($reset) { # required if a VM shuts down during a backup and we get a resume # request before the backup finishes for example mon_cmd($vmid, "system_reset"); } PVE::QemuServer::Network::add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont'; mon_cmd($vmid, $resume_cmd); }, ); } 1; ================================================ FILE: src/PVE/QemuServer/StateFile.pm ================================================ package PVE::QemuServer::StateFile; use strict; use warnings; use PVE::Cluster; use PVE::Network; sub get_migration_ip { my ($nodename, $cidr) = @_; if (!defined($cidr)) { my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg'); $cidr = $dc_conf->{migration}->{network}; } if (defined($cidr)) { my $ips = PVE::Network::get_local_ip_from_cidr($cidr); die "could not get IP: no address configured on local node for network '$cidr'\n" if scalar(@$ips) == 0; die "could not get IP: multiple addresses configured on local node for network '$cidr'\n" if scalar(@$ips) > 1; return $ips->[0]; } return PVE::Cluster::remote_node_ip($nodename, 1); } # $migration_ip must be defined if using insecure TCP migration sub statefile_cmdline_option { my ($storecfg, $vmid, $statefile, $migration_type, $migration_ip) = @_; my $statefile_is_a_volume = 0; my $res = {}; my $cmd = []; if ($statefile eq 'tcp') { my $migrate = $res->{migrate} = { proto => 'tcp' }; $migrate->{addr} = "localhost"; die "no migration type set\n" if !defined($migration_type); if ($migration_type eq 'insecure') { $migrate->{addr} = $migration_ip // die "internal error - no migration IP"; $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr}); } # see #4501: port reservation should be done close to usage - tell QEMU where to listen # via QMP later push @$cmd, '-incoming', 'defer'; push @$cmd, '-S'; } elsif ($statefile eq 'unix') { # should be default for secure migrations as a ssh TCP forward # tunnel is not deterministic reliable ready and fails regurarly # to set up in time, so use UNIX socket forwards my $migrate = $res->{migrate} = { proto => 'unix' }; $migrate->{addr} = "/run/qemu-server/$vmid.migrate"; unlink $migrate->{addr}; $migrate->{uri} = "unix:$migrate->{addr}"; push @$cmd, '-incoming', $migrate->{uri}; push @$cmd, '-S'; } elsif (-e $statefile) { push @$cmd, '-loadstate', $statefile; } else { my $statepath = PVE::Storage::path($storecfg, $statefile); $statefile_is_a_volume = 1; push @$cmd, '-loadstate', $statepath; } return ($cmd, $res->{migrate}, $statefile_is_a_volume); } 1; ================================================ FILE: src/PVE/QemuServer/USB.pm ================================================ package PVE::QemuServer::USB; use strict; use warnings; use PVE::QemuServer::PCI qw(print_pci_addr); use PVE::QemuServer::Machine; use PVE::QemuServer::Helpers qw(min_version windows_version); use PVE::JSONSchema; use PVE::Mapping::USB; use base 'Exporter'; our @EXPORT_OK = qw( parse_usb_device get_usb_controllers get_usb_devices ); my $OLD_MAX_USB = 5; our $MAX_USB_DEVICES = 14; my $USB_ID_RE = qr/(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})/; my $USB_PATH_RE = qr/(\d+)\-(\d+(\.\d+)*)/; my $usb_fmt = { host => { default_key => 1, optional => 1, type => 'string', pattern => qr/(?:(?:$USB_ID_RE)|(?:$USB_PATH_RE)|[Ss][Pp][Ii][Cc][Ee])/, format_description => 'HOSTUSBDEVICE|spice', description => < { optional => 1, type => 'string', format_description => 'mapping-id', format => 'pve-configid', description => "The ID of a cluster wide mapping. Either this or the default-key 'host'" . " must be set.", }, usb3 => { optional => 1, type => 'boolean', description => "Specifies whether if given host option is a USB3 device or port." . " For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag" . " is irrelevant (all devices are plugged into a xhci controller).", default => 0, }, }; PVE::JSONSchema::register_format('pve-qm-usb', $usb_fmt); our $usbdesc = { optional => 1, type => 'string', format => $usb_fmt, description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype" . " l26 or windows > 7, n can be up to 14).", }; PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc); sub parse_usb_device { my ($value, $mapping) = @_; return if $value && $mapping; # not a valid configuration my $res = {}; if (defined($value)) { if ($value =~ m/^$USB_ID_RE$/) { $res->{vendorid} = $2; $res->{productid} = $4; } elsif ($value =~ m/^$USB_PATH_RE$/) { $res->{hostbus} = $1; $res->{hostport} = $2; } elsif ($value =~ m/^spice$/i) { $res->{spice} = 1; } } elsif (defined($mapping)) { my $devices = PVE::Mapping::USB::find_on_current_node($mapping); die "USB device mapping not found for '$mapping'\n" if !$devices || !scalar($devices->@*); die "More than one USB mapping per host not supported\n" if scalar($devices->@*) > 1; eval { PVE::Mapping::USB::assert_valid($mapping, $devices->[0]); }; if (my $err = $@) { die "USB Mapping invalid (hardware probably changed): $err\n"; } my $device = $devices->[0]; if ($device->{path}) { $res = parse_usb_device($device->{path}); } else { $res = parse_usb_device($device->{id}); } } return $res; } my sub assert_usb_index_is_useable { my ($index, $use_qemu_xhci) = @_; die "using usb$index is only possible with machine type >= 7.1 and ostype l26 or windows > 7\n" if $index >= $OLD_MAX_USB && !$use_qemu_xhci; return undef; } sub get_usb_controllers { my ($conf, $bridges, $arch, $machine_version) = @_; my $devices = []; my $pciaddr = ""; my $ostype = $conf->{ostype}; my $use_qemu_xhci = min_version($machine_version, 7, 1) && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7); my $is_q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf); if ($arch eq 'aarch64') { $pciaddr = print_pci_addr('ehci', $bridges, $arch); push @$devices, '-device', "usb-ehci,id=ehci$pciaddr"; } elsif (!$is_q35) { $pciaddr = print_pci_addr("piix3", $bridges, $arch); push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2"; } my ($use_usb2, $use_usb3) = 0; my $any_usb = 0; for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) { next if !$conf->{"usb$i"}; assert_usb_index_is_useable($i, $use_qemu_xhci); my $d = eval { PVE::JSONSchema::parse_property_string($usb_fmt, $conf->{"usb$i"}) } or next; $any_usb = 1; $use_usb3 = 1 if $d->{usb3}; $use_usb2 = 1 if !$d->{usb3}; } if (!$use_qemu_xhci && !$is_q35 && $use_usb2 && $arch ne 'aarch64') { # include usb device config if still on x86 before-xhci machines and if USB 3 is not used push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg'; } $pciaddr = print_pci_addr("xhci", $bridges, $arch); if ($use_qemu_xhci && $any_usb) { push @$devices, '-device', print_qemu_xhci_controller($pciaddr); } elsif ($use_usb3) { push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr"; } return @$devices; } sub get_usb_devices { my ($conf, $features, $bootorder, $machine_version) = @_; my $devices = []; my $ostype = $conf->{ostype}; my $use_qemu_xhci = min_version($machine_version, 7, 1) && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7); for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) { my $devname = "usb$i"; next if !$conf->{$devname}; assert_usb_index_is_useable($i, $use_qemu_xhci); my $d = eval { PVE::JSONSchema::parse_property_string($usb_fmt, $conf->{$devname}) }; next if !$d; my $port = $use_qemu_xhci ? $i + 1 : undef; if ($d->{host} && $d->{host} =~ m/^spice$/) { # usb redir support for spice my $bus = 'ehci'; $bus = 'xhci' if ($d->{usb3} && $features->{spice_usb3}) || $use_qemu_xhci; push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir"; push @$devices, '-device', print_spice_usbdevice($i, $bus, $port); warn "warning: spice usb port set as bootdevice, ignoring\n" if $bootorder->{$devname}; } else { push @$devices, '-device', print_usbdevice_full($conf, $devname, $d, $bootorder, $port); } } return @$devices; } sub print_qemu_xhci_controller { my ($pciaddr) = @_; return "qemu-xhci,p2=15,p3=15,id=xhci$pciaddr"; } sub print_spice_usbdevice { my ($index, $bus, $port) = @_; my $device = "usb-redir,chardev=usbredirchardev$index,id=usbredirdev$index,bus=$bus.0"; if (defined($port)) { $device .= ",port=$port"; } return $device; } sub print_usbdevice_full { my ($conf, $deviceid, $device, $bootorder, $port) = @_; return if !$device; my $usbdevice = "usb-host"; # if it is a usb3 device or with newer qemu, attach it to the xhci controller, else omit the bus option if ($device->{usb3} || defined($port)) { $usbdevice .= ",bus=xhci.0"; $usbdevice .= ",port=$port" if defined($port); } my $parsed = parse_usb_device($device->{host}, $device->{mapping}); if (defined($parsed->{vendorid}) && defined($parsed->{productid})) { $usbdevice .= ",vendorid=0x$parsed->{vendorid},productid=0x$parsed->{productid}"; } elsif (defined($parsed->{hostbus}) && defined($parsed->{hostport})) { $usbdevice .= ",hostbus=$parsed->{hostbus},hostport=$parsed->{hostport}"; } else { die "no usb id or path given\n"; } $usbdevice .= ",id=$deviceid"; $usbdevice .= ",bootindex=$bootorder->{$deviceid}" if $bootorder->{$deviceid}; return $usbdevice; } 1; ================================================ FILE: src/PVE/QemuServer/Virtiofs.pm ================================================ package PVE::QemuServer::Virtiofs; use strict; use warnings; use Fcntl qw(F_GETFD F_SETFD FD_CLOEXEC); use IO::Socket::UNIX; use POSIX; use Socket qw(SOCK_STREAM); use PVE::JSONSchema qw(parse_property_string); use PVE::Mapping::Dir; use PVE::QemuServer::Helpers; use PVE::RESTEnvironment qw(log_warn); use base qw(Exporter); our @EXPORT_OK = qw( max_virtiofs start_all_virtiofsd ); my $MAX_VIRTIOFS = 10; my $socket_path_root = "/run/qemu-server/virtiofsd"; my $virtiofs_fmt = { 'dirid' => { type => 'string', default_key => 1, description => "Mapping identifier of the directory mapping to be shared with the guest." . " Also used as a mount tag inside the VM.", format_description => 'mapping-id', format => 'pve-configid', }, 'cache' => { type => 'string', description => "The caching policy the file system should use (auto, always, metadata, never).", enum => [qw(auto always metadata never)], default => "auto", optional => 1, }, 'direct-io' => { type => 'boolean', description => "Honor the O_DIRECT flag passed down by guest applications.", default => 0, optional => 1, }, 'expose-xattr' => { type => 'boolean', description => "Enable support for extended attributes for this mount.", default => 0, optional => 1, }, 'expose-acl' => { type => 'boolean', description => "Enable support for POSIX ACLs (enabled ACL implies xattr) for this mount.", default => 0, optional => 1, }, }; PVE::JSONSchema::register_format('pve-qm-virtiofs', $virtiofs_fmt); my $virtiofsdesc = { optional => 1, type => 'string', format => $virtiofs_fmt, description => "Configuration for sharing a directory between host and guest using Virtio-fs.", }; PVE::JSONSchema::register_standard_option("pve-qm-virtiofs", $virtiofsdesc); sub max_virtiofs { return $MAX_VIRTIOFS; } sub assert_virtiofs_config { my ($ostype, $virtiofs) = @_; my $dir_cfg = PVE::Mapping::Dir::find_on_current_node($virtiofs->{dirid}); my $acl = $virtiofs->{'expose-acl'}; if ($acl && PVE::QemuServer::Helpers::windows_version($ostype)) { die "Please disable ACLs for virtiofs on Windows VMs, otherwise" . " the virtiofs shared directory cannot be mounted.\n"; } eval { PVE::Mapping::Dir::assert_valid($dir_cfg) }; die "directory mapping invalid: $@\n" if $@; } sub config { my ($conf, $vmid, $devices) = @_; for (my $i = 0; $i < max_virtiofs(); $i++) { my $opt = "virtiofs$i"; next if !$conf->{$opt}; my $virtiofs = parse_property_string('pve-qm-virtiofs', $conf->{$opt}); assert_virtiofs_config($conf->{ostype}, $virtiofs); push @$devices, '-chardev', "socket,id=virtiofs$i,path=$socket_path_root/vm$vmid-fs$i"; # queue-size is set 1024 because of bug with Windows guests: # https://bugzilla.redhat.com/show_bug.cgi?id=1873088 # 1024 is also always used in the virtiofs documentations: # https://gitlab.com/virtio-fs/virtiofsd#examples push @$devices, '-device', 'vhost-user-fs-pci,queue-size=1024' . ",chardev=virtiofs$i,tag=$virtiofs->{dirid}"; } } sub virtiofs_enabled { my ($conf) = @_; my $virtiofs_enabled = 0; for (my $i = 0; $i < max_virtiofs(); $i++) { my $opt = "virtiofs$i"; next if !$conf->{$opt}; parse_property_string('pve-qm-virtiofs', $conf->{$opt}); $virtiofs_enabled = 1; } return $virtiofs_enabled; } sub start_all_virtiofsd { my ($conf, $vmid) = @_; my $virtiofs_sockets = []; for (my $i = 0; $i < max_virtiofs(); $i++) { my $opt = "virtiofs$i"; next if !$conf->{$opt}; my $virtiofs = parse_property_string('pve-qm-virtiofs', $conf->{$opt}); # See https://github.com/virtio-win/kvm-guest-drivers-windows/issues/1136 my $prefer_inode_fh = PVE::QemuServer::Helpers::windows_version($conf->{ostype}) ? 1 : 0; my $virtiofs_socket = start_virtiofsd($vmid, $i, $virtiofs, $prefer_inode_fh); push @$virtiofs_sockets, $virtiofs_socket; } return $virtiofs_sockets; } sub start_virtiofsd { my ($vmid, $fsid, $virtiofs, $prefer_inode_fh) = @_; mkdir $socket_path_root; my $socket_path = "$socket_path_root/vm$vmid-fs$fsid"; unlink($socket_path); my $socket = IO::Socket::UNIX->new( Type => SOCK_STREAM, Local => $socket_path, Listen => 1, ) or die "cannot create socket - $!\n"; my $flags = fcntl($socket, F_GETFD, 0) or die "failed to get file descriptor flags: $!\n"; fcntl($socket, F_SETFD, $flags & ~FD_CLOEXEC) or die "failed to remove FD_CLOEXEC from file descriptor\n"; my $dir_cfg = PVE::Mapping::Dir::find_on_current_node($virtiofs->{dirid}); my $virtiofsd_bin = '/usr/libexec/virtiofsd'; if (!-f $virtiofsd_bin) { die "virtiofsd is not installed. To use virtio-fs, install virtiofsd via apt.\n"; } my $fd = $socket->fileno(); my $path = $dir_cfg->{path}; my $could_not_fork_err = "could not fork to start virtiofsd\n"; my $pid = fork(); if ($pid == 0) { POSIX::setsid(); $0 = "task pve-vm$vmid-virtiofs$fsid"; my $pid2 = fork(); if ($pid2 == 0) { my $cmd = [$virtiofsd_bin, "--fd=$fd", "--shared-dir=$path"]; push @$cmd, '--xattr' if $virtiofs->{'expose-xattr'}; push @$cmd, '--posix-acl' if $virtiofs->{'expose-acl'}; push @$cmd, '--announce-submounts'; push @$cmd, '--allow-direct-io' if $virtiofs->{'direct-io'}; push @$cmd, '--cache=' . $virtiofs->{cache} if $virtiofs->{cache}; push @$cmd, '--inode-file-handles=prefer' if $prefer_inode_fh; push @$cmd, '--syslog'; exec(@$cmd); } elsif (!defined($pid2)) { die $could_not_fork_err; } else { POSIX::_exit(0); } } elsif (!defined($pid)) { die $could_not_fork_err; } else { waitpid($pid, 0); } # return socket to keep it alive, # so that QEMU will wait for virtiofsd to start return $socket; } sub close_sockets { my @sockets = @_; for my $socket (@sockets) { shutdown($socket, 2); close($socket); } } 1; ================================================ FILE: src/PVE/QemuServer/VolumeChain.pm ================================================ package PVE::QemuServer::VolumeChain; use strict; use warnings; use File::Basename qw(basename dirname); use JSON; use PVE::Storage; use PVE::QemuServer::Blockdev qw(generate_file_blockdev generate_format_blockdev); use PVE::QemuServer::BlockJob; use PVE::QemuServer::Drive; use PVE::QemuServer::Monitor qw(qmp_cmd); sub blockdev_external_snapshot { my ($storecfg, $qmp_peer, $machine_version, $deviceid, $drive, $snap, $parent_snap) = @_; print "Creating a new current volume with $snap as backing snap\n"; my $volid = $drive->{file}; #rename current to snap && preallocate add a new current file with reference to snap1 backing-file PVE::Storage::volume_snapshot($storecfg, $volid, $snap); #reopen current to snap blockdev_replace( $storecfg, $qmp_peer, $machine_version, $deviceid, $drive, 'current', $snap, $parent_snap, ); #be sure to add drive in write mode delete($drive->{ro}); my $new_file_blockdev = generate_file_blockdev($storecfg, $drive); my $new_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $new_file_blockdev); my $snap_file_blockdev = generate_file_blockdev($storecfg, $drive, $machine_version, { 'snapshot-name' => $snap }); my $snap_fmt_blockdev = generate_format_blockdev( $storecfg, $drive, $snap_file_blockdev, { 'snapshot-name' => $snap }, ); #backing need to be forced to undef in blockdev, to avoid reopen of backing-file on blockdev-add $new_fmt_blockdev->{backing} = undef; qmp_cmd($qmp_peer, 'blockdev-add', %$new_fmt_blockdev); print "blockdev-snapshot: reopen current with $snap backing image\n"; qmp_cmd( $qmp_peer, 'blockdev-snapshot', node => $snap_fmt_blockdev->{'node-name'}, overlay => $new_fmt_blockdev->{'node-name'}, ); } sub blockdev_delete { my ($storecfg, $qmp_peer, $drive, $file_blockdev, $fmt_blockdev, $snap) = @_; eval { PVE::QemuServer::Blockdev::detach($qmp_peer, $fmt_blockdev->{'node-name'}); }; warn "detaching block node for $file_blockdev->{filename} failed - $@" if $@; #delete the file (don't use vdisk_free as we don't want to delete all snapshot chain) print "delete old $file_blockdev->{filename}\n"; my $storage_name = PVE::Storage::parse_volume_id($drive->{file}); my $volid = $drive->{file}; PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, 1); } my sub blockdev_relative_backing_file { my ($backing, $backed) = @_; my $backing_file = $backing->{filename}; my $backed_file = $backed->{filename}; if (dirname($backing_file) eq dirname($backed_file)) { # make backing file relative if in same directory return basename($backing_file); } return $backing_file; } sub blockdev_replace { my ( $storecfg, $qmp_peer, $machine_version, $deviceid, $drive, $src_snap, $target_snap, $parent_snap, ) = @_; print "blockdev replace $src_snap by $target_snap\n"; my $volid = $drive->{file}; my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive); my $src_name_options = {}; my $src_blockdev_name; if ($src_snap eq 'current') { # there might be other nodes on top like zeroinit, look up the current node below throttle $src_blockdev_name = PVE::QemuServer::Blockdev::get_node_name_below_throttle($qmp_peer, $deviceid, 1); } else { $src_name_options = { 'snapshot-name' => $src_snap }; $src_blockdev_name = PVE::QemuServer::Blockdev::get_node_name('fmt', $drive_id, $volid, $src_name_options); } my $target_file_blockdev = generate_file_blockdev( $storecfg, $drive, $machine_version, { 'snapshot-name' => $target_snap }, ); my $target_fmt_blockdev = generate_format_blockdev( $storecfg, $drive, $target_file_blockdev, { 'snapshot-name' => $target_snap }, ); if ($target_snap eq 'current' || $src_snap eq 'current') { #rename from|to current #add backing to target if ($parent_snap) { my $parent_fmt_nodename = PVE::QemuServer::Blockdev::get_node_name( 'fmt', $drive_id, $volid, { 'snapshot-name' => $parent_snap }, ); $target_fmt_blockdev->{backing} = $parent_fmt_nodename; } qmp_cmd($qmp_peer, 'blockdev-add', %$target_fmt_blockdev); #reopen the current throttlefilter nodename with the target fmt nodename my $throttle_blockdev = PVE::QemuServer::Blockdev::generate_throttle_blockdev( $drive, $target_fmt_blockdev->{'node-name'}, {}, ); qmp_cmd($qmp_peer, 'blockdev-reopen', options => [$throttle_blockdev]); } else { #intermediate snapshot qmp_cmd($qmp_peer, 'blockdev-add', %$target_fmt_blockdev); #reopen the parent node with the new target fmt backing node my $parent_file_blockdev = generate_file_blockdev( $storecfg, $drive, $machine_version, { 'snapshot-name' => $parent_snap }, ); my $parent_fmt_blockdev = generate_format_blockdev( $storecfg, $drive, $parent_file_blockdev, { 'snapshot-name' => $parent_snap }, ); $parent_fmt_blockdev->{backing} = $target_fmt_blockdev->{'node-name'}; qmp_cmd($qmp_peer, 'blockdev-reopen', options => [$parent_fmt_blockdev]); my $backing_file = blockdev_relative_backing_file($target_file_blockdev, $parent_file_blockdev); #change backing-file in qcow2 metadatas qmp_cmd( $qmp_peer, 'change-backing-file', device => $deviceid, 'image-node-name' => $parent_fmt_blockdev->{'node-name'}, 'backing-file' => $backing_file, ); } # delete old file|fmt nodes eval { PVE::QemuServer::Blockdev::detach($qmp_peer, $src_blockdev_name); }; warn "detaching block node for $src_snap failed - $@" if $@; } sub blockdev_commit { my ($storecfg, $qmp_peer, $machine_version, $deviceid, $drive, $src_snap, $target_snap) = @_; my $volid = $drive->{file}; my $target_was_read_only; print "block-commit $src_snap to base:$target_snap\n"; my $target_file_blockdev = generate_file_blockdev( $storecfg, $drive, $machine_version, { 'snapshot-name' => $target_snap }, ); my $target_fmt_blockdev = generate_format_blockdev( $storecfg, $drive, $target_file_blockdev, { 'snapshot-name' => $target_snap }, ); my $src_file_blockdev = generate_file_blockdev( $storecfg, $drive, $machine_version, { 'snapshot-name' => $src_snap }, ); my $src_fmt_blockdev = generate_format_blockdev( $storecfg, $drive, $src_file_blockdev, { 'snapshot-name' => $src_snap }, ); if ($target_was_read_only = $target_fmt_blockdev->{'read-only'}) { print "reopening internal read-only block node for '$target_snap' as writable\n"; $target_fmt_blockdev->{'read-only'} = JSON::false; $target_file_blockdev->{'read-only'} = JSON::false; qmp_cmd($qmp_peer, 'blockdev-reopen', options => [$target_fmt_blockdev]); # For the guest, the drive is still read-only, because the top throttle node is. } eval { my $job_id = "commit-$deviceid"; my $jobs = {}; my $opts = { 'job-id' => $job_id, device => $deviceid }; $opts->{'base-node'} = $target_fmt_blockdev->{'node-name'}; $opts->{'top-node'} = $src_fmt_blockdev->{'node-name'}; qmp_cmd($qmp_peer, "block-commit", %$opts); $jobs->{$job_id} = {}; # If the 'current' state is committed to its backing snapshot, the job will not complete # automatically, because there is a writer, i.e. the guest. It is necessary to use the # 'complete' completion mode, so that the 'current' block node is replaced with the backing # node upon completion. Like that, IO after the commit operation will already land in the # backing node, which will be renamed since it will be the new top of the chain (done by the # caller). # # For other snapshots in the chain, it can be assumed that they have no writer, so # 'block-commit' will complete automatically. my $complete = $src_snap && $src_snap ne 'current' ? 'auto' : 'complete'; PVE::QemuServer::BlockJob::monitor($qmp_peer, undef, $jobs, $complete, 0, 'commit'); blockdev_delete( $storecfg, $qmp_peer, $drive, $src_file_blockdev, $src_fmt_blockdev, $src_snap, ); }; my $err = $@; if ($target_was_read_only) { # Even when restoring the read-only flag on the format and file nodes fails, the top # throttle node still has it, ensuring it is read-only for the guest. print "re-applying read-only flag for internal block node for '$target_snap'\n"; $target_fmt_blockdev->{'read-only'} = JSON::true; $target_file_blockdev->{'read-only'} = JSON::true; eval { qmp_cmd($qmp_peer, 'blockdev-reopen', options => [$target_fmt_blockdev]); }; print "failed to re-apply read-only flag - $@\n" if $@; } die $err if $err; } sub blockdev_stream { my ( $storecfg, $qmp_peer, $machine_version, $deviceid, $drive, $snap, $parent_snap, $target_snap, ) = @_; my $volid = $drive->{file}; $target_snap = undef if $target_snap eq 'current'; my $parent_file_blockdev = generate_file_blockdev( $storecfg, $drive, $machine_version, { 'snapshot-name' => $parent_snap }, ); my $parent_fmt_blockdev = generate_format_blockdev( $storecfg, $drive, $parent_file_blockdev, { 'snapshot-name' => $parent_snap }, ); my $target_file_blockdev = generate_file_blockdev( $storecfg, $drive, $machine_version, { 'snapshot-name' => $target_snap }, ); my $target_fmt_blockdev = generate_format_blockdev( $storecfg, $drive, $target_file_blockdev, { 'snapshot-name' => $target_snap }, ); my $snap_file_blockdev = generate_file_blockdev($storecfg, $drive, $machine_version, { 'snapshot-name' => $snap }); my $snap_fmt_blockdev = generate_format_blockdev( $storecfg, $drive, $snap_file_blockdev, { 'snapshot-name' => $snap }, ); my $backing_file = blockdev_relative_backing_file($parent_file_blockdev, $target_file_blockdev); my $job_id = "stream-$deviceid"; my $jobs = {}; my $options = { 'job-id' => $job_id, device => $target_fmt_blockdev->{'node-name'} }; $options->{'base-node'} = $parent_fmt_blockdev->{'node-name'}; $options->{'backing-file'} = $backing_file; qmp_cmd($qmp_peer, 'block-stream', %$options); $jobs->{$job_id} = {}; PVE::QemuServer::BlockJob::monitor($qmp_peer, undef, $jobs, 'auto', 0, 'stream'); blockdev_delete( $storecfg, $qmp_peer, $drive, $snap_file_blockdev, $snap_fmt_blockdev, $snap, ); } 1; ================================================ FILE: src/PVE/QemuServer.pm ================================================ package PVE::QemuServer; use strict; use warnings; use Cwd 'abs_path'; use Digest::SHA; use Fcntl ':flock'; use Fcntl; use File::Basename; use File::Copy qw(copy); use File::Path; use Getopt::Long; use IO::Dir; use IO::File; use IO::Handle; use IO::Select; use IO::Socket::UNIX; use IPC::Open3; use JSON; use MIME::Base64; use POSIX; use Storable qw(dclone); use Time::HiRes qw(gettimeofday usleep); use URI::Escape; use UUID; use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file); use PVE::CGroup; use PVE::CpuSet; use PVE::DataCenterConfig; use PVE::Exception qw(raise raise_param_exc); use PVE::Format qw(render_duration render_bytes); use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne); use PVE::Mapping::Dir; use PVE::Mapping::PCI; use PVE::Mapping::USB; use PVE::Network::SDN::Vnets; use PVE::INotify; use PVE::JSONSchema qw(get_standard_option parse_property_string); use PVE::ProcFSTools; use PVE::PBSClient; use PVE::RESTEnvironment qw(log_warn); use PVE::RPCEnvironment; use PVE::SafeSyslog; use PVE::Storage; use PVE::SysFSTools; use PVE::Systemd; use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach $IPV6RE); use PVE::QMPClient; use PVE::QemuConfig; use PVE::QemuConfig::NoWrite; use PVE::QemuMigrate::Helpers; use PVE::QemuServer::Agent qw(get_qga_key parse_guest_agent qga_check_running); use PVE::QemuServer::Blockdev; use PVE::QemuServer::BlockJob; use PVE::QemuServer::Cfg2Cmd; use PVE::QemuServer::Helpers qw(config_aware_timeout get_iscsi_initiator_name get_host_arch min_version kvm_user_version windows_version); use PVE::QemuServer::Cloudinit; use PVE::QemuServer::CGroup; use PVE::QemuServer::CPUConfig qw( print_cpu_device get_cpu_options get_cpu_bitness is_native_arch get_amd_sev_object get_intel_tdx_object get_cvm_type ); use PVE::QemuServer::Drive qw( is_valid_drivename checked_volume_format drive_is_cloudinit drive_is_cdrom parse_drive print_drive storage_allows_io_uring_default ); use PVE::QemuServer::DriveDevice qw(print_drivedevice_full scsihw_infos); use PVE::QemuServer::Machine; use PVE::QemuServer::Memory qw(get_current_memory); use PVE::QemuServer::MetaInfo; use PVE::QemuServer::Monitor qw(mon_cmd qmp_cmd vm_qmp_peer); use PVE::QemuServer::Network; use PVE::QemuServer::OVMF; use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci); use PVE::QemuServer::QemuImage; use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel); use PVE::QemuServer::QSD; use PVE::QemuServer::RNG qw(parse_rng print_rng_device_commandline print_rng_object_commandline); use PVE::QemuServer::RunState; use PVE::QemuServer::StateFile; use PVE::QemuServer::USB; use PVE::QemuServer::Virtiofs qw(max_virtiofs start_all_virtiofsd); use PVE::QemuServer::VolumeChain; use PVE::QemuServer::DBusVMState; my $have_ha_config; eval { require PVE::HA::Config; $have_ha_config = 1; }; my sub vm_is_ha_managed { my ($vmid) = @_; die "cannot check if VM is managed by HA, missing HA Config module!\n" if !$have_ha_config; return PVE::HA::Config::vm_is_ha_managed($vmid); } my $cpuinfo = PVE::ProcFSTools::read_cpuinfo(); # Note about locking: we use flock on the config file protect against concurrent actions. # Additionally, we have a 'lock' setting in the config file. This can be set to 'migrate', # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set. # But you can ignore this kind of lock with the --skiplock flag. cfs_register_file( '/qemu-server/', \&parse_vm_config, \&write_vm_config, ); PVE::JSONSchema::register_standard_option( 'pve-qm-stateuri', { description => "Some command save/restore state from this location.", type => 'string', maxLength => 128, optional => 1, }, ); # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way my $nodename_cache; sub nodename { $nodename_cache //= PVE::INotify::nodename(); return $nodename_cache; } my $watchdog_fmt = { model => { default_key => 1, type => 'string', enum => [qw(i6300esb ib700)], description => "Watchdog type to emulate.", default => 'i6300esb', optional => 1, }, action => { type => 'string', enum => [qw(reset shutdown poweroff pause debug none)], description => "The action to perform if after activation the guest fails to poll the watchdog in time.", optional => 1, }, }; PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt); my $vga_fmt = { type => { description => "Select the VGA type. Using type 'cirrus' is not recommended.", type => 'string', default => 'std', optional => 1, default_key => 1, enum => [ qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware) ], }, memory => { description => "Sets the VGA memory (in MiB). Has no effect with serial display.", type => 'integer', optional => 1, minimum => 4, maximum => 512, }, clipboard => { description => 'Enable a specific clipboard. If not set, depending on the display type the SPICE one' . ' will be added. Live migration with a VNC clipboard is not possible with QEMU' . ' machine version < 10.1.', type => 'string', enum => ['vnc'], optional => 1, }, }; my $ivshmem_fmt = { size => { type => 'integer', minimum => 1, description => "The size of the file in MB.", }, name => { type => 'string', pattern => '[a-zA-Z0-9\-]+', optional => 1, format_description => 'string', description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.", }, }; my $audio_fmt = { device => { type => 'string', enum => [qw(ich9-intel-hda intel-hda AC97)], description => "Configure an audio device.", }, driver => { type => 'string', enum => ['spice', 'none'], default => 'spice', optional => 1, description => "Driver backend for the audio device.", }, }; my $spice_enhancements_fmt = { foldersharing => { type => 'boolean', optional => 1, default => '0', description => "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM.", }, videostreaming => { type => 'string', enum => ['off', 'all', 'filter'], default => 'off', optional => 1, description => "Enable video streaming. Uses compression for detected video streams.", }, }; my $confdesc = { onboot => { optional => 1, type => 'boolean', description => "Specifies whether a VM will be started during system bootup.", default => 0, }, autostart => { optional => 1, type => 'boolean', description => "Automatic restart after crash (currently ignored).", default => 0, }, hotplug => { optional => 1, type => 'string', format => 'pve-hotplug-features', description => "Selectively enable hotplug features. This is a comma separated list of" . " hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable" . " hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`." . " USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or" . " windows > 7.", default => 'network,disk,usb', }, reboot => { optional => 1, type => 'boolean', description => "Allow reboot. If set to '0' the VM exit on reboot.", default => 1, }, lock => { optional => 1, type => 'string', description => "Lock/unlock the VM.", enum => [ qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)], }, cpulimit => { optional => 1, type => 'number', description => "Limit of CPU usage.", verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has" . " total of '2' CPU time. Value '0' indicates no CPU limit.", minimum => 0, maximum => 128, default => 0, }, cpuunits => { optional => 1, type => 'integer', description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.", verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler." . " The larger the number is, the more CPU time this VM gets. Number is relative to" . " weights of all the other running VMs.", minimum => 1, maximum => 262144, default => 'cgroup v1: 1024, cgroup v2: 100', }, memory => { optional => 1, type => 'string', description => "Memory properties.", format => $PVE::QemuServer::Memory::memory_fmt, }, 'amd-sev' => { description => "Secure Encrypted Virtualization (SEV) features by AMD CPUs", optional => 1, format => 'pve-qemu-sev-fmt', type => 'string', }, 'intel-tdx' => { description => "Trusted Domain Extension (TDX) features by Intel CPUs", optional => 1, format => 'pve-qemu-tdx-fmt', type => 'string', }, balloon => { optional => 1, type => 'integer', description => "Amount of target RAM for the VM in MiB. The balloon driver is enabled by default," . " unless it is explicitly disabled by setting the value to zero.", minimum => 0, }, shares => { optional => 1, type => 'integer', description => "Amount of memory shares for auto-ballooning. The larger the number is, the" . " more memory this VM gets. Number is relative to weights of all other running VMs." . " Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.", minimum => 0, maximum => 50000, default => 1000, }, keyboard => { optional => 1, type => 'string', description => "Keyboard layout for VNC server. This option is generally not required and" . " is often better handled from within the guest OS.", enum => PVE::Tools::kvmkeymaplist(), default => undef, }, name => { optional => 1, type => 'string', format => 'dns-name', description => "Set a name for the VM. Only used on the configuration web interface.", }, scsihw => { optional => 1, type => 'string', description => "SCSI controller model", enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)], default => 'lsi', }, description => { optional => 1, type => 'string', description => "Description for the VM. Shown in the web-interface VM's summary." . " This is saved as comment inside the configuration file.", maxLength => 1024 * 8, }, ostype => { optional => 1, type => 'string', # NOTE: When extending, also consider extending `%guest_types` in `Import/ESXi.pm`. enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)], default => 'other', description => "Specify guest operating system.", verbose_description => < { optional => 1, type => 'string', format => 'pve-qm-boot', description => "Specify guest boot order. Use the 'order=' sub-property as usage with no" . " key or 'legacy=' is deprecated.", }, bootdisk => { optional => 1, type => 'string', format => 'pve-qm-bootdisk', description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.", pattern => '(ide|sata|scsi|virtio)\d+', }, smp => { optional => 1, type => 'integer', description => "The number of CPUs. Please use option -sockets instead.", minimum => 1, default => 1, }, sockets => { optional => 1, type => 'integer', description => "The number of CPU sockets.", minimum => 1, default => 1, }, cores => { optional => 1, type => 'integer', description => "The number of cores per socket.", minimum => 1, default => 1, }, numa => { optional => 1, type => 'boolean', description => "Enable/disable NUMA.", default => 0, }, hugepages => { optional => 1, type => 'string', description => "Enables hugepages memory.\n\nSets the size of hugepages in MiB. If the value " . "is set to 'any' then 1 GiB hugepages will be used if possible, " . "otherwise the size will fall back to 2 MiB.", enum => [qw(any 2 1024)], }, keephugepages => { optional => 1, type => 'boolean', default => 0, description => "Use together with hugepages. If enabled, hugepages will not not be deleted" . " after VM shutdown and can be used for subsequent starts.", }, vcpus => { optional => 1, type => 'integer', description => "Number of hotplugged vcpus.", minimum => 1, default => 0, }, acpi => { optional => 1, type => 'boolean', description => "Enable/disable ACPI.", default => 1, }, agent => { optional => 1, description => "Enable/disable communication with the QEMU Guest Agent and its properties.", type => 'string', format => $PVE::QemuServer::Agent::agent_fmt, }, kvm => { optional => 1, type => 'boolean', description => "Enable/disable KVM hardware virtualization.", default => 1, }, tdf => { optional => 1, type => 'boolean', description => "Enable/disable time drift fix.", default => 0, }, localtime => { optional => 1, type => 'boolean', description => "Set the real time clock (RTC) to local time. This is enabled by default if" . " the `ostype` indicates a Microsoft Windows OS.", }, freeze => { optional => 1, type => 'boolean', description => "Freeze CPU at startup (use 'c' monitor command to start execution).", }, vga => { optional => 1, type => 'string', format => $vga_fmt, description => "Configure the VGA hardware.", verbose_description => "Configure the VGA Hardware. If you want to use high resolution" . " modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU" . " 2.9 the default VGA display type is 'std' for all OS types besides some Windows" . " versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE" . " display server. For win* OS you can select how many independent displays you want," . " Linux guests can add displays them self.\nYou can also run without any graphic card," . " using a serial device as terminal.", }, watchdog => { optional => 1, type => 'string', format => 'pve-qm-watchdog', description => "Create a virtual hardware watchdog device.", verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest" . " action), the watchdog must be periodically polled by an agent inside the guest or" . " else the watchdog will reset the guest (or execute the respective action specified)", }, startdate => { optional => 1, type => 'string', typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)", description => "Set the initial date of the real time clock. Valid format for date are:" . "'now' or '2006-06-17T16:01:21' or '2006-06-17'.", pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)', default => 'now', }, startup => get_standard_option('pve-startup-order'), template => { optional => 1, type => 'boolean', description => "Enable/disable Template.", default => 0, }, args => { optional => 1, type => 'string', description => "Arbitrary arguments passed to kvm.", verbose_description => < { optional => 1, type => 'boolean', default => 1, description => "Enable/disable the USB tablet device.", verbose_description => "Enable/disable the USB tablet device. This device is usually needed" . " to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with" . " normal VNC clients. If you're running lots of console-only guests on one host, you" . " may consider disabling this to save some context switches. This is turned off by" . " default if you use spice (`qm set --vga qxl`).", }, migrate_speed => { optional => 1, type => 'integer', description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.", minimum => 0, default => 0, }, migrate_downtime => { optional => 1, type => 'number', description => "Set maximum tolerated downtime (in seconds) for migrations. Should the" . " migration not be able to converge in the very end, because too much newly dirtied" . " RAM needs to be transferred, the limit will be increased automatically step-by-step" . " until migration can converge. Will be capped to 2000 seconds (maximum in QEMU).", minimum => 0, default => 0.1, }, cdrom => { optional => 1, type => 'string', format => 'pve-qm-ide', typetext => '', description => "This is an alias for option -ide2", }, cpu => { optional => 1, description => "Emulated CPU type.", type => 'string', format => 'pve-vm-cpu-conf', }, parent => get_standard_option( 'pve-snapshot-name', { optional => 1, description => "Parent snapshot name. This is used internally, and should not be modified.", }, ), snaptime => { optional => 1, description => "Timestamp for snapshots.", type => 'integer', minimum => 0, }, vmstate => { optional => 1, type => 'string', format => 'pve-volume-id', description => "Reference to a volume which stores the VM state. This is used internally" . " for snapshots.", }, vmstatestorage => get_standard_option( 'pve-storage-id', { description => "Default storage for VM state volumes/files.", optional => 1, }, ), runningmachine => get_standard_option( 'pve-qemu-machine', { description => "Specifies the QEMU machine type of the running vm. This is used internally" . " for snapshots.", }, ), runningcpu => { description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used" . " internally for snapshots.", optional => 1, type => 'string', pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re, format_description => 'QEMU -cpu parameter', }, 'running-nets-host-mtu' => { type => 'string', pattern => 'net\d+=\d+(,net\d+=\d+)*', optional => 1, description => 'List of VirtIO network devices and their effective host_mtu setting. A value of 0' . ' means that the host_mtu parameter is to be avoided for the corresponding device.' . ' This is used internally for snapshots.', }, machine => get_standard_option('pve-qemu-machine'), arch => get_standard_option('pve-qm-cpu-arch', { optional => 1 }), smbios1 => { description => "Specify SMBIOS type 1 fields.", type => 'string', format => 'pve-qm-smbios1', maxLength => 512, optional => 1, }, protection => { optional => 1, type => 'boolean', description => "Sets the protection flag of the VM. This will disable the remove VM and" . " remove disk operations.", default => 0, }, bios => { optional => 1, type => 'string', enum => [qw(seabios ovmf)], description => "Select BIOS implementation.", default => 'seabios', }, vmgenid => { type => 'string', pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])', format_description => 'UUID', description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'" . " to disable explicitly.", verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer" . " value identifier to the guest OS. This allows to notify the guest operating system" . " when the virtual machine is executed with a different configuration (e.g. snapshot" . " execution or creation from a template). The guest operating system notices the" . " change, and is then able to react as appropriate by marking its copies of" . " distributed databases as dirty, re-initializing its random number generator, etc.\n" . "Note that auto-creation only works when done through API/CLI create or update methods" . ", but not when manually editing the config file.", default => "1 (autogenerated)", optional => 1, }, hookscript => { type => 'string', format => 'pve-volume-id', optional => 1, description => "Script that will be executed during various steps in the vms lifetime.", }, ivshmem => { type => 'string', format => $ivshmem_fmt, description => "Inter-VM shared memory. Useful for direct communication between VMs, or to" . " the host.", optional => 1, }, audio0 => { type => 'string', format => $audio_fmt, description => "Configure a audio device, useful in combination with QXL/Spice.", optional => 1, }, spice_enhancements => { type => 'string', format => $spice_enhancements_fmt, description => "Configure additional enhancements for SPICE.", optional => 1, }, tags => { type => 'string', format => 'pve-tag-list', description => 'Tags of the VM. This is only meta information.', optional => 1, }, rng0 => { type => 'string', format => 'pve-qm-rng', description => "Configure a VirtIO-based Random Number Generator.", optional => 1, }, meta => { type => 'string', format => $PVE::QemuServer::MetaInfo::meta_info_fmt, description => "Some (read-only) meta-information about this guest.", optional => 1, }, affinity => { type => 'string', format => 'pve-cpuset', description => "List of host cores used to execute guest processes, for example: 0,5,8-11", optional => 1, }, 'allow-ksm' => { type => 'boolean', description => "Allow memory pages of this guest to be merged via KSM (Kernel Samepage" . " Merging).", optional => 1, default => 1, }, }; my $cicustom_fmt = { meta => { type => 'string', optional => 1, description => 'Specify a custom file containing all meta data passed to the VM via' . ' cloud-init. This is provider specific meaning configdrive2 and nocloud differ.', format => 'pve-volume-id', format_description => 'volume', }, network => { type => 'string', optional => 1, description => 'To pass a custom file containing all network data to the VM via cloud-init.', format => 'pve-volume-id', format_description => 'volume', }, user => { type => 'string', optional => 1, description => 'To pass a custom file containing all user data to the VM via cloud-init.', format => 'pve-volume-id', format_description => 'volume', }, vendor => { type => 'string', optional => 1, description => 'To pass a custom file containing all vendor data to the VM via cloud-init.', format => 'pve-volume-id', format_description => 'volume', }, }; PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt); # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu my $confdesc_cloudinit = { citype => { optional => 1, type => 'string', description => 'Specifies the cloud-init configuration format. The default depends on the' . ' configured operating system type (`ostype`. We use the `nocloud` format for Linux,' . ' and `configdrive2` for windows.', enum => ['configdrive2', 'nocloud', 'opennebula'], }, ciuser => { optional => 1, type => 'string', description => "cloud-init: User name to change ssh keys and password for instead of the" . " image's configured default user.", }, cipassword => { optional => 1, type => 'string', description => 'cloud-init: Password to assign the user. Using this is generally not' . ' recommended. Use ssh keys instead. Also note that older cloud-init versions do not' . ' support hashed passwords.', }, ciupgrade => { optional => 1, type => 'boolean', description => 'cloud-init: do an automatic package upgrade after the first boot.', default => 1, }, cicustom => { optional => 1, type => 'string', description => 'cloud-init: Specify custom files to replace the automatically generated' . ' ones at start.', format => 'pve-qm-cicustom', }, searchdomain => { optional => 1, type => 'string', description => 'cloud-init: Sets DNS search domains for a container. Create will' . ' automatically use the setting from the host if neither searchdomain nor nameserver' . ' are set.', }, nameserver => { optional => 1, type => 'string', format => 'address-list', description => 'cloud-init: Sets DNS server IP address for a container. Create will' . ' automatically use the setting from the host if neither searchdomain nor nameserver' . ' are set.', }, sshkeys => { optional => 1, type => 'string', format => 'urlencoded', description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).", }, }; # what about other qemu settings ? #cpu => 'string', #machine => 'string', #fda => 'file', #fdb => 'file', #mtdblock => 'file', #sd => 'file', #pflash => 'file', #snapshot => 'bool', #bootp => 'file', ##tftp => 'dir', ##smb => 'dir', #kernel => 'file', #append => 'string', #initrd => 'file', ##soundhw => 'string', while (my ($k, $v) = each %$confdesc) { PVE::JSONSchema::register_standard_option("pve-qm-$k", $v); } my $MAX_NETS = 32; my $MAX_SERIAL_PORTS = 4; my $MAX_PARALLEL_PORTS = 3; for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++) { $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc; } for (my $i = 0; $i < max_virtiofs(); $i++) { $confdesc->{"virtiofs$i"} = get_standard_option('pve-qm-virtiofs'); } for (my $i = 0; $i < $MAX_NETS; $i++) { $confdesc->{"net$i"} = $PVE::QemuServer::Network::netdesc; $confdesc_cloudinit->{"ipconfig$i"} = $PVE::QemuServer::Network::ipconfigdesc; } foreach my $key (keys %$confdesc_cloudinit) { $confdesc->{$key} = $confdesc_cloudinit->{$key}; } PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset); sub pve_verify_cpuset { my ($set_text, $noerr) = @_; my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) }; if ($@) { return if $noerr; die "unable to parse cpuset option\n"; } return PVE::CpuSet->new($members)->short_string(); } PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path); sub verify_volume_id_or_qm_path { my ($volid, $noerr) = @_; return $volid if $volid eq 'none' || $volid eq 'cdrom'; return verify_volume_id_or_absolute_path($volid, $noerr); } PVE::JSONSchema::register_format( 'pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path, ); sub verify_volume_id_or_absolute_path { my ($volid, $noerr) = @_; return $volid if $volid =~ m|^/|; $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') }; if ($@) { return if $noerr; die $@; } return $volid; } my $serialdesc = { optional => 1, type => 'string', pattern => '(/dev/.+|socket)', description => "Create a serial device inside the VM (n is 0 to 3)", verbose_description => < 1, type => 'string', pattern => '/dev/parport\d+|/dev/usb/lp\d+', description => "Map host parallel devices (n is 0 to 2).", verbose_description => <{"parallel$i"} = $paralleldesc; } for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) { $confdesc->{"serial$i"} = $serialdesc; } for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) { $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc; } for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) { $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key}; } for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) { $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc; } my $boot_fmt = { legacy => { optional => 1, default_key => 1, type => 'string', description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)." . " Deprecated, use 'order=' instead.", pattern => '[acdn]{1,4}', format_description => "[acdn]{1,4}", # note: this is also the fallback if boot: is not given at all default => 'cdn', }, order => { optional => 1, type => 'string', format => 'pve-qm-bootdev-list', format_description => "device[;device...]", description => <{$dev}; return 1; }; return $dev if $check->("net"); return $dev if $check->("usb"); return $dev if $check->("hostpci"); return if $noerr; die "invalid boot device '$dev'\n"; } sub print_bootorder { my ($devs) = @_; return "" if !@$devs; my $data = { order => join(';', @$devs) }; return PVE::JSONSchema::print_property_string($data, $boot_fmt); } my $kvm_api_version = 0; sub kvm_version { return $kvm_api_version if $kvm_api_version; open my $fh, '<', '/dev/kvm' or return; # 0xae00 => KVM_GET_API_VERSION $kvm_api_version = ioctl($fh, 0xae00, 0); close($fh); return $kvm_api_version; } my sub extract_version { my ($machine_type, $version) = @_; $version = kvm_user_version() if !defined($version); return PVE::QemuServer::Machine::extract_version($machine_type, $version); } sub kernel_has_vhost_net { return -c '/dev/vhost-net'; } sub option_exists { my $key = shift; return defined($confdesc->{$key}); } # try to convert old style file names to volume IDs sub filename_to_volume_id { my ($vmid, $file, $media) = @_; if (!( $file eq 'none' || $file eq 'cdrom' || $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/ )) { return if $file =~ m|/|; if ($media && $media eq 'cdrom') { $file = "local:iso/$file"; } else { $file = "local:$vmid/$file"; } } return $file; } sub verify_media_type { my ($opt, $vtype, $media) = @_; return if !$media; my $etype; if ($media eq 'disk') { $etype = 'images'; } elsif ($media eq 'cdrom') { $etype = 'iso'; } else { die "internal error"; } return if ($vtype eq $etype); raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" }); } sub cleanup_drive_path { my ($opt, $storecfg, $drive) = @_; # try to convert filesystem paths to volume IDs if ( ($drive->{file} !~ m/^(cdrom|none)$/) && ($drive->{file} !~ m|^/dev/.+|) && ($drive->{file} !~ m/^([^:]+):(.+)$/) && ($drive->{file} !~ m/^\d+$/) ) { my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file}); raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage" }) if !$vtype; $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso'; verify_media_type($opt, $vtype, $drive->{media}); $drive->{file} = $volid; } $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/; } sub parse_hotplug_features { my ($data) = @_; my $res = {}; return $res if $data eq '0'; $data = $confdesc->{hotplug}->{default} if $data eq '1'; foreach my $feature (PVE::Tools::split_list($data)) { if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) { $res->{$1} = 1; } else { die "invalid hotplug feature '$feature'\n"; } } return $res; } PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features); sub pve_verify_hotplug_features { my ($value, $noerr) = @_; return $value if parse_hotplug_features($value); return if $noerr; die "unable to parse hotplug option\n"; } sub assert_clipboard_config { my ($vga) = @_; my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/; if ( $vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc' && $vga->{type} && $vga->{type} !~ $clipboard_regex ) { die "vga type $vga->{type} is not compatible with VNC clipboard\n"; } } sub print_tabletdevice_full { my ($conf, $arch) = @_; my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf); # we use uhci for old VMs because tablet driver was buggy in older qemu my $usbbus; if ($q35 || $arch eq 'aarch64') { $usbbus = 'ehci'; } else { $usbbus = 'uhci'; } return "usb-tablet,id=tablet,bus=$usbbus.0,port=1"; } sub print_keyboarddevice_full { my ($conf, $arch) = @_; return if $arch ne 'aarch64'; return "usb-kbd,id=keyboard,bus=ehci.0,port=2"; } sub print_drive_commandline_full { my ($storecfg, $vmid, $drive, $live_restore_name) = @_; my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive); my ($storeid) = PVE::Storage::parse_volume_id($drive->{file}, 1); my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef; my $vtype = $storeid ? (PVE::Storage::parse_volname($storecfg, $drive->{file}))[0] : undef; my ($path, $format) = PVE::QemuServer::Drive::get_path_and_format($storecfg, $drive, $live_restore_name); if ($scfg && $scfg->{'snapshot-as-volume-chain'} && $format && $format eq 'qcow2') { # the print_drive_commandline_full() function is only used if machine version is < 10.0 die "storage for '$drive->{file}' is configured for snapshots as a volume chain - this" . " requires QEMU machine version >= 10.0. See" . " https://pve.proxmox.com/wiki/QEMU_Machine_Version_Upgrade\n"; } my $is_rbd = $path =~ m/^rbd:/; my $opts = ''; my @qemu_drive_options = qw(media cache rerror werror discard); foreach my $o (@qemu_drive_options) { $opts .= ",$o=$drive->{$o}" if defined($drive->{$o}); } # snapshot only accepts on|off if (defined($drive->{snapshot})) { my $v = $drive->{snapshot} ? 'on' : 'off'; $opts .= ",snapshot=$v"; } if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off'); } foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) { my ($dir, $qmpname) = @$type; if (my $v = $drive->{"mbps$dir"}) { $opts .= ",throttling.bps$qmpname=" . int($v * 1024 * 1024); } if (my $v = $drive->{"mbps${dir}_max"}) { $opts .= ",throttling.bps$qmpname-max=" . int($v * 1024 * 1024); } if (my $v = $drive->{"bps${dir}_max_length"}) { $opts .= ",throttling.bps$qmpname-max-length=$v"; } if (my $v = $drive->{"iops${dir}"}) { $opts .= ",throttling.iops$qmpname=$v"; } if (my $v = $drive->{"iops${dir}_max"}) { $opts .= ",throttling.iops$qmpname-max=$v"; } if (my $v = $drive->{"iops${dir}_max_length"}) { $opts .= ",throttling.iops$qmpname-max-length=$v"; } } if ($live_restore_name) { $format = "rbd" if $is_rbd; die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n" if !$format; $opts .= ",format=alloc-track,file.driver=$format"; } elsif ($format) { $opts .= ",format=$format"; } my $cache_direct = PVE::QemuServer::Drive::drive_uses_cache_direct($drive, $scfg); $opts .= ",cache=none" if !$drive->{cache} && $cache_direct; my $aio = PVE::QemuServer::Drive::aio_cmdline_option($scfg, $drive, $cache_direct); $opts .= ",aio=$aio"; die "$drive_id: explicit media parameter is required for iso images\n" if !defined($drive->{media}) && defined($vtype) && $vtype eq 'iso'; if (!drive_is_cdrom($drive)) { my $detectzeroes = PVE::QemuServer::Drive::detect_zeroes_cmdline_option($drive); # note: 'detect-zeroes' works per blockdev and we want it to persist # after the alloc-track is removed, so put it on 'file' directly my $dz_param = $live_restore_name ? "file.detect-zeroes" : "detect-zeroes"; $opts .= ",$dz_param=$detectzeroes" if $detectzeroes; } if ($live_restore_name) { $opts .= ",backing=$live_restore_name"; $opts .= ",auto-remove=on"; } # my $file_param = $live_restore_name ? "file.file.filename" : "file"; my $file_param = "file"; if ($live_restore_name) { # non-rbd drivers require the underlying file to be a separate block # node, so add a second .file indirection $file_param .= ".file" if !$is_rbd; $file_param .= ".filename"; } my $pathinfo = $path ? "$file_param=$path," : ''; return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts"; } sub print_pbs_blockdev { my ($pbs_conf, $pbs_name) = @_; my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on"; $blockdev .= ",repository=$pbs_conf->{repository}"; $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace}; $blockdev .= ",snapshot=$pbs_conf->{snapshot}"; $blockdev .= ",archive=$pbs_conf->{archive}"; $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile}; return $blockdev; } sub print_netdevice_full { my ( $vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_version, $host_mtu_migration, # force this value for host_mtu, 0 means force absence of param ) = @_; my $device = $net->{model}; if ($net->{model} eq 'virtio') { $device = 'virtio-net-pci'; } my $pciaddr = print_pci_addr("$netid", $bridges, $arch); my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid"; if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio') { # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in # and out of each queue plus one config interrupt and control vector queue my $vectors = $net->{queues} * 2 + 2; $tmpstr .= ",vectors=$vectors,mq=on"; if (min_version($machine_version, 7, 1)) { $tmpstr .= ",packed=on"; } } if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio') { $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256"; } $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex}; my $mtu = $net->{mtu}; my $migration_skip_host_mtu = defined($host_mtu_migration) && $host_mtu_migration == 0; print "netdev $netid: not adding 'host_mtu' parameter for migration compat\n" if $migration_skip_host_mtu; if ($net->{model} eq 'virtio' && $net->{bridge} && !$migration_skip_host_mtu) { my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge}); if ($host_mtu_migration) { print "netdev $netid: using 'host_mtu=$host_mtu_migration' for migration compat\n"; $mtu = $host_mtu_migration; } if (!defined($mtu) || $mtu == 1) { $mtu = $bridge_mtu; } elsif ($mtu < 576) { die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n"; } elsif ($mtu > $bridge_mtu) { die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'" . " - adjust the MTU for the network device in the VM configuration, while ensuring" . " that the bridge is configured as desired.\n"; } if (min_version($machine_version, 10, 0, 1) || $host_mtu_migration) { # Always add host_mtu for migration compatibility, because the presence of host_mtu # means that the virtual hardware is generated differently (at least for i440fx) $tmpstr .= ",host_mtu=$mtu"; } else { $tmpstr .= ",host_mtu=$mtu" if $mtu != 1500; } } elsif (defined($mtu)) { my $msg_prefix = "netdev $netid: ignoring MTU '$mtu'"; if ($migration_skip_host_mtu) { # When the machine version is less than 10.0+pve1 and the MTU is 1500, not having the # host_mtu parameter is fully expected. Only log when not expected to avoid confusion. if (min_version($machine_version, 10, 0, 1) || $mtu != 1500) { log_warn( "$msg_prefix, not used on the source side according to migration parameters"); } } elsif (!$net->{bridge}) { log_warn("$msg_prefix, no bridge configured"); } else { log_warn("$msg_prefix, not using VirtIO"); } } if ($use_old_bios_files) { my $romfile; if ($device eq 'virtio-net-pci') { $romfile = 'pxe-virtio.rom'; } elsif ($device eq 'e1000') { $romfile = 'pxe-e1000.rom'; } elsif ($device eq 'e1000e') { $romfile = 'pxe-e1000e.rom'; } elsif ($device eq 'ne2k') { $romfile = 'pxe-ne2k_pci.rom'; } elsif ($device eq 'pcnet') { $romfile = 'pxe-pcnet.rom'; } elsif ($device eq 'rtl8139') { $romfile = 'pxe-rtl8139.rom'; } $tmpstr .= ",romfile=$romfile" if $romfile; } return $tmpstr; } sub print_netdev_full { my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_; my $i = ''; if ($netid =~ m/^net(\d+)$/) { $i = int($1); } die "got strange net id '$i'\n" if $i >= ${MAX_NETS}; my $ifname = "tap${vmid}i$i"; # kvm uses TUNSETIFF ioctl, and that limits ifname length die "interface name '$ifname' is too long (max 15 character)\n" if length($ifname) >= 16; my $vhostparam = ''; if (is_native_arch($arch)) { $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio'; } my $vmname = $conf->{name} || "vm$vmid"; my $netdev = ""; my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge"; if ($net->{bridge}) { $netdev = "type=tap,id=$netid,ifname=${ifname},script=/usr/libexec/qemu-server/$script" . ",downscript=/usr/libexec/qemu-server/pve-bridgedown$vhostparam"; } else { $netdev = "type=user,id=$netid,hostname=$vmname"; } $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio'); return $netdev; } my $vga_map = { 'cirrus' => 'cirrus-vga', 'std' => 'VGA', 'vmware' => 'vmware-svga', 'virtio' => 'virtio-vga', 'virtio-gl' => 'virtio-vga-gl', }; sub print_vga_device { my ($conf, $vga, $arch, $machine_version, $id, $qxlnum, $bridges) = @_; my $type = $vga_map->{ $vga->{type} }; if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') { $type = 'virtio-gpu'; } my $vgamem_mb = $vga->{memory}; my $max_outputs = ''; if ($qxlnum) { $type = $id ? 'qxl' : 'qxl-vga'; if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) { # set max outputs so linux can have up to 4 qxl displays with one device if (min_version($machine_version, 4, 1)) { $max_outputs = ",max_outputs=4"; } } } die "no device-type for $vga->{type}\n" if !$type; my $memory = ""; if ($vgamem_mb) { if ($vga->{type} =~ /^virtio/) { my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b"); $memory = ",max_hostmem=$bytes"; } elsif ($qxlnum) { # from https://www.spice-space.org/multiple-monitors.html $memory = ",vgamem_mb=$vga->{memory}"; my $ram = $vgamem_mb * 4; my $vram = $vgamem_mb * 2; $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram"; } else { $memory = ",vgamem_mb=$vga->{memory}"; } } elsif ($qxlnum && $id) { $memory = ",ram_size=67108864,vram_size=33554432"; } my $edidoff = ""; if ($type eq 'VGA' && windows_version($conf->{ostype})) { $edidoff = ",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf'); } my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf); my $vgaid = "vga" . ($id // ''); my $pciaddr; if ($q35 && $vgaid eq 'vga') { # the first display uses pcie.0 bus on q35 machines $pciaddr = print_pcie_addr($vgaid); } else { $pciaddr = print_pci_addr($vgaid, $bridges, $arch); } if ($vga->{type} eq 'virtio-gl') { my $base = '/usr/lib/x86_64-linux-gnu/lib'; die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n" if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1"; die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n" if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*"); } return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}"; } sub vm_is_volid_owner { my ($storecfg, $vmid, $volid) = @_; if ($volid !~ m|^/|) { my ($path, $owner); eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); }; if ($owner && ($owner == $vmid)) { return 1; } } return; } sub vmconfig_register_unused_drive { my ($storecfg, $vmid, $conf, $drive) = @_; if (drive_is_cloudinit($drive)) { eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) }; warn $@ if $@; delete $conf->{'special-sections'}->{cloudinit}; } elsif (!drive_is_cdrom($drive)) { my $volid = $drive->{file}; if (vm_is_volid_owner($storecfg, $vmid, $volid)) { PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid); } } } # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool] my $smbios1_fmt = { uuid => { type => 'string', pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}', format_description => 'UUID', description => "Set SMBIOS1 UUID.", optional => 1, }, version => { type => 'string', pattern => '[A-Za-z0-9+\/]+={0,2}', format_description => 'Base64 encoded string', description => "Set SMBIOS1 version.", optional => 1, }, serial => { type => 'string', pattern => '[A-Za-z0-9+\/]+={0,2}', format_description => 'Base64 encoded string', description => "Set SMBIOS1 serial number.", optional => 1, }, manufacturer => { type => 'string', pattern => '[A-Za-z0-9+\/]+={0,2}', format_description => 'Base64 encoded string', description => "Set SMBIOS1 manufacturer.", optional => 1, }, product => { type => 'string', pattern => '[A-Za-z0-9+\/]+={0,2}', format_description => 'Base64 encoded string', description => "Set SMBIOS1 product ID.", optional => 1, }, sku => { type => 'string', pattern => '[A-Za-z0-9+\/]+={0,2}', format_description => 'Base64 encoded string', description => "Set SMBIOS1 SKU string.", optional => 1, }, family => { type => 'string', pattern => '[A-Za-z0-9+\/]+={0,2}', format_description => 'Base64 encoded string', description => "Set SMBIOS1 family string.", optional => 1, }, base64 => { type => 'boolean', description => 'Flag to indicate that the SMBIOS values are base64 encoded', optional => 1, }, }; sub parse_smbios1 { my ($data) = @_; my $res = eval { parse_property_string($smbios1_fmt, $data) }; warn $@ if $@; return $res; } sub print_smbios1 { my ($smbios1) = @_; return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt); } PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt); sub parse_watchdog { my ($value) = @_; return if !$value; my $res = eval { parse_property_string($watchdog_fmt, $value) }; warn $@ if $@; return $res; } sub parse_vga { my ($value) = @_; return {} if !$value; my $res = eval { parse_property_string($vga_fmt, $value) }; warn $@ if $@; return $res; } sub qemu_created_version_fixups { my ($conf, $forcemachine, $kvmver) = @_; my $meta = PVE::QemuServer::MetaInfo::parse_meta_info($conf->{meta}) // {}; my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine); # check if we need to apply some handling for VMs that always use the latest machine version but # had a machine version transition happen that affected HW such that, e.g., an OS config change # would be required (we do not want to pin machine version for non-windows OS type) my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine}); if ( (!defined($machine_conf->{type}) || $machine_conf->{type} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1 && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations && min_version($kvmver, 6, 1) # only need to apply the change since 6.1 ) { my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf); if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') { # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view # and thus with the predictable interface naming of systemd return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off']; } } return; } # add JSON properties for create and set function sub json_config_properties { my ($prop, $with_disk_alloc, $with_snapshot_info) = @_; my $skip_json_config_opts; if (!$with_snapshot_info) { $skip_json_config_opts = { parent => 1, snaptime => 1, vmstate => 1, 'running-nets-host-mtu' => 1, runningmachine => 1, runningcpu => 1, meta => 1, }; } foreach my $opt (keys %$confdesc) { next if $skip_json_config_opts->{$opt}; if ($with_disk_alloc && is_valid_drivename($opt)) { $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt}; } else { $prop->{$opt} = $confdesc->{$opt}; } } return $prop; } # Properties that we can read from an OVF file sub json_ovf_properties { my $prop = {}; for my $device (PVE::QemuServer::Drive::valid_drive_names()) { $prop->{$device} = { type => 'string', format => 'pve-volume-id-or-absolute-path', description => "Disk image that gets imported to $device", optional => 1, }; } $prop->{cores} = { type => 'integer', description => "The number of CPU cores.", optional => 1, }; $prop->{memory} = { type => 'integer', description => "Amount of RAM for the VM in MB.", optional => 1, }; $prop->{name} = { type => 'string', description => "Name of the VM.", optional => 1, }; return $prop; } # return copy of $confdesc_cloudinit to generate documentation sub cloudinit_config_properties { return dclone($confdesc_cloudinit); } sub cloudinit_pending_properties { my $p = { map { $_ => 1 } keys $confdesc_cloudinit->%*, name => 1, }; $p->{"net$_"} = 1 for 0 .. ($MAX_NETS - 1); return $p; } sub check_type { my ($key, $value, $schema) = @_; die "check_type: no schema defined\n" if !$schema; die "unknown setting '$key'\n" if !$schema->{$key}; my $type = $schema->{$key}->{type}; if (!defined($value)) { die "got undefined value\n"; } if ($value =~ m/[\n\r]/) { die "property contains a line feed\n"; } if ($type eq 'boolean') { return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i); return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i); die "type check ('boolean') failed - got '$value'\n"; } elsif ($type eq 'integer') { return int($1) if $value =~ m/^(\d+)$/; die "type check ('integer') failed - got '$value'\n"; } elsif ($type eq 'number') { return $value if $value =~ m/^(\d+)(\.\d+)?$/; die "type check ('number') failed - got '$value'\n"; } elsif ($type eq 'string') { if (my $fmt = $schema->{$key}->{format}) { PVE::JSONSchema::check_format($fmt, $value); return $value; } $value =~ s/^\"(.*)\"$/$1/; return $value; } else { die "internal error"; } } sub destroy_vm { my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_; eval { PVE::QemuConfig::cleanup_fleecing_images($vmid, $storecfg) }; log_warn("attempt to clean up left-over fleecing images failed - $@") if $@; my $conf = PVE::QemuConfig->load_config($vmid); if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) { PVE::QemuConfig->check_lock($conf); } if ($conf->{template}) { # check if any base image is still used by a linked clone PVE::QemuConfig->foreach_volume_full( $conf, { include_unused => 1 }, sub { my ($ds, $drive) = @_; return if drive_is_cdrom($drive); my $volid = $drive->{file}; return if !$volid || $volid =~ m|^/|; die "base volume '$volid' is still in use by linked cloned\n" if PVE::Storage::volume_is_base_and_used($storecfg, $volid); }, ); } my $volids = {}; my $remove_owned_drive = sub { my ($ds, $drive) = @_; return if drive_is_cdrom($drive, 1); my $volid = $drive->{file}; return if !$volid || $volid =~ m|^/|; return if $volids->{$volid}; my ($path, $owner) = PVE::Storage::path($storecfg, $volid); return if !$path || !$owner || ($owner != $vmid); $volids->{$volid} = 1; eval { PVE::Storage::vdisk_free($storecfg, $volid) }; warn "Could not remove disk '$volid', check manually: $@" if $@; }; # only remove disks owned by this VM (referenced in the config) my $include_opts = { include_unused => 1, extra_keys => ['vmstate'], }; PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive); for my $snap (values %{ $conf->{snapshots} }) { next if !defined($snap->{vmstate}); my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1); next if !defined($drive); $remove_owned_drive->('vmstate', $drive); } PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive); if ($purge_unreferenced) { # also remove unreferenced disk my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images'); PVE::Storage::foreach_volid( $vmdisks, sub { my ($volid, $sid, $volname, $d) = @_; eval { PVE::Storage::vdisk_free($storecfg, $volid) }; warn $@ if $@; }, ); } eval { PVE::QemuServer::Network::delete_ifaces_ipams_ips($conf, $vmid) }; warn $@ if $@; if (defined $replacement_conf) { PVE::QemuConfig->write_config($vmid, $replacement_conf); } else { PVE::QemuConfig->destroy_config($vmid); } } my $fleecing_section_schema = { 'fleecing-images' => { type => 'string', format => 'pve-volume-id-list', description => "For internal use only. List of fleecing images allocated during backup." . " If no backup is running, these are left-overs that failed to be removed.", optional => 1, }, }; sub parse_vm_config { my ($filename, $raw, $strict) = @_; return if !defined($raw); # note that pending, snapshot and special sections are currently skipped when a backup is taken my $res = { digest => Digest::SHA::sha1_hex($raw), snapshots => {}, pending => undef, 'special-sections' => {}, }; my $handle_error = sub { my ($msg) = @_; if ($strict) { die $msg; } else { warn $msg; } }; $filename =~ m|/qemu-server/(\d+)\.conf$| || die "got strange filename '$filename'"; my $vmid = $1; my $conf = $res; my $descr; my $finish_description = sub { if (defined($descr)) { $descr =~ s/\s+$//; $conf->{description} = $descr; } $descr = undef; }; my $special_schemas = { cloudinit => $confdesc, # not actually used right now, see below fleecing => $fleecing_section_schema, }; my $special_sections_re_string = join('|', keys $special_schemas->%*); my $special_sections_re_1 = qr/($special_sections_re_string)/; my $section = { name => '', type => 'main', schema => $confdesc }; my @lines = split(/\n/, $raw); foreach my $line (@lines) { next if $line =~ m/^\s*$/; if ($line =~ m/^\[PENDING\]\s*$/i) { $section = { name => 'pending', type => 'pending', schema => $confdesc }; $finish_description->(); $handle_error->("vm $vmid - duplicate section: $section->{name}\n") if defined($res->{ $section->{name} }); $conf = $res->{ $section->{name} } = {}; next; } elsif ($line =~ m/^\[special:$special_sections_re_1\]\s*$/i) { $section = { name => $1, type => 'special', schema => $special_schemas->{$1} }; $finish_description->(); $handle_error->("vm $vmid - duplicate special section: $section->{name}\n") if defined($res->{'special-sections'}->{ $section->{name} }); $conf = $res->{'special-sections'}->{ $section->{name} } = {}; next; } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) { $section = { name => $1, type => 'snapshot', schema => $confdesc }; $finish_description->(); $handle_error->("vm $vmid - duplicate snapshot section: $section->{name}\n") if defined($res->{snapshots}->{ $section->{name} }); $conf = $res->{snapshots}->{ $section->{name} } = {}; next; } elsif ($line =~ m/^\[([^\]]*)\]\s*$/i) { my $unknown_section = $1; $section = undef; $finish_description->(); $handle_error->("vm $vmid - skipping unknown section: '$unknown_section'\n"); next; } next if !defined($section); if ($line =~ m/^\#(.*)$/) { $descr = '' if !defined($descr); $descr .= PVE::Tools::decode_text($1) . "\n"; next; } if ($line =~ m/^(description):\s*(.*\S)\s*$/) { $descr = '' if !defined($descr); $descr .= PVE::Tools::decode_text($2); } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) { $conf->{snapstate} = $1; } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) { my $key = $1; my $value = $2; $conf->{$key} = $value; } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) { my $value = $1; if ($section->{name} eq 'pending' && $section->{type} eq 'pending') { $conf->{delete} = $value; # we parse this later } else { $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n"); } } elsif ($line =~ m/^([a-z][a-z_\-]*\d*):\s*(.+?)\s*$/) { my $key = $1; my $value = $2; if ($section->{name} eq 'cloudinit' && $section->{type} eq 'special') { # ignore validation only used for informative purpose $conf->{$key} = $value; next; } eval { $value = check_type($key, $value, $section->{schema}); }; if ($@) { $handle_error->("vm $vmid - unable to parse value of '$key' - $@"); } else { $key = 'ide2' if $key eq 'cdrom'; my $fmt = $section->{schema}->{$key}->{format}; if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) { my $v = parse_drive($key, $value); if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) { $v->{file} = $volid; $value = print_drive($v); } else { $handle_error->("vm $vmid - unable to parse value of '$key'\n"); next; } } $conf->{$key} = $value; } } else { $handle_error->("vm $vmid - unable to parse config: $line\n"); } } $finish_description->(); delete $res->{snapstate}; # just to be sure $res->{pending} = {} if !defined($res->{pending}); return $res; } sub write_vm_config { my ($filename, $conf) = @_; delete $conf->{snapstate}; # just to be sure if ($conf->{cdrom}) { die "option ide2 conflicts with cdrom\n" if $conf->{ide2}; $conf->{ide2} = $conf->{cdrom}; delete $conf->{cdrom}; } # we do not use 'smp' any longer if ($conf->{sockets}) { delete $conf->{smp}; } elsif ($conf->{smp}) { $conf->{sockets} = $conf->{smp}; delete $conf->{cores}; delete $conf->{smp}; } my $used_volids = {}; my $cleanup_config = sub { my ($cref, $pending, $snapname) = @_; foreach my $key (keys %$cref) { next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' || $key eq 'snapstate' || $key eq 'pending' || $key eq 'special-sections'; my $value = $cref->{$key}; if ($key eq 'delete') { die "propertry 'delete' is only allowed in [PENDING]\n" if !$pending; # fixme: check syntax? next; } eval { $value = check_type($key, $value, $confdesc); }; die "unable to parse value of '$key' - $@" if $@; $cref->{$key} = $value; if (!$snapname && is_valid_drivename($key)) { my $drive = parse_drive($key, $value); $used_volids->{ $drive->{file} } = 1 if $drive && $drive->{file}; } } }; &$cleanup_config($conf); &$cleanup_config($conf->{pending}, 1); foreach my $snapname (keys %{ $conf->{snapshots} }) { die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending'; &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname); } # remove 'unusedX' settings if we re-add a volume foreach my $key (keys %$conf) { my $value = $conf->{$key}; if ($key =~ m/^unused/ && $used_volids->{$value}) { delete $conf->{$key}; } } my $generate_raw_config = sub { my ($conf, $pending) = @_; my $raw = ''; # add description as comment to top of file if (defined(my $descr = $conf->{description})) { if ($descr) { foreach my $cl (split(/\n/, $descr)) { $raw .= '#' . PVE::Tools::encode_text($cl) . "\n"; } } else { $raw .= "#\n" if $pending; } } foreach my $key (sort keys %$conf) { next if $key =~ /^(digest|description|pending|snapshots|special-sections)$/; $raw .= "$key: $conf->{$key}\n"; } return $raw; }; my $raw = &$generate_raw_config($conf); if (scalar(keys %{ $conf->{pending} })) { $raw .= "\n[PENDING]\n"; $raw .= &$generate_raw_config($conf->{pending}, 1); } for my $special (sort keys $conf->{'special-sections'}->%*) { next if $special eq 'cloudinit' && !PVE::QemuConfig->has_cloudinit($conf); $raw .= "\n[special:$special]\n"; $raw .= &$generate_raw_config($conf->{'special-sections'}->{$special}); } foreach my $snapname (sort keys %{ $conf->{snapshots} }) { $raw .= "\n[$snapname]\n"; $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname}); } return $raw; } sub get_default_property_value { my ($name) = @_; return $confdesc->{$name}->{default}; } sub load_defaults { my $res = {}; # we use static defaults from our JSON schema configuration foreach my $key (keys %$confdesc) { if (defined(my $default = $confdesc->{$key}->{default})) { $res->{$key} = $default; } } return $res; } sub config_list { my $vmlist = PVE::Cluster::get_vmlist(); my $res = {}; return $res if !$vmlist || !$vmlist->{ids}; my $ids = $vmlist->{ids}; my $nodename = nodename(); foreach my $vmid (keys %$ids) { my $d = $ids->{$vmid}; next if !$d->{node} || $d->{node} ne $nodename; next if !$d->{type} || $d->{type} ne 'qemu'; $res->{$vmid}->{exists} = 1; } return $res; } # check if used storages are available on all nodes (use by migrate) sub check_storage_availability { my ($storecfg, $conf, $node) = @_; PVE::QemuConfig->foreach_volume( $conf, sub { my ($ds, $drive) = @_; my $volid = $drive->{file}; return if !$volid; my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1); return if !$sid; # check if storage is available on both nodes my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid); PVE::Storage::storage_check_enabled($storecfg, $sid, $node); my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid); die "$volid: content type '$vtype' is not available on storage '$sid'\n" if !$scfg->{content}->{$vtype}; }, ); } # list nodes where all VM images are available (used by has_feature API) sub shared_nodes { my ($conf, $storecfg) = @_; my $nodelist = PVE::Cluster::get_nodelist(); my $nodehash = { map { $_ => 1 } @$nodelist }; my $nodename = nodename(); PVE::QemuConfig->foreach_volume( $conf, sub { my ($ds, $drive) = @_; my $volid = $drive->{file}; return if !$volid; my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); if ($storeid) { my $scfg = PVE::Storage::storage_config($storecfg, $storeid); if ($scfg->{disable}) { $nodehash = {}; } elsif (my $avail = $scfg->{nodes}) { foreach my $node (keys %$nodehash) { delete $nodehash->{$node} if !$avail->{$node}; } } elsif (!$scfg->{shared}) { foreach my $node (keys %$nodehash) { delete $nodehash->{$node} if $node ne $nodename; } } } }, ); return $nodehash; } sub check_local_storage_availability { my ($conf, $storecfg) = @_; my $nodelist = PVE::Cluster::get_nodelist(); my $nodehash = { map { $_ => {} } @$nodelist }; PVE::QemuConfig->foreach_volume( $conf, sub { my ($ds, $drive) = @_; my $volid = $drive->{file}; return if !$volid; my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); if ($storeid) { my $scfg = PVE::Storage::storage_config($storecfg, $storeid); if ($scfg->{disable}) { foreach my $node (keys %$nodehash) { $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1; } } elsif (my $avail = $scfg->{nodes}) { foreach my $node (keys %$nodehash) { if (!$avail->{$node}) { $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1; } } } } }, ); foreach my $node (values %$nodehash) { if (my $unavail = $node->{unavailable_storages}) { $node->{unavailable_storages} = [sort keys %$unavail]; } } return $nodehash; } # Compat only, use assert_config_exists_on_node and vm_running_locally where possible sub check_running { my ($vmid, $nocheck, $node) = @_; # $nocheck is set when called during a migration, in which case the config # file might still or already reside on the *other* node # - because rename has already happened, and current node is source # - because rename hasn't happened yet, and current node is target # - because rename has happened, current node is target, but hasn't yet # processed it yet PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck; return PVE::QemuServer::Helpers::vm_running_locally($vmid); } sub vzlist { my $vzlist = config_list(); my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist; while (defined(my $de = $fd->read)) { next if $de !~ m/^(\d+)\.pid$/; my $vmid = $1; next if !defined($vzlist->{$vmid}); if (my $pid = check_running($vmid)) { $vzlist->{$vmid}->{pid} = $pid; } } return $vzlist; } our $vmstatus_return_properties = { vmid => get_standard_option('pve-vmid'), status => { description => "QEMU process status.", type => 'string', enum => ['stopped', 'running'], }, mem => { description => "Currently used memory in bytes. Does not take into account kernel" . " same-page merging (KSM). Uses information from ballooning when available.", type => 'integer', optional => 1, renderer => 'bytes', }, maxmem => { description => "Maximum memory in bytes.", type => 'integer', optional => 1, renderer => 'bytes', }, memhost => { description => "Current memory usage on the host. Does not take into account kernel" . " same-page merging (KSM).", type => 'integer', optional => 1, renderer => 'bytes', }, maxdisk => { description => "Root disk size in bytes.", type => 'integer', optional => 1, renderer => 'bytes', }, diskread => { description => "The amount of bytes the guest read from it's block devices since the guest" . " was started. (Note: This info is not available for all storage types.)", type => 'integer', optional => 1, renderer => 'bytes', }, diskwrite => { description => "The amount of bytes the guest wrote from it's block devices since the guest" . " was started. (Note: This info is not available for all storage types.)", type => 'integer', optional => 1, renderer => 'bytes', }, name => { description => "VM (host)name.", type => 'string', optional => 1, }, netin => { description => "The amount of traffic in bytes that was sent to the guest over the network" . " since it was started.", type => 'integer', optional => 1, renderer => 'bytes', }, netout => { description => "The amount of traffic in bytes that was sent from the guest over the network" . " since it was started.", type => 'integer', optional => 1, renderer => 'bytes', }, qmpstatus => { description => "VM run state from the 'query-status' QMP monitor command.", type => 'string', optional => 1, }, pid => { description => "PID of the QEMU process, if the VM is running.", type => 'integer', optional => 1, }, uptime => { description => "Uptime in seconds.", type => 'integer', optional => 1, renderer => 'duration', }, cpu => { description => "Current CPU usage.", type => 'number', optional => 1, }, cpus => { description => "Maximum usable CPUs.", type => 'number', optional => 1, }, lock => { description => "The current config lock, if any.", type => 'string', optional => 1, }, tags => { description => "The current configured tags, if any", type => 'string', optional => 1, }, 'running-machine' => { description => "The currently running machine type (if running).", type => 'string', optional => 1, }, 'running-qemu' => { description => "The QEMU version the VM is currently using (if running).", type => 'string', optional => 1, }, template => { description => "Determines if the guest is a template.", type => 'boolean', optional => 1, default => 0, }, serial => { description => "Guest has serial device configured.", type => 'boolean', optional => 1, }, pressurecpusome => { description => "CPU Some pressure stall average over the last 10 seconds.", type => 'number', optional => 1, }, pressurecpufull => { description => "CPU Full pressure stall average over the last 10 seconds.", type => 'number', optional => 1, }, pressureiosome => { description => "IO Some pressure stall average over the last 10 seconds.", type => 'number', optional => 1, }, pressureiofull => { description => "IO Full pressure stall average over the last 10 seconds.", type => 'number', optional => 1, }, pressurememorysome => { description => "Memory Some pressure stall average over the last 10 seconds.", type => 'number', optional => 1, }, pressurememoryfull => { description => "Memory Full pressure stall average over the last 10 seconds.", type => 'number', optional => 1, }, }; my $last_proc_pid_stat; # get VM status information # This must be fast and should not block ($full == false) # We only query KVM using QMP if $full == true (this can be slow) sub vmstatus { my ($opt_vmid, $full) = @_; my $res = {}; my $storecfg = PVE::Storage::config(); my $list = vzlist(); my $defaults = load_defaults(); my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1); my $cpucount = $cpuinfo->{cpus} || 1; foreach my $vmid (keys %$list) { next if $opt_vmid && ($vmid ne $opt_vmid); my $conf = PVE::QemuConfig->load_config($vmid); my $d = { vmid => int($vmid) }; $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid}; # fixme: better status? $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped'; my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf); if (defined($size)) { $d->{disk} = 0; # no info available $d->{maxdisk} = $size; } else { $d->{disk} = 0; $d->{maxdisk} = 0; } $d->{cpus} = ($conf->{sockets} || $defaults->{sockets}) * ($conf->{cores} || $defaults->{cores}); $d->{cpus} = $cpucount if $d->{cpus} > $cpucount; $d->{cpus} = $conf->{vcpus} if $conf->{vcpus}; $d->{name} = $conf->{name} || "VM $vmid"; $d->{maxmem} = get_current_memory($conf->{memory}) * (1024 * 1024); if ($conf->{balloon}) { $d->{balloon_min} = $conf->{balloon} * (1024 * 1024); $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : $defaults->{shares}; } $d->{uptime} = 0; $d->{cpu} = 0; $d->{mem} = 0; $d->{memhost} = 0; $d->{netout} = 0; $d->{netin} = 0; $d->{template} = 1 if PVE::QemuConfig->is_template($conf); $d->{serial} = 1 if conf_has_serial($conf); $d->{lock} = $conf->{lock} if $conf->{lock}; $d->{tags} = $conf->{tags} if defined($conf->{tags}); $res->{$vmid} = $d; } my $netdev = PVE::ProcFSTools::read_proc_net_dev(); foreach my $dev (keys %$netdev) { next if $dev !~ m/^tap([1-9]\d*)i/; my $vmid = $1; my $d = $res->{$vmid}; next if !$d; $d->{netout} += $netdev->{$dev}->{receive}; $d->{netin} += $netdev->{$dev}->{transmit}; if ($full) { $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive}); $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit}); } } my $ctime = gettimeofday; foreach my $vmid (keys %$list) { my $d = $res->{$vmid}; my $pid = $d->{pid}; next if !$pid; my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid); next if !$pstat; # not running my $used = $pstat->{utime} + $pstat->{stime}; $d->{uptime} = int(($uptime - $pstat->{starttime}) / $cpuinfo->{user_hz}); my $cgroup = PVE::QemuServer::CGroup->new($vmid); my $cgroup_mem = eval { $cgroup->get_memory_stat() } // {}; warn "unable to get memory stat for $vmid - $@" if $@; $d->{memhost} = $cgroup_mem->{mem} // 0; $d->{mem} = $d->{memhost}; # default to cgroup, balloon info can override this below my $pressures = PVE::ProcFSTools::read_cgroup_pressure("qemu.slice/${vmid}.scope"); $d->{pressurecpusome} = $pressures->{cpu}->{some}->{avg10} * 1; $d->{pressurecpufull} = $pressures->{cpu}->{full}->{avg10} * 1; $d->{pressureiosome} = $pressures->{io}->{some}->{avg10} * 1; $d->{pressureiofull} = $pressures->{io}->{full}->{avg10} * 1; $d->{pressurememorysome} = $pressures->{memory}->{some}->{avg10} * 1; $d->{pressurememoryfull} = $pressures->{memory}->{full}->{avg10} * 1; my $old = $last_proc_pid_stat->{$pid}; if (!$old) { $last_proc_pid_stat->{$pid} = { time => $ctime, used => $used, cpu => 0, }; next; } my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz}; if ($dtime > 1000) { my $dutime = $used - $old->{used}; $d->{cpu} = (($dutime / $dtime) * $cpucount) / $d->{cpus}; $last_proc_pid_stat->{$pid} = { time => $ctime, used => $used, cpu => $d->{cpu}, }; } else { $d->{cpu} = $old->{cpu}; } } return $res if !$full; my $qmpclient = PVE::QMPClient->new(); my $ballooncb = sub { my ($vmid, $resp) = @_; my $info = $resp->{'return'}; return if !$info->{max_mem}; my $d = $res->{$vmid}; # use memory assigned to VM $d->{maxmem} = $info->{max_mem}; $d->{balloon} = $info->{actual}; if (defined($info->{total_mem}) && defined($info->{free_mem})) { $d->{mem} = $info->{total_mem} - $info->{free_mem}; $d->{freemem} = $info->{free_mem}; } $d->{ballooninfo} = $info; }; my $blockstatscb = sub { my ($vmid, $resp) = @_; my $data = $resp->{'return'} || []; my $totalrdbytes = 0; my $totalwrbytes = 0; for my $blockstat (@$data) { $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes}; $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes}; # With the switch to -blockdev, the block backend in QEMU has no name, so need to also # consider the qdev ID. Note that empty CD-ROM drives do not have a node name, so that # cannot be used as a fallback instead of the qdev ID. my $drive_id; if ($blockstat->{device}) { $drive_id = $blockstat->{device} =~ s/drive-//r; } elsif ($blockstat->{qdev}) { $drive_id = PVE::QemuServer::Blockdev::qdev_id_to_drive_id($blockstat->{qdev}); } else { print "blockstats callback: unexpected missing drive ID\n"; next; } $res->{$vmid}->{blockstat}->{$drive_id} = $blockstat->{stats}; } $res->{$vmid}->{diskread} = $totalrdbytes; $res->{$vmid}->{diskwrite} = $totalwrbytes; }; my $machinecb = sub { my ($vmid, $resp) = @_; my $data = $resp->{'return'} || []; $res->{$vmid}->{'running-machine'} = PVE::QemuServer::Machine::current_from_query_machines($data); }; my $versioncb = sub { my ($vmid, $resp) = @_; my $data = $resp->{'return'} // {}; my $version = 'unknown'; if (my $v = $data->{qemu}) { $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro}; } $res->{$vmid}->{'running-qemu'} = $version; }; my $proxmox_support_cb = sub { my ($vmid, $resp) = @_; $res->{$vmid}->{'proxmox-support'} = $resp->{'return'} // {}; }; my $statuscb = sub { my ($vmid, $resp) = @_; my $qmp_peer = vm_qmp_peer($vmid); $qmpclient->queue_cmd($qmp_peer, $proxmox_support_cb, 'query-proxmox-support'); $qmpclient->queue_cmd($qmp_peer, $blockstatscb, 'query-blockstats'); $qmpclient->queue_cmd($qmp_peer, $machinecb, 'query-machines'); $qmpclient->queue_cmd($qmp_peer, $versioncb, 'query-version'); # this fails if balloon driver is not loaded, so this must be # the last command (following command are aborted if this fails). $qmpclient->queue_cmd($qmp_peer, $ballooncb, 'query-balloon'); my $status = 'unknown'; if (!defined($status = $resp->{'return'}->{status})) { warn "unable to get VM status\n"; return; } $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status}; }; foreach my $vmid (keys %$list) { next if $opt_vmid && ($vmid ne $opt_vmid); next if !$res->{$vmid}->{pid}; # not running $qmpclient->queue_cmd(vm_qmp_peer($vmid), $statuscb, 'query-status'); } $qmpclient->queue_execute(undef, 2); foreach my $vmid (keys %$list) { next if $opt_vmid && ($vmid ne $opt_vmid); $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus}; } return $res; } sub conf_has_serial { my ($conf) = @_; for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) { if ($conf->{"serial$i"}) { return 1; } } return 0; } sub conf_has_audio { my ($conf, $id) = @_; $id //= 0; my $audio = $conf->{"audio$id"}; return if !defined($audio); my $audioproperties = parse_property_string($audio_fmt, $audio); my $audiodriver = $audioproperties->{driver} // 'spice'; return { dev => $audioproperties->{device}, dev_id => "audiodev$id", backend => $audiodriver, backend_id => "$audiodriver-backend${id}", }; } sub audio_devs { my ($audio, $audiopciaddr, $machine_version) = @_; my $devs = []; my $id = $audio->{dev_id}; my $audiodev = ""; if (min_version($machine_version, 4, 2)) { $audiodev = ",audiodev=$audio->{backend_id}"; } if ($audio->{dev} eq 'AC97') { push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev"; } elsif ($audio->{dev} =~ /intel\-hda$/) { push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}"; push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev"; push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev"; } else { die "unknown audio device '$audio->{dev}', implement me!"; } push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}"; return $devs; } sub get_tpm_paths { my ($vmid) = @_; return { socket => "/var/run/qemu-server/$vmid.swtpm", pid => "/var/run/qemu-server/$vmid.swtpm.pid", }; } sub add_tpm_device { my ($vmid, $devices, $conf) = @_; return if !$conf->{tpmstate0}; my $paths = get_tpm_paths($vmid); push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}"; push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar"; push @$devices, "-device", "tpm-tis,tpmdev=tpmdev"; } sub start_swtpm { my ($storecfg, $vmid, $tpmdrive, $migration) = @_; return if !$tpmdrive; my $state; my $tpm = parse_drive("tpmstate0", $tpmdrive); my ($storeid) = PVE::Storage::parse_volume_id($tpm->{file}, 1); if ($storeid) { if (PVE::QemuServer::Drive::drive_uses_qsd_fuse($storecfg, $tpm)) { PVE::QemuServer::QSD::start($vmid); $state = PVE::QemuServer::QSD::add_fuse_export($vmid, $tpm, 'tpmstate0'); } else { $state = PVE::Storage::map_volume($storecfg, $tpm->{file}); } } else { $state = $tpm->{file}; } my $paths = get_tpm_paths($vmid); # during migration, we will get state from remote # if (!$migration) { # run swtpm_setup to create a new TPM state if it doesn't exist yet my $setup_cmd = [ "swtpm_setup", "--tpmstate", "file://$state", "--createek", "--create-ek-cert", "--create-platform-cert", "--lock-nvram", "--config", "/etc/swtpm_setup.conf", # do not use XDG configs "--runas", "0", # force creation as root, error if not possible "--not-overwrite", # ignore existing state, do not modify ]; push @$setup_cmd, "--tpm2" if $tpm->{version} && $tpm->{version} eq 'v2.0'; # TPM 2.0 supports ECC crypto, use if possible push @$setup_cmd, "--ecc" if $tpm->{version} && $tpm->{version} eq 'v2.0'; run_command( $setup_cmd, outfunc => sub { print "swtpm_setup: $1\n"; }, ); } # Used to distinguish different invocations in the log. my $log_prefix = "[id=" . int(time()) . "] "; my $emulator_cmd = [ "swtpm", "socket", "--tpmstate", "backend-uri=file://$state,mode=0600", "--ctrl", "type=unixio,path=$paths->{socket},mode=0600", "--pid", "file=$paths->{pid}", "--terminate", # terminate on QEMU disconnect "--daemon", "--log", "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix", ]; push @$emulator_cmd, "--tpm2" if $tpm->{version} && $tpm->{version} eq 'v2.0'; run_command($emulator_cmd, outfunc => sub { print $1; }); my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid while (!-e $paths->{pid}) { die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0; usleep(50_000); } # return untainted PID of swtpm daemon so it can be killed on error file_read_firstline($paths->{pid}) =~ m/(\d+)/; return $1; } sub vga_conf_has_spice { my ($vga) = @_; my $vgaconf = parse_vga($vga); my $vgatype = $vgaconf->{type}; return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/; return $1 || 1; } # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags # to use in a QEMU command line (-cpu element), first array_intersect the result # of query_supported_ with query_understood_. This is necessary because: # # a) query_understood_ returns flags the host cannot use and # b) query_supported_ (rather the QMP call) doesn't actually return CPU # flags, but CPU settings - with most of them being flags. Those settings # (and some flags, curiously) cannot be specified as a "-cpu" argument. # # query_supported_ needs to start up to 2 temporary VMs and is therefore rather # expensive. If you need the value returned from this, you can get it much # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with # $accel being 'kvm' or 'tcg'. # # pvestatd calls this function on startup and whenever the QEMU/KVM version # changes, automatically populating pmxcfs. # # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] } # since kvm and tcg machines support different flags # sub query_supported_cpu_flags { my ($arch) = @_; my $host_arch = get_host_arch(); $arch //= $host_arch; my $default_machine = PVE::QemuServer::Machine::default_machine_for_arch($arch); my $flags = {}; my $kvm_supported = defined(kvm_version()) && $arch eq $host_arch; my $qemu_cmd = PVE::QemuServer::Helpers::get_command_for_arch($arch); my $fakevmid = -1; my $pidfile = PVE::QemuServer::Helpers::vm_pidfile_name($fakevmid); # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command my $query_supported_run_qemu = sub { my ($kvm) = @_; my $flags = {}; my $cmd = [ $qemu_cmd, '-machine', $default_machine, '-display', 'none', '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off", '-mon', 'chardev=qmp,mode=control', '-pidfile', $pidfile, '-S', '-daemonize', ]; if (!$kvm) { push @$cmd, '-accel', 'tcg'; } else { push @$cmd, '-cpu', 'host'; } my $rc = run_command($cmd, noerr => 1, quiet => 0); die "QEMU flag querying VM exited with code " . $rc if $rc; eval { my $cmd_result = mon_cmd( $fakevmid, 'query-cpu-model-expansion', type => 'full', model => { name => $kvm ? 'host' : 'max' }, ); my $props = $cmd_result->{model}->{props}; foreach my $prop (keys %$props) { next if $props->{$prop} ne '1'; # QEMU returns some flags multiple times, with '_', '.' or '-' # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...). # We only keep those with underscores, to match /proc/cpuinfo $prop =~ s/\.|-/_/g; $flags->{$prop} = 1; } }; my $err = $@; # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1); die $err if $err; return [sort keys %$flags]; }; # We need to query QEMU twice, since KVM and TCG have different supported flags PVE::QemuConfig->lock_config( $fakevmid, sub { $flags->{tcg} = eval { $query_supported_run_qemu->(0) }; warn "warning: failed querying supported tcg flags: $@\n" if $@; if ($kvm_supported) { $flags->{kvm} = eval { $query_supported_run_qemu->(1) }; warn "warning: failed querying supported kvm flags: $@\n" if $@; } }, ); return $flags; } # Understood CPU flags are written to a file at 'pve-qemu' compile time my $understood_cpu_flag_dir = "/usr/share/kvm"; sub query_understood_cpu_flags { my $arch = get_host_arch(); my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch"; die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n" if !-e $filepath; my $raw = file_get_contents($filepath); $raw =~ s/^\s+|\s+$//g; my @flags = split(/\s+/, $raw); return \@flags; } # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default # anymore. But smm=off seems to be required when using SeaBIOS and serial display. my sub should_disable_smm { my ($conf, $vga, $machine) = @_; return if $machine =~ m/^virt/; # there is no smm flag that could be disabled return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') && $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/; } my sub get_vga_properties { my ($conf, $arch, $machine_version, $winversion) = @_; my $vga = parse_vga($conf->{vga}); my $qxlnum = vga_conf_has_spice($conf->{vga}); $vga->{type} = 'qxl' if $qxlnum; if (!$vga->{type}) { if ($arch eq 'aarch64') { $vga->{type} = 'virtio'; } elsif (min_version($machine_version, 2, 9)) { $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus'; } else { $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus'; } } return ($vga, $qxlnum); } sub config_to_command { my ($storecfg, $vmid, $conf, $defaults, $options) = @_; my ($forcemachine, $forcecpu, $live_restore_backing, $dry_run) = $options->@{qw(force-machine force-cpu live-restore-backing dry-run)}; my $is_template = PVE::QemuConfig->is_template($conf); # minimize config for templates, they can only start for backup, # so most options besides the disks are irrelevant if ($is_template) { my $newconf = { template => 1, # in case below code checks that kvm => 0, # to prevent an error on hosts without virtualization extensions vga => 'none', # to not start a vnc server scsihw => $conf->{scsihw}, # so that the scsi disks are correctly added bios => $conf->{bios}, # so efidisk gets included if it exists name => $conf->{name}, # so it's correct in the process list }; # copy all disks over for my $device (PVE::QemuServer::Drive::valid_drive_names()) { $newconf->{$device} = $conf->{$device}; } # remaining configs stay default # mark config to prevent writing it out PVE::QemuConfig::NoWrite->mark_config($newconf); $conf = $newconf; } my ($machineFlags, $rtcFlags) = ([], []); my $devices = []; my $bridges = {}; my $ostype = $conf->{ostype}; my $winversion = windows_version($ostype); my $kvm = $conf->{kvm}; my $nodename = nodename(); my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine}); my $arch = PVE::QemuServer::Helpers::get_vm_arch($conf); my $kvm_binary = PVE::QemuServer::Helpers::get_command_for_arch($arch); my $kvmver = kvm_user_version($kvm_binary); if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 6) { $kvmver //= "undefined"; die "Detected old QEMU binary ('$kvmver', at least 6.0 is required)\n"; } my $machine_type = PVE::QemuServer::Machine::get_vm_machine($conf, $forcemachine); my $machine_version = extract_version($machine_type, $kvmver); $kvm //= 1 if is_native_arch($arch); $machine_version =~ m/(\d+)\.(\d+)/; my ($machine_major, $machine_minor) = ($1, $2); if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) { warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n"; } elsif (!min_version($kvmver, $machine_major, $machine_minor)) { die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type'," . " please upgrade node '$nodename'\n"; } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) { my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version); die "Installed qemu-server (max feature level for $machine_major.$machine_minor is" . " pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade" . " node '$nodename'\n"; } # if a specific +pve version is required for a feature, use $version_guard # instead of min_version to allow machines to be run with the minimum # required version my $required_pve_version = 0; my $version_guard = sub { my ($major, $minor, $pve) = @_; return 0 if !min_version($machine_version, $major, $minor, $pve); my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor"); return 1 if min_version($machine_version, $major, $minor, $max_pve + 1); $required_pve_version = $pve if $pve && $pve > $required_pve_version; return 1; }; if ($kvm && !defined kvm_version()) { die "KVM virtualisation configured, but not available. Either disable in VM configuration" . " or enable in BIOS.\n"; } my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf); my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1'); my $use_old_bios_files = undef; ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type); my $cmd = []; if ($conf->{affinity}) { push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity}; } push @$cmd, $kvm_binary; push @$cmd, '-id', $vmid; my $vmname = $conf->{name} || "vm$vmid"; push @$cmd, '-name', "$vmname,debug-threads=on"; push @$cmd, '-no-shutdown'; my $use_virtio = 0; my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket(vm_qmp_peer($vmid)); push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off"; push @$cmd, '-mon', "chardev=qmp,mode=control"; if (min_version($machine_version, 2, 12)) { # QEMU 9.2 introduced a new 'reconnect-ms' option while deprecating the 'reconnect' option my $reconnect_param = "reconnect=5"; if (min_version($kvmver, 9, 2)) { # this depends on the binary version $reconnect_param = "reconnect-ms=5000"; } push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,$reconnect_param"; push @$cmd, '-mon', "chardev=qmp-event,mode=control"; } push @$cmd, '-pidfile', PVE::QemuServer::Helpers::vm_pidfile_name($vmid); push @$cmd, '-daemonize'; if ($conf->{smbios1}) { my $smbios_conf = parse_smbios1($conf->{smbios1}); if ($smbios_conf->{base64}) { # Do not pass base64 flag to qemu delete $smbios_conf->{base64}; my $smbios_string = ""; foreach my $key (keys %$smbios_conf) { my $value; if ($key eq "uuid") { $value = $smbios_conf->{uuid}; } else { $value = decode_base64($smbios_conf->{$key}); } # qemu accepts any binary data, only commas need escaping by double comma $value =~ s/,/,,/g; $smbios_string .= "," . $key . "=" . $value if $value; } push @$cmd, '-smbios', "type=1" . $smbios_string; } else { push @$cmd, '-smbios', "type=1,$conf->{smbios1}"; } } if ($conf->{bios} && $conf->{bios} eq 'ovmf') { die "OVMF (UEFI) BIOS is not supported on 32-bit CPU types\n" if !$forcecpu && get_cpu_bitness($conf->{cpu}, $arch) == 32; my $hw_info = { 'cvm-type' => get_cvm_type($conf), arch => $arch, 'machine-version' => $machine_version, q35 => $q35, }; my ($ovmf_cmd, $ovmf_machine_flags) = PVE::QemuServer::OVMF::print_ovmf_commandline( $conf, $storecfg, $vmid, $hw_info, $version_guard, $is_template, ); push $cmd->@*, $ovmf_cmd->@*; push $machineFlags->@*, $ovmf_machine_flags->@*; } if ($q35) { # tell QEMU to load q35 config early # we use different pcie-port hardware for qemu >= 4.0 for passthrough if (min_version($machine_version, 4, 0)) { push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg'; } else { push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg'; } } if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) { push @$cmd, $fixups->@*; } if ($conf->{vmgenid}) { push @$devices, '-device', 'vmgenid,guid=' . $conf->{vmgenid}; } # add usb controllers my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_version); push @$devices, @usbcontrollers if @usbcontrollers; my ($vga, $qxlnum) = get_vga_properties($conf, $arch, $machine_version, $winversion); # enable absolute mouse coordinates (needed by vnc) my $tablet = $conf->{tablet}; if (!defined($tablet)) { $tablet = $defaults->{tablet}; $tablet = 0 if $qxlnum; # disable for spice because it is not needed $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card) } if ($tablet) { push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet; my $kbd = print_keyboarddevice_full($conf, $arch); push @$devices, '-device', $kbd if defined($kbd); } my $bootorder = device_bootorder($conf); # host pci device passthrough my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices( $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $bootorder, $dry_run, ); # usb devices my $usb_dev_features = {}; $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0); my @usbdevices = PVE::QemuServer::USB::get_usb_devices( $conf, $usb_dev_features, $bootorder, $machine_version, ); push @$devices, @usbdevices if @usbdevices; # serial devices for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) { my $path = $conf->{"serial$i"} or next; if ($path eq 'socket') { my $socket = "/var/run/qemu-server/${vmid}.serial$i"; push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off"; # On aarch64, serial0 is the UART device. QEMU only allows # connecting UART devices via the '-serial' command line, as # the device has a fixed slot on the hardware... if ($arch eq 'aarch64' && $i == 0) { push @$devices, '-serial', "chardev:serial$i"; } else { push @$devices, '-device', "isa-serial,chardev=serial$i"; } } else { die "no such serial device\n" if !-c $path; push @$devices, '-chardev', "serial,id=serial$i,path=$path"; push @$devices, '-device', "isa-serial,chardev=serial$i"; } } # parallel devices for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) { if (my $path = $conf->{"parallel$i"}) { die "no such parallel device\n" if !-c $path; my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel'; push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path"; push @$devices, '-device', "isa-parallel,chardev=parallel$i"; } } if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) { my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch); my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version); push @$devices, @$audio_devs; } # Add a TPM only if the VM is not a template, # to support backing up template VMs even if the TPM disk is write-protected. add_tpm_device($vmid, $devices, $conf) if !$is_template; my $sockets = 1; $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused $sockets = $conf->{sockets} if $conf->{sockets}; my $cores = $conf->{cores} || 1; my $maxcpus = $sockets * $cores; my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus; my $allowed_vcpus = $cpuinfo->{cpus}; die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus); if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) { push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus"; for (my $i = 2; $i <= $vcpus; $i++) { my $cpustr = print_cpu_device($conf, $arch, $i); push @$cmd, '-device', $cpustr; } } else { push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus"; } push @$cmd, '-nodefaults'; push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg"; push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0; push @$cmd, '-no-reboot' if defined($conf->{reboot}) && $conf->{reboot} == 0; if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none') { push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, undef, $qxlnum, $bridges); push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid); push @$cmd, '-vnc', "unix:$socket,password=on"; } else { push @$cmd, '-vga', 'none' if $vga->{type} eq 'none'; push @$cmd, '-nographic'; } # For now, handles only specific parts, but the final goal is to cover everything. my $cfg2cmd_opts = { forcemachine => $forcemachine }; my $cfg2cmd = PVE::QemuServer::Cfg2Cmd->new($conf, $defaults, $version_guard, $cfg2cmd_opts); my $generated = $cfg2cmd->generate(); push $cmd->@*, '-global', $_ for ($generated->global_flags() // [])->@*; push $machineFlags->@*, ($generated->machine_flags() // [])->@*; push $rtcFlags->@*, ($generated->rtc_flags() // [])->@*; if ($forcecpu) { push @$cmd, '-cpu', $forcecpu; } else { push @$cmd, get_cpu_options( $conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough, ); } my $virtiofs_enabled = PVE::QemuServer::Virtiofs::virtiofs_enabled($conf); PVE::QemuServer::Memory::config( $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $virtiofs_enabled, $cmd, $machineFlags, ); push @$cmd, '-S' if $conf->{freeze}; push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard}); my $guest_agent = parse_guest_agent($conf); if ($guest_agent->{enabled}) { my $qgasocket = PVE::QemuServer::Helpers::qmp_socket( { name => "VM $vmid", id => $vmid, type => 'qga' }); push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0"; if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') { my $pciaddr = print_pci_addr("qga0", $bridges, $arch); push @$devices, '-device', "virtio-serial,id=qga0$pciaddr"; push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0'; } elsif ($guest_agent->{type} eq 'isa') { push @$devices, '-device', "isa-serial,chardev=qga0"; } } my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef; if ($rng && $version_guard->(4, 1, 2)) { my $rng_object = print_rng_object_commandline('rng0', $rng); my $rng_device = print_rng_device_commandline('rng0', $rng, $bridges, $arch); push @$devices, '-object', $rng_object; push @$devices, '-device', $rng_device; } my $spice_port; assert_clipboard_config($vga); my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/; if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) { if ($qxlnum > 1) { if ($winversion) { for (my $i = 1; $i < $qxlnum; $i++) { push @$devices, '-device', print_vga_device( $conf, $vga, $arch, $machine_version, $i, $qxlnum, $bridges, ); } } else { # assume other OS works like Linux my ($ram, $vram) = ("134217728", "67108864"); if ($vga->{memory}) { $ram = PVE::Tools::convert_size($qxlnum * 4 * $vga->{memory}, 'mb' => 'b'); $vram = PVE::Tools::convert_size($qxlnum * 2 * $vga->{memory}, 'mb' => 'b'); } push @$cmd, '-global', "qxl-vga.ram_size=$ram"; push @$cmd, '-global', "qxl-vga.vram_size=$vram"; } } my $pciaddr = print_pci_addr("spice", $bridges, $arch); push @$devices, '-device', "virtio-serial,id=spice$pciaddr"; if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') { push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on'; } else { push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent'; } push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0"; if ($is_spice) { my $pfamily = PVE::Tools::get_host_address_family($nodename); my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily); die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs; my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr}); $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost); my $spice_enhancement_str = $conf->{spice_enhancements} // ''; my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str); if ($spice_enhancement->{foldersharing}) { push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0"; push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0"; } my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on"; $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming}; push @$devices, '-spice', "$spice_opts"; } } # enable balloon by default, unless explicitly disabled if (!defined($conf->{balloon}) || $conf->{balloon}) { my $pciaddr = print_pci_addr("balloon0", $bridges, $arch); my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr"; $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2); push @$devices, '-device', $ballooncmd; } if ($conf->{watchdog}) { my $wdopts = parse_watchdog($conf->{watchdog}); my $pciaddr = print_pci_addr("watchdog", $bridges, $arch); my $watchdog = $wdopts->{model} || 'i6300esb'; push @$devices, '-device', "$watchdog$pciaddr"; push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action}; } my $scsicontroller = {}; my $ahcicontroller = {}; my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw}; # Add iscsi initiator name if available if (my $initiator = get_iscsi_initiator_name()) { push @$devices, '-iscsi', "initiator-name=$initiator"; } PVE::QemuConfig->foreach_volume( $conf, sub { my ($ds, $drive) = @_; # ignore efidisk here, already added in bios/fw handling code above return if $drive->{interface} eq 'efidisk'; # similar for TPM return if $drive->{interface} eq 'tpmstate'; $use_virtio = 1 if $ds =~ m/^virtio/; $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds}; if ($drive->{interface} eq 'virtio') { push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread}; } if ($drive->{interface} eq 'scsi') { my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf->{scsihw}, $drive->{index}); die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n" if $drive->{index} > 13 && !&$version_guard(4, 1, 2); my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch); my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw; my $iothread = ''; if ( $conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread} ) { $iothread .= ",iothread=iothread-$controller_prefix$controller"; push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller"; } elsif ($drive->{iothread}) { log_warn( "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n" ); } my $queues = ''; if ( $conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues} ) { $queues = ",num_queues=$drive->{queues}"; } push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller}; $scsicontroller->{$controller} = 1; } if ($drive->{interface} eq 'sata') { my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS); my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch); push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller}; $ahcicontroller->{$controller} = 1; } my $live_restore = $live_restore_backing->{$ds}; if (min_version($machine_version, 10, 0)) { # for the switch to -blockdev if ($drive->{file} ne 'none') { my $throttle_group = PVE::QemuServer::Blockdev::generate_throttle_group($drive); push @$cmd, '-object', to_json($throttle_group, { canonical => 1 }); my $extra_blockdev_options = {}; $extra_blockdev_options->{'live-restore'} = $live_restore if $live_restore; $extra_blockdev_options->{'read-only'} = 1 if $is_template; my $blockdev = PVE::QemuServer::Blockdev::generate_drive_blockdev( $storecfg, $drive, $machine_version, $extra_blockdev_options, ); push @$devices, '-blockdev', to_json($blockdev, { canonical => 1 }); } } else { my $live_blockdev_name = undef; if ($live_restore) { $live_blockdev_name = $live_restore->{name}; push @$devices, '-blockdev', $live_restore->{blockdev}; } my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive, $live_blockdev_name); if ($is_template) { # TODO PVE 10.x - since the temporary config for starting templates for backup # uses the latest machine version, this should already be dead code. It's kept # for now if for whatever reason an older QEMU build is used (e.g. bisecting). my $interface = $drive->{interface}; $drive_cmd .= ',readonly=on' if $interface ne 'ide' && $interface ne 'sata'; } push @$devices, '-drive', $drive_cmd; } push @$devices, '-device', print_drivedevice_full( $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type, ); }, ); my $nets_host_mtu = { map { split('=', $_) } PVE::Tools::split_list($options->{'nets-host-mtu'}) }; for (my $i = 0; $i < $MAX_NETS; $i++) { my $netname = "net$i"; next if !$conf->{$netname}; my $d = PVE::QemuServer::Network::parse_net($conf->{$netname}); next if !$d; # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later? $use_virtio = 1 if $d->{model} eq 'virtio'; $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname}; my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname); push @$devices, '-netdev', $netdevfull; # force +pve1 if machine version 10.0, for host_mtu differentiation $version_guard->(10, 0, 1); my $netdevicefull = print_netdevice_full( $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_version, $nets_host_mtu->{$netname}, ); push @$devices, '-device', $netdevicefull; } if ($conf->{ivshmem}) { my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem}); my $bus; if ($q35) { $bus = print_pcie_addr("ivshmem"); } else { $bus = print_pci_addr("ivshmem", $bridges, $arch); } my $ivshmem_name = $ivshmem->{name} // $vmid; my $path = '/dev/shm/pve-shm-' . $ivshmem_name; push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,"; push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path" . ",size=$ivshmem->{size}M"; } # pci.4 is nested in pci.1 $bridges->{1} = 1 if $bridges->{4}; if (!$q35) { # add pci bridges if (min_version($machine_version, 2, 3)) { $bridges->{1} = 1; $bridges->{2} = 1; } $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/; } for my $k (sort { $b cmp $a } keys %$bridges) { next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3 my $k_name = $k; if ($k == 2 && $legacy_igd) { $k_name = "$k-igd"; } my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch); my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr"; if ($q35) { # add after -readconfig pve-q35.cfg splice @$devices, 2, 0, '-device', $devstr; } else { unshift @$devices, '-device', $devstr if $k > 0; } } if (!$kvm) { push @$machineFlags, 'accel=tcg'; } my $power_state_flags = PVE::QemuServer::Machine::get_power_state_flags($machine_conf, $arch, $version_guard); push $cmd->@*, $power_state_flags->@* if defined($power_state_flags); push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type); my $machine_type_min = $machine_type; $machine_type_min =~ s/\+pve\d+$//; $machine_type_min .= "+pve$required_pve_version"; push @$machineFlags, "type=${machine_type_min}"; PVE::QemuServer::Machine::assert_valid_machine_property($machine_conf); if (my $viommu = $machine_conf->{viommu}) { my $viommu_devstr = ''; if ($machine_conf->{'aw-bits'}) { $viommu_devstr .= ",aw-bits=$machine_conf->{'aw-bits'}"; # TODO remove message once this gets properly checked/warned about in QEMU itself. print "vIOMMU 'aw-bits' set to $machine_conf->{'aw-bits'}. Sometimes it is necessary to" . " set the CPU's 'guest-phys-bits' to the same value.\n"; } if ($viommu eq 'intel') { $viommu_devstr = "intel-iommu,intremap=on,caching-mode=on$viommu_devstr"; unshift @$devices, '-device', $viommu_devstr; push @$machineFlags, 'kernel-irqchip=split'; } elsif ($viommu eq 'virtio') { $viommu_devstr = "virtio-iommu-pci$viommu_devstr"; push @$devices, '-device', $viommu_devstr; } } if ($conf->{'amd-sev'}) { push @$devices, '-object', get_amd_sev_object($conf->{'amd-sev'}, $conf->{bios}); push @$machineFlags, 'confidential-guest-support=sev0'; } elsif ($conf->{'intel-tdx'}) { my $tdx_object = get_intel_tdx_object($conf->{'intel-tdx'}, $conf->{bios}); push @$devices, '-object', to_json($tdx_object, { canonical => 1 }); push @$machineFlags, 'confidential-guest-support=tdx0'; push @$machineFlags, 'kernel_irqchip=split'; } push @$machineFlags, 'mem-merge=off' if defined($conf->{'allow-ksm'}) && !$conf->{'allow-ksm'}; PVE::QemuServer::Virtiofs::config($conf, $vmid, $devices); push @$cmd, @$devices; push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags); push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags); if (my $vmstate = $conf->{vmstate}) { my $statepath = PVE::Storage::path($storecfg, $vmstate); push @$cmd, '-loadstate', $statepath; print "activating and using '$vmstate' as vmstate\n"; } if ($is_template) { # TODO PVE 10.x - since the temporary config for starting templates for backup uses the # latest machine version, this should already be dead code. It's kept for now if for # whatever reason an older QEMU build is used (e.g. bisecting). # needed to workaround base volumes being read-only push @$cmd, '-snapshot'; } # add custom args if ($conf->{args}) { my $aa = PVE::Tools::split_args($conf->{args}); push @$cmd, @$aa; } return wantarray ? ($cmd, $spice_port, $pci_devices, $conf) : $cmd; } sub spice_port { my ($vmid) = @_; my $res = mon_cmd($vmid, 'query-spice'); return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n"; } sub vm_devices_list { my ($vmid) = @_; my $res = mon_cmd($vmid, 'query-pci'); my $devices_to_check = []; my $devices = {}; foreach my $pcibus (@$res) { push @$devices_to_check, @{ $pcibus->{devices} },; } while (@$devices_to_check) { my $to_check = []; for my $d (@$devices_to_check) { $devices->{ $d->{'qdev_id'} } = 1 if $d->{'qdev_id'}; next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices}; $devices->{ $d->{'qdev_id'} } += scalar(@{ $d->{'pci_bridge'}->{devices} }); push @$to_check, @{ $d->{'pci_bridge'}->{devices} }; } $devices_to_check = $to_check; } # Block device IDs need to be checked at the qdev level, since with '-blockdev', the 'device' # property will not be set. my $resblock = mon_cmd($vmid, 'query-block'); for my $block ($resblock->@*) { my $qdev_id = $block->{qdev} or next; my $drive_id = PVE::QemuServer::Blockdev::qdev_id_to_drive_id($qdev_id); $devices->{$drive_id} = 1; } my $resmice = mon_cmd($vmid, 'query-mice'); foreach my $mice (@$resmice) { if ($mice->{name} eq 'QEMU HID Tablet') { $devices->{tablet} = 1; last; } } # for usb devices there is no query-usb # but we can iterate over the entries in # qom-list path=/machine/peripheral my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral'); foreach my $per (@$resperipheral) { if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) { $devices->{ $per->{name} } = 1; } } return $devices; } sub vm_deviceplug { my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_; my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf); my $devices_list = vm_devices_list($vmid); return 1 if defined($devices_list->{$deviceid}); # add PCI bridge if we need it for the device qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); if ($deviceid eq 'tablet') { qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch)); } elsif ($deviceid eq 'keyboard') { qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch)); } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) { my $id = $1; qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id"); qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1)); } elsif ($deviceid =~ m/^usb(\d+)$/) { qemu_deviceadd( $vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1), ); } elsif ($deviceid =~ m/^(virtio)(\d+)$/) { qemu_iothread_add($vmid, $deviceid, $device); qemu_driveadd($storecfg, $vmid, $device); my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type); qemu_deviceadd($vmid, $devicefull); eval { qemu_deviceaddverify($vmid, $deviceid); }; if (my $err = $@) { eval { qemu_drivedel($vmid, $deviceid); }; warn $@ if $@; die $err; } } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) { my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi"; my $pciaddr = print_pci_addr($deviceid, undef, $arch); my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw; my $devicefull = "$scsihw_type,id=$deviceid$pciaddr"; if ($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) { qemu_iothread_add($vmid, $deviceid, $device); $devicefull .= ",iothread=iothread-$deviceid"; } if ($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) { $devicefull .= ",num_queues=$device->{queues}"; } qemu_deviceadd($vmid, $devicefull); qemu_deviceaddverify($vmid, $deviceid); } elsif ($deviceid =~ m/^(scsi)(\d+)$/) { qemu_findorcreatescsihw($storecfg, $conf, $vmid, $device, $arch, $machine_type); qemu_driveadd($storecfg, $vmid, $device); my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type); eval { qemu_deviceadd($vmid, $devicefull); }; if (my $err = $@) { eval { qemu_drivedel($vmid, $deviceid); }; warn $@ if $@; die $err; } } elsif ($deviceid =~ m/^(net)(\d+)$/) { return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid); my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf); my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type); my $use_old_bios_files = undef; ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type); my $netdevicefull = print_netdevice_full( $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_version, ); qemu_deviceadd($vmid, $netdevicefull); eval { qemu_deviceaddverify($vmid, $deviceid); qemu_set_link_status($vmid, $deviceid, !$device->{link_down}); }; if (my $err = $@) { eval { qemu_netdevdel($vmid, $deviceid); }; warn $@ if $@; die $err; } } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) { my $bridgeid = $2; my $pciaddr = print_pci_addr($deviceid, undef, $arch); my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr"; qemu_deviceadd($vmid, $devicefull); qemu_deviceaddverify($vmid, $deviceid); } else { die "can't hotplug device '$deviceid'\n"; } return 1; } # fixme: this should raise exceptions on error! sub vm_deviceunplug { my ($vmid, $conf, $deviceid) = @_; my $devices_list = vm_devices_list($vmid); return 1 if !defined($devices_list->{$deviceid}); my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf); die "can't unplug bootdisk '$deviceid'\n" if grep { $_ eq $deviceid } @$bootdisks; if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') { qemu_devicedel($vmid, $deviceid); } elsif ($deviceid =~ m/^usbredirdev\d+$/) { qemu_devicedel($vmid, $deviceid); qemu_devicedelverify($vmid, $deviceid); } elsif ($deviceid =~ m/^usb\d+$/) { qemu_devicedel($vmid, $deviceid); qemu_devicedelverify($vmid, $deviceid); } elsif ($deviceid =~ m/^(virtio)(\d+)$/) { my $device = parse_drive($deviceid, $conf->{$deviceid}); qemu_devicedel($vmid, $deviceid); qemu_devicedelverify($vmid, $deviceid); qemu_drivedel($vmid, $deviceid); qemu_iothread_del($vmid, $deviceid, $device); } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) { qemu_devicedel($vmid, $deviceid); qemu_devicedelverify($vmid, $deviceid, 15); } elsif ($deviceid =~ m/^(scsi)(\d+)$/) { my $device = parse_drive($deviceid, $conf->{$deviceid}); qemu_devicedel($vmid, $deviceid); qemu_devicedelverify($vmid, $deviceid); qemu_drivedel($vmid, $deviceid); qemu_deletescsihw($conf, $vmid, $deviceid); qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device) if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single'); } elsif ($deviceid =~ m/^(net)(\d+)$/) { qemu_devicedel($vmid, $deviceid); qemu_devicedelverify($vmid, $deviceid); qemu_netdevdel($vmid, $deviceid); } else { die "can't unplug device '$deviceid'\n"; } return 1; } sub qemu_spice_usbredir_chardev_add { my ($vmid, $id) = @_; mon_cmd( $vmid, "chardev-add", ( id => $id, backend => { type => 'spicevmc', data => { type => "usbredir", }, }, ), ); } sub qemu_iothread_add { my ($vmid, $deviceid, $device) = @_; if ($device->{iothread}) { my $iothreads = vm_iothreads_list($vmid); qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"}; } } sub qemu_iothread_del { my ($vmid, $deviceid, $device) = @_; if ($device->{iothread}) { my $iothreads = vm_iothreads_list($vmid); qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"}; } } sub qemu_driveadd { my ($storecfg, $vmid, $device) = @_; my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid); # for the switch to -blockdev if (PVE::QemuServer::Machine::is_machine_version_at_least($machine_type, 10, 0)) { PVE::QemuServer::Blockdev::attach($storecfg, $vmid, $device, {}); return 1; } else { my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef); $drive =~ s/\\/\\\\/g; my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"", 60); # If the command succeeds qemu prints: "OK" return 1 if $ret =~ m/OK/s; die "adding drive failed: $ret\n"; } } sub qemu_drivedel { my ($vmid, $deviceid) = @_; my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid); # for the switch to -blockdev if (PVE::QemuServer::Machine::is_machine_version_at_least($machine_type, 10, 0)) { PVE::QemuServer::Blockdev::detach(vm_qmp_peer($vmid), "drive-$deviceid"); return 1; } else { my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid", 10 * 60); $ret =~ s/^\s+//; return 1 if $ret eq ""; # NB: device not found errors mean the drive was auto-deleted and we ignore the error return 1 if $ret =~ m/Device \'.*?\' not found/s; die "deleting drive $deviceid failed : $ret\n"; } } sub qemu_deviceaddverify { my ($vmid, $deviceid) = @_; for (my $i = 0; $i <= 5; $i++) { my $devices_list = vm_devices_list($vmid); return 1 if defined($devices_list->{$deviceid}); sleep 1; } die "error on hotplug device '$deviceid'\n"; } sub qemu_devicedelverify { my ($vmid, $deviceid, $timeout) = @_; $timeout //= 5; # need to verify that the device is correctly removed as device_del # is async and empty return is not reliable for (my $i = 0; $i <= $timeout; $i++) { my $devices_list = vm_devices_list($vmid); return 1 if !defined($devices_list->{$deviceid}); sleep 1; } die "error on hot-unplugging device '$deviceid' - still busy in guest?\n"; } sub qemu_findorcreatescsihw { my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_; my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf->{scsihw}, $device->{index}); my $scsihwid = "$controller_prefix$controller"; my $devices_list = vm_devices_list($vmid); if (!defined($devices_list->{$scsihwid})) { vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type); } return 1; } sub qemu_deletescsihw { my ($conf, $vmid, $opt) = @_; my $device = parse_drive($opt, $conf->{$opt}); if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) { vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}"); return 1; } my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf->{scsihw}, $device->{index}); my $devices_list = vm_devices_list($vmid); foreach my $opt (keys %{$devices_list}) { if (is_valid_drivename($opt)) { my $drive = parse_drive($opt, $conf->{$opt}); if ( $drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev - 1) * ($controller + 1)) ) { return 1; } } } my $scsihwid = "scsihw$controller"; vm_deviceunplug($vmid, $conf, $scsihwid); return 1; } sub qemu_add_pci_bridge { my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_; my $bridges = {}; my $bridgeid; print_pci_addr($device, $bridges, $arch); while (my ($k, $v) = each %$bridges) { $bridgeid = $k; } return 1 if !defined($bridgeid) || $bridgeid < 1; my $bridge = "pci.$bridgeid"; my $devices_list = vm_devices_list($vmid); if (!defined($devices_list->{$bridge})) { vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type); } return 1; } sub qemu_set_link_status { my ($vmid, $device, $up) = @_; mon_cmd( $vmid, "set_link", name => $device, up => $up ? JSON::true : JSON::false, ); } sub qemu_netdevadd { my ($vmid, $conf, $arch, $device, $deviceid) = @_; my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1); my %options = split(/[=,]/, $netdev); if (defined(my $vhost = $options{vhost})) { $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost)); } if (defined(my $queues = $options{queues})) { $options{queues} = $queues + 0; } mon_cmd($vmid, "netdev_add", %options); return 1; } sub qemu_netdevdel { my ($vmid, $deviceid) = @_; mon_cmd($vmid, "netdev_del", id => $deviceid); } sub qemu_usb_hotplug { my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_; return if !$device; # remove the old one first vm_deviceunplug($vmid, $conf, $deviceid); # check if xhci controller is necessary and available my $added_xhci; my $devicelist = vm_devices_list($vmid); if (!$devicelist->{xhci}) { my $pciaddr = print_pci_addr("xhci", undef, $arch); qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr)); $added_xhci = 1; } # add the new one eval { vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type); }; if (my $err = $@) { if ($added_xhci) { eval { vm_deviceunplug($vmid, $conf, 'xhci'); }; warn "failed to unplug xhci controller - $@" if $@; } die $err; } return; } sub qemu_cpu_hotplug { my ($vmid, $conf, $vcpus) = @_; my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid); my $sockets = 1; $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused $sockets = $conf->{sockets} if $conf->{sockets}; my $cores = $conf->{cores} || 1; my $maxcpus = $sockets * $cores; $vcpus = $maxcpus if !$vcpus; die "you can't add more vcpus than maxcpus\n" if $vcpus > $maxcpus; my $currentvcpus = $conf->{vcpus} || $maxcpus; if ($vcpus < $currentvcpus) { if (PVE::QemuServer::Machine::is_machine_version_at_least($machine_type, 2, 7)) { for (my $i = $currentvcpus; $i > $vcpus; $i--) { qemu_devicedel($vmid, "cpu$i"); my $retry = 0; my $currentrunningvcpus = undef; while (1) { $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast"); last if scalar(@{$currentrunningvcpus}) == $i - 1; raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5; $retry++; sleep 1; } #update conf after each successful cpu unplug $conf->{vcpus} = scalar(@{$currentrunningvcpus}); PVE::QemuConfig->write_config($vmid, $conf); } } else { die "cpu hot-unplugging requires qemu version 2.7 or higher\n"; } return; } my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast"); die "vcpus in running vm does not match its configuration\n" if scalar(@{$currentrunningvcpus}) != $currentvcpus; if (PVE::QemuServer::Machine::is_machine_version_at_least($machine_type, 2, 7)) { my $arch = PVE::QemuServer::Helpers::get_vm_arch($conf); for (my $i = $currentvcpus + 1; $i <= $vcpus; $i++) { my $cpustr = print_cpu_device($conf, $arch, $i); qemu_deviceadd($vmid, $cpustr); my $retry = 0; my $currentrunningvcpus = undef; while (1) { $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast"); last if scalar(@{$currentrunningvcpus}) == $i; raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10; sleep 1; $retry++; } #update conf after each successful cpu hotplug $conf->{vcpus} = scalar(@{$currentrunningvcpus}); PVE::QemuConfig->write_config($vmid, $conf); } } else { for (my $i = $currentvcpus; $i < $vcpus; $i++) { mon_cmd($vmid, "cpu-add", id => int($i)); } } } sub qemu_volume_snapshot { my ($vmid, $deviceid, $storecfg, $drive, $snap) = @_; my $volid = $drive->{file}; my $running = check_running($vmid); my $do_snapshots_type = do_snapshots_type($storecfg, $volid, $deviceid, $running); if ($do_snapshots_type eq 'internal') { print "internal qemu snapshot\n"; my $qmp_peer = PVE::QemuServer::Drive::drive_qmp_peer($storecfg, $vmid, $drive); qmp_cmd($qmp_peer, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap); } elsif ($do_snapshots_type eq 'external') { my $machine_version = PVE::QemuServer::Machine::get_current_qemu_machine($vmid); if (!PVE::QemuServer::Machine::is_machine_version_at_least($machine_version, 10, 0)) { die "storage for '$volid' is configured for snapshots as a volume chain - this requires" . " QEMU machine version >= 10.0. See" . " https://pve.proxmox.com/wiki/QEMU_Machine_Version_Upgrade\n"; } my $storeid = (PVE::Storage::parse_volume_id($volid))[0]; my $scfg = PVE::Storage::storage_config($storecfg, $storeid); print "external qemu snapshot\n"; my $snapshots = PVE::Storage::volume_snapshot_info($storecfg, $volid); my $parent_snap = $snapshots->{'current'}->{parent}; my $qmp_peer = PVE::QemuServer::Drive::drive_qmp_peer($storecfg, $vmid, $drive); PVE::QemuServer::VolumeChain::blockdev_external_snapshot( $storecfg, $qmp_peer, $machine_version, $deviceid, $drive, $snap, $parent_snap, ); } elsif ($do_snapshots_type eq 'storage') { PVE::Storage::volume_snapshot($storecfg, $volid, $snap); } } sub qemu_volume_snapshot_delete { my ($vmid, $storecfg, $drive, $snap) = @_; my $volid = $drive->{file}; my $running = check_running($vmid); my $attached_deviceid; if ($running) { my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->foreach_volume( $conf, sub { my ($ds, $drive) = @_; $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid; }, ); } my $do_snapshots_type = do_snapshots_type($storecfg, $volid, $attached_deviceid, $running); if ($do_snapshots_type eq 'internal') { my $qmp_peer = PVE::QemuServer::Drive::drive_qmp_peer($storecfg, $vmid, $drive); qmp_cmd( $qmp_peer, 'blockdev-snapshot-delete-internal-sync', device => $attached_deviceid, name => $snap, ); } elsif ($do_snapshots_type eq 'external') { my $machine_version = PVE::QemuServer::Machine::get_current_qemu_machine($vmid); if (!PVE::QemuServer::Machine::is_machine_version_at_least($machine_version, 10, 0)) { die "storage for '$volid' is configured for snapshots as a volume chain - this requires" . " QEMU machine version >= 10.0. See" . " https://pve.proxmox.com/wiki/QEMU_Machine_Version_Upgrade\n"; } print "delete qemu external snapshot\n"; my $path = PVE::Storage::path($storecfg, $volid); my $snapshots = PVE::Storage::volume_snapshot_info($storecfg, $volid); die "could not find snapshot '$snap' for volume '$volid'\n" if !defined($snapshots->{$snap}); my $parentsnap = $snapshots->{$snap}->{parent}; my $childsnap = $snapshots->{$snap}->{child}; my $qmp_peer = PVE::QemuServer::Drive::drive_qmp_peer($storecfg, $vmid, $drive); # if we delete the first snasphot, we commit because the first snapshot original base image, it should be big. # improve-me: if firstsnap > child : commit, if firstsnap < child do a stream. if (!$parentsnap) { print "delete first snapshot $snap\n"; PVE::QemuServer::VolumeChain::blockdev_commit( $storecfg, $qmp_peer, $machine_version, $attached_deviceid, $drive, $childsnap, $snap, ); PVE::Storage::rename_snapshot($storecfg, $volid, $snap, $childsnap); PVE::QemuServer::VolumeChain::blockdev_replace( $storecfg, $qmp_peer, $machine_version, $attached_deviceid, $drive, $snap, $childsnap, $snapshots->{$childsnap}->{child}, ); } else { #intermediate snapshot, we always stream the snapshot to child snapshot print "stream intermediate snapshot $snap to $childsnap\n"; PVE::QemuServer::VolumeChain::blockdev_stream( $storecfg, $qmp_peer, $machine_version, $attached_deviceid, $drive, $snap, $parentsnap, $childsnap, ); } } elsif ($do_snapshots_type eq 'storage') { PVE::Storage::volume_snapshot_delete( $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef, ); } } sub foreach_volid { my ($conf, $func, @param) = @_; my $volhash = {}; my $test_volid = sub { my ($key, $drive, $snapname, $pending) = @_; my $volid = $drive->{file}; return if !$volid; $volhash->{$volid}->{cdrom} //= 1; $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive); $volhash->{$volid}->{is_cloudinit} //= 0; $volhash->{$volid}->{is_cloudinit} = 1 if drive_is_cloudinit($drive); my $replicate = $drive->{replicate} // 1; $volhash->{$volid}->{replicate} //= 0; $volhash->{$volid}->{replicate} = 1 if $replicate; $volhash->{$volid}->{shared} //= 0; $volhash->{$volid}->{shared} = 1 if $drive->{shared}; $volhash->{$volid}->{is_unused} //= 0; $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/; $volhash->{$volid}->{is_attached} //= 0; $volhash->{$volid}->{is_attached} = 1 if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending; $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1 if defined($snapname); $volhash->{$volid}->{referenced_in_pending} = 1 if $pending; my $size = $drive->{size}; $volhash->{$volid}->{size} //= $size if $size; $volhash->{$volid}->{is_vmstate} //= 0; $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate'; $volhash->{$volid}->{is_tpmstate} //= 0; $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0'; $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key); }; my $include_opts = { extra_keys => ['vmstate'], include_unused => 1, }; PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid); PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1) if defined($conf->{pending}) && $conf->{pending}->%*; foreach my $snapname (keys %{ $conf->{snapshots} }) { my $snap = $conf->{snapshots}->{$snapname}; PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname); } foreach my $volid (keys %$volhash) { &$func($volid, $volhash->{$volid}, @param); } } my $fast_plug_option = { 'description' => 1, 'hookscript' => 1, 'lock' => 1, 'migrate_downtime' => 1, 'migrate_speed' => 1, 'name' => 1, 'onboot' => 1, 'protection' => 1, 'shares' => 1, 'startup' => 1, 'tags' => 1, 'vmstatestorage' => 1, }; for my $opt (keys %$confdesc_cloudinit) { $fast_plug_option->{$opt} = 1; } # hotplug changes in [PENDING] # $selection hash can be used to only apply specified options, for # example: { cores => 1 } (only apply changed 'cores') # $errors ref is used to return error messages sub vmconfig_hotplug_pending { my ($vmid, $conf, $storecfg, $selection, $errors) = @_; my $defaults = load_defaults(); my $arch = PVE::QemuServer::Helpers::get_vm_arch($conf); my $machine_type = PVE::QemuServer::Machine::get_vm_machine($conf); # commit values which do not have any impact on running VM first # Note: those option cannot raise errors, we we do not care about # $selection and always apply them. my $add_error = sub { my ($opt, $msg) = @_; $errors->{$opt} = "hotplug problem - $msg"; }; my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties(); my $cloudinit_record_changed = sub { my ($conf, $opt, $old, $new) = @_; return if !$cloudinit_pending_properties->{$opt}; my $ci = ($conf->{'special-sections'}->{cloudinit} //= {}); my $recorded = $ci->{$opt}; my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // ''); if (defined($new)) { if (defined($old)) { # an existing value is being modified if (defined($recorded)) { # the value was already not in sync if ($new eq $recorded) { # a value is being reverted to the cloud-init state: delete $ci->{$opt}; delete $added{$opt}; } else { # the value was changed multiple times, do nothing } } elsif ($added{$opt}) { # the value had been marked as added and is being changed, do nothing } else { # the value is new, record it: $ci->{$opt} = $old; } } else { # a new value is being added if (defined($recorded)) { # it was already not in sync if ($new eq $recorded) { # a value is being reverted to the cloud-init state: delete $ci->{$opt}; delete $added{$opt}; } else { # the value had temporarily been removed, do nothing } } elsif ($added{$opt}) { # the value had been marked as added already, do nothing } else { # the value is new, add it $added{$opt} = 1; } } } elsif (!defined($old)) { # a non-existent value is being removed? ignore... } else { # a value is being deleted if (defined($recorded)) { # a value was already recorded, just keep it } elsif ($added{$opt}) { # the value was marked as added, remove it delete $added{$opt}; } else { # a previously unrecorded value is being removed, record the old value: $ci->{$opt} = $old; } } my $added = join(',', sort keys %added); $ci->{added} = $added if length($added); }; my $changes = 0; foreach my $opt (keys %{ $conf->{pending} }) { # add/change if ($fast_plug_option->{$opt}) { my $new = delete $conf->{pending}->{$opt}; $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new); $conf->{$opt} = $new; $changes = 1; } } if ($changes) { PVE::QemuConfig->write_config($vmid, $conf); } my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1'); my $usb_hotplug; my $usb_was_unplugged = 0; my $is_usb_hotplug_supported = sub { return $usb_hotplug if defined($usb_hotplug); my $ostype = $conf->{ostype}; my $version = extract_version($machine_type, get_running_qemu_version($vmid)); $usb_hotplug = $hotplug_features->{usb} && min_version($version, 7, 1) && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7) ? 1 : 0; return $usb_hotplug; }; my $cgroup = PVE::QemuServer::CGroup->new($vmid); my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete}); foreach my $opt (sort keys %$pending_delete_hash) { next if $selection && !$selection->{$opt}; my $force = $pending_delete_hash->{$opt}->{force}; eval { if ($opt eq 'hotplug') { die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/); } elsif ($opt eq 'tablet') { die "skip\n" if !$hotplug_features->{usb}; if ($defaults->{tablet}) { vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type); vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type) if $arch eq 'aarch64'; } else { vm_deviceunplug($vmid, $conf, 'tablet'); vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64'; } } elsif ($opt =~ m/^usb(\d+)$/) { my $index = $1; die "skip\n" if !$is_usb_hotplug_supported->(); vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port vm_deviceunplug($vmid, $conf, $opt); $usb_was_unplugged = 1; } elsif ($opt eq 'vcpus') { die "skip\n" if !$hotplug_features->{cpu}; qemu_cpu_hotplug($vmid, $conf, undef); } elsif ($opt eq 'balloon') { # enable balloon device is not hotpluggable die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0; # here we reset the ballooning value to memory my $balloon = get_current_memory($conf->{memory}); mon_cmd($vmid, "balloon", value => $balloon * 1024 * 1024); } elsif ($fast_plug_option->{$opt}) { # do nothing } elsif ($opt =~ m/^net(\d+)$/) { die "skip\n" if !$hotplug_features->{network}; vm_deviceunplug($vmid, $conf, $opt); my $net = PVE::QemuServer::Network::parse_net($conf->{$opt}); PVE::Network::SDN::Vnets::del_ips_from_mac( $net->{bridge}, $net->{macaddr}, $conf->{name}, ); } elsif (is_valid_drivename($opt)) { die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(efidisk|ide|sata|tpmstate)(\d+)/; vm_deviceunplug($vmid, $conf, $opt); vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force); } elsif ($opt =~ m/^memory$/) { die "skip\n" if !$hotplug_features->{memory}; PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf); } elsif ($opt eq 'cpuunits') { $cgroup->change_cpu_shares(undef); } elsif ($opt eq 'cpulimit') { $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values } else { die "skip\n"; } }; if (my $err = $@) { &$add_error($opt, $err) if $err ne "skip\n"; } else { my $old = delete $conf->{$opt}; $cloudinit_record_changed->($conf, $opt, $old, undef); PVE::QemuConfig->remove_from_pending_delete($conf, $opt); } } my $cloudinit_opt; foreach my $opt (keys %{ $conf->{pending} }) { next if $selection && !$selection->{$opt}; my $value = $conf->{pending}->{$opt}; eval { if ($opt eq 'hotplug') { die "skip\n" if ($value =~ /memory/) || ($conf->{hotplug} && $conf->{hotplug} =~ /memory/); die "skip\n" if ($value =~ /cpu/) || ($conf->{hotplug} && $conf->{hotplug} =~ /cpu/); } elsif ($opt eq 'tablet') { die "skip\n" if !$hotplug_features->{usb}; if ($value == 1) { vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type); vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type) if $arch eq 'aarch64'; } elsif ($value == 0) { vm_deviceunplug($vmid, $conf, 'tablet'); vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64'; } } elsif ($opt =~ m/^usb(\d+)$/) { my $index = $1; die "skip\n" if !$is_usb_hotplug_supported->(); my $d = eval { parse_property_string('pve-qm-usb', $value) }; my $id = $opt; if ($d->{host} =~ m/^spice$/i) { $id = "usbredirdev$index"; } qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type); } elsif ($opt eq 'vcpus') { die "skip\n" if !$hotplug_features->{cpu}; qemu_cpu_hotplug($vmid, $conf, $value); } elsif ($opt eq 'balloon') { # enable/disable ballooning device is not hotpluggable my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon}); my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon}); die "skip\n" if $old_balloon_enabled != $new_balloon_enabled; # allow manual ballooning if shares is set to zero if ((defined($conf->{shares}) && ($conf->{shares} == 0))) { my $memory = get_current_memory($conf->{memory}); my $balloon = $conf->{pending}->{balloon} || $memory; mon_cmd($vmid, "balloon", value => $balloon * 1024 * 1024); } } elsif ($opt =~ m/^net(\d+)$/) { # some changes can be done without hotplug vmconfig_update_net( $storecfg, $conf, $hotplug_features->{network}, $vmid, $opt, $value, $arch, $machine_type, ); } elsif (is_valid_drivename($opt)) { die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0'; # some changes can be done without hotplug my $drive = parse_drive($opt, $value); if (drive_is_cloudinit($drive)) { $cloudinit_opt = [$opt, $drive]; # apply all the other changes first, then generate the cloudinit disk die "skip\n"; } vmconfig_update_disk( $storecfg, $conf, $hotplug_features->{disk}, $vmid, $opt, $value, $arch, $machine_type, ); } elsif ($opt =~ m/^memory$/) { #dimms die "skip\n" if !$hotplug_features->{memory}; $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value); } elsif ($opt eq 'cpuunits') { my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp $cgroup->change_cpu_shares($new_cpuunits); } elsif ($opt eq 'cpulimit') { my $cpulimit = $conf->{pending}->{$opt} == 0 ? undef : int($conf->{pending}->{$opt} * 100); $cgroup->change_cpu_quota($cpulimit, undef); } elsif ($opt eq 'agent') { vmconfig_update_agent($conf, $opt, $value); } else { die "skip\n"; # skip non-hot-pluggable options } }; if (my $err = $@) { &$add_error($opt, $err) if $err ne "skip\n"; } else { $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value); $conf->{$opt} = $value; delete $conf->{pending}->{$opt}; } } if (defined($cloudinit_opt)) { my ($opt, $drive) = @$cloudinit_opt; my $value = $conf->{pending}->{$opt}; eval { my $temp = { %$conf, $opt => $value }; PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid); vmconfig_update_disk( $storecfg, $conf, $hotplug_features->{disk}, $vmid, $opt, $value, $arch, $machine_type, ); }; if (my $err = $@) { &$add_error($opt, $err) if $err ne "skip\n"; } else { $conf->{$opt} = $value; delete $conf->{pending}->{$opt}; } } # unplug xhci controller if no usb device is left if ($usb_was_unplugged) { my $has_usb = 0; for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) { next if !defined($conf->{"usb$i"}); $has_usb = 1; last; } if (!$has_usb) { vm_deviceunplug($vmid, $conf, 'xhci'); } } PVE::QemuConfig->write_config($vmid, $conf); if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) { PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid); } } sub try_deallocate_drive { my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_; if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) { my $volid = $drive->{file}; if (vm_is_volid_owner($storecfg, $vmid, $volid)) { my $sid = PVE::Storage::parse_volume_id($volid); $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']); # check if the disk is really unused die "unable to delete '$volid' - volume is still in use (snapshot?)\n" if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid); PVE::Storage::vdisk_free($storecfg, $volid); return 1; } else { # If vm is not owner of this disk remove from config return 1; } } return; } sub vmconfig_delete_or_detach_drive { my ($vmid, $storecfg, $conf, $opt, $force) = @_; my $drive = parse_drive($opt, $conf->{$opt}); my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); if ($force) { $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']); try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force); } else { vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive); } } sub vmconfig_apply_pending { my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_; return if !scalar(keys %{ $conf->{pending} }); my $add_apply_error = sub { my ($opt, $msg) = @_; my $err_msg = "unable to apply pending change $opt : $msg"; $errors->{$opt} = $err_msg; warn $err_msg; }; # cold plug my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete}); foreach my $opt (sort keys %$pending_delete_hash) { my $force = $pending_delete_hash->{$opt}->{force}; eval { if ($opt =~ m/^unused/) { die "internal error"; } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) { vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force); } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) { my $net = PVE::QemuServer::Network::parse_net($conf->{$opt}); eval { PVE::Network::SDN::Vnets::del_ips_from_mac( $net->{bridge}, $net->{macaddr}, $conf->{name}, ); }; warn if $@; } }; if (my $err = $@) { $add_apply_error->($opt, $err); } else { PVE::QemuConfig->remove_from_pending_delete($conf, $opt); delete $conf->{$opt}; } } PVE::QemuConfig->cleanup_pending($conf); my $generate_cloudinit = $skip_cloud_init ? 0 : undef; foreach my $opt (keys %{ $conf->{pending} }) { # add/change next if $opt eq 'delete'; # just to be sure eval { if (defined($conf->{$opt}) && is_valid_drivename($opt)) { my $old_drive = parse_drive($opt, $conf->{$opt}); vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive); if ($opt eq 'efidisk0') { my $new_drive = parse_drive($opt, $conf->{pending}->{$opt}); PVE::QemuServer::OVMF::drive_change( $storecfg, $vmid, $old_drive, $new_drive, ); $conf->{pending}->{$opt} = print_drive($new_drive); } } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) { my $new_net = PVE::QemuServer::Network::parse_net($conf->{pending}->{$opt}); if ($conf->{$opt}) { my $old_net = PVE::QemuServer::Network::parse_net($conf->{$opt}); if ( defined($old_net->{bridge}) && defined($old_net->{macaddr}) && (safe_string_ne($old_net->{bridge}, $new_net->{bridge}) || safe_string_ne($old_net->{macaddr}, $new_net->{macaddr})) ) { PVE::Network::SDN::Vnets::del_ips_from_mac( $old_net->{bridge}, $old_net->{macaddr}, $conf->{name}, ); PVE::Network::SDN::Vnets::add_next_free_cidr( $new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1, ); } } else { PVE::Network::SDN::Vnets::add_next_free_cidr( $new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1, ); } } }; if (my $err = $@) { $add_apply_error->($opt, $err); } else { if (is_valid_drivename($opt)) { my $drive = parse_drive($opt, $conf->{pending}->{$opt}); $generate_cloudinit //= 1 if drive_is_cloudinit($drive); } $conf->{$opt} = delete $conf->{pending}->{$opt}; } } # write all changes at once to avoid unnecessary i/o PVE::QemuConfig->write_config($vmid, $conf); if ($generate_cloudinit) { if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) { # After successful generation and if there were changes to be applied, update the # config to drop the 'cloudinit' special section. PVE::QemuConfig->write_config($vmid, $conf); } } } sub vmconfig_update_net { my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_; my $newnet = PVE::QemuServer::Network::parse_net($value); if ($conf->{$opt}) { my $oldnet = PVE::QemuServer::Network::parse_net($conf->{$opt}); if ( safe_string_ne($oldnet->{model}, $newnet->{model}) || safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) || safe_num_ne($oldnet->{queues}, $newnet->{queues}) || safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) || !($newnet->{bridge} && $oldnet->{bridge}) ) { # bridge/nat mode change # for non online change, we try to hot-unplug die "skip\n" if !$hotplug; vm_deviceunplug($vmid, $conf, $opt); PVE::Network::SDN::Vnets::del_ips_from_mac( $oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name}, ); } else { die "internal error" if $opt !~ m/net(\d+)/; my $iface = "tap${vmid}i$1"; if ( safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) || safe_num_ne($oldnet->{tag}, $newnet->{tag}) || safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) || safe_num_ne($oldnet->{firewall}, $newnet->{firewall}) ) { PVE::Network::tap_unplug($iface); #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example) if ( safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) || safe_num_ne($oldnet->{tag}, $newnet->{tag}) ) { qemu_set_link_status($vmid, $opt, 0); } if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) { PVE::Network::SDN::Vnets::del_ips_from_mac( $oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name}, ); PVE::Network::SDN::Vnets::add_next_free_cidr( $newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1, ); } PVE::QemuServer::Network::tap_plug( $iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate}, ); } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) { # Rate can be applied on its own but any change above needs to # include the rate in tap_plug since OVS resets everything. PVE::Network::tap_rate_limit($iface, $newnet->{rate}); } # set link_down on changed bridge/tag as well, because we detach the # network device in the section above if the bridge or tag changed if ( safe_string_ne($oldnet->{link_down}, $newnet->{link_down}) || safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) || safe_num_ne($oldnet->{tag}, $newnet->{tag}) ) { qemu_set_link_status($vmid, $opt, !$newnet->{link_down}); } return 1; } } if ($hotplug) { PVE::Network::SDN::Vnets::add_next_free_cidr( $newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1, ); PVE::Network::SDN::Vnets::add_dhcp_mapping( $newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name}, ); vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type); } else { die "skip\n"; } } sub vmconfig_update_agent { my ($conf, $opt, $value) = @_; die "skip\n" if !$conf->{$opt}; my $hotplug_options = { fstrim_cloned_disks => 1 }; my $old_agent = parse_guest_agent($conf); my $agent = parse_guest_agent({ $opt => $value }); for my $option (keys %$agent) { # added/changed options next if defined($hotplug_options->{$option}); die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option}); } for my $option (keys %$old_agent) { # removed options next if defined($hotplug_options->{$option}); die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option}); } return; # either no actual change (e.g., format string reordered) or just hotpluggable changes } sub vmconfig_update_disk { my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_; my $drive = parse_drive($opt, $value); if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) { my $media = $drive->{media} || 'disk'; my $oldmedia = $old_drive->{media} || 'disk'; die "unable to change media type\n" if $media ne $oldmedia; if (!drive_is_cdrom($old_drive)) { if ($drive->{file} ne $old_drive->{file}) { die "skip\n" if !$hotplug; # unplug and register as unused vm_deviceunplug($vmid, $conf, $opt); vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive); } else { # update existing disk # skip non hotpluggable value if ( safe_string_ne($drive->{aio}, $old_drive->{aio}) || safe_string_ne($drive->{discard}, $old_drive->{discard}) || safe_string_ne($drive->{iothread}, $old_drive->{iothread}) || safe_string_ne($drive->{queues}, $old_drive->{queues}) || safe_string_ne($drive->{product}, $old_drive->{product}) || safe_string_ne($drive->{cache}, $old_drive->{cache}) || safe_string_ne($drive->{ssd}, $old_drive->{ssd}) || safe_string_ne($drive->{vendor}, $old_drive->{vendor}) || safe_string_ne($drive->{ro}, $old_drive->{ro}) ) { die "skip\n"; } # apply throttle if ( safe_num_ne($drive->{mbps}, $old_drive->{mbps}) || safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) || safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) || safe_num_ne($drive->{iops}, $old_drive->{iops}) || safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) || safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) || safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) || safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) || safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) || safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) || safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) || safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) || safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) || safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) || safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) || safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) || safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) || safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length}) ) { PVE::QemuServer::Blockdev::set_io_throttle( $vmid, "drive-$opt", ($drive->{mbps} || 0) * 1024 * 1024, ($drive->{mbps_rd} || 0) * 1024 * 1024, ($drive->{mbps_wr} || 0) * 1024 * 1024, $drive->{iops} || 0, $drive->{iops_rd} || 0, $drive->{iops_wr} || 0, ($drive->{mbps_max} || 0) * 1024 * 1024, ($drive->{mbps_rd_max} || 0) * 1024 * 1024, ($drive->{mbps_wr_max} || 0) * 1024 * 1024, $drive->{iops_max} || 0, $drive->{iops_rd_max} || 0, $drive->{iops_wr_max} || 0, $drive->{bps_max_length} || 1, $drive->{bps_rd_max_length} || 1, $drive->{bps_wr_max_length} || 1, $drive->{iops_max_length} || 1, $drive->{iops_rd_max_length} || 1, $drive->{iops_wr_max_length} || 1, ); } return 1; } } else { # cdrom eval { PVE::QemuServer::Blockdev::change_medium($storecfg, $vmid, $opt, $drive); }; my $err = $@; if ($drive->{file} eq 'none' && drive_is_cloudinit($old_drive)) { vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive); } die $err if $err; return 1; } } die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/; # hotplug new disks PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|; vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type); } sub vmconfig_update_cloudinit_drive { my ($storecfg, $conf, $vmid) = @_; my $cloudinit_ds = undef; my $cloudinit_drive = undef; PVE::QemuConfig->foreach_volume( $conf, sub { my ($ds, $drive) = @_; if (PVE::QemuServer::drive_is_cloudinit($drive)) { $cloudinit_ds = $ds; $cloudinit_drive = $drive; } }, ); return if !$cloudinit_drive; if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) { PVE::QemuConfig->write_config($vmid, $conf); } my $running = PVE::QemuServer::check_running($vmid); if ($running) { PVE::QemuServer::Blockdev::change_medium($storecfg, $vmid, $cloudinit_ds, $cloudinit_drive); } } # called in locked context by incoming migration sub vm_migrate_get_nbd_disks { my ($storecfg, $conf, $replicated_volumes) = @_; my $local_volumes = {}; PVE::QemuConfig->foreach_volume( $conf, sub { my ($ds, $drive) = @_; return if drive_is_cdrom($drive); return if $ds eq 'tpmstate0'; my $volid = $drive->{file}; return if !$volid; my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); my $scfg = PVE::Storage::storage_config($storecfg, $storeid); return if $scfg->{shared}; my $format = checked_volume_format($storecfg, $volid); # replicated disks re-use existing state via bitmap my $use_existing = $replicated_volumes->{$volid} ? 1 : 0; $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format]; }, ); return $local_volumes; } # called in locked context by incoming migration sub vm_migrate_alloc_nbd_disks { my ($storecfg, $vmid, $source_volumes, $storagemap) = @_; my $nbd = {}; foreach my $opt (sort keys %$source_volumes) { my ($volid, $storeid, $drive, $use_existing, $format) = @{ $source_volumes->{$opt} }; if ($use_existing) { $nbd->{$opt}->{drivestr} = print_drive($drive); $nbd->{$opt}->{volid} = $volid; $nbd->{$opt}->{replicated} = 1; next; } $storeid = PVE::JSONSchema::map_id($storagemap, $storeid); # order of precedence, filtered by whether storage supports it: # 1. explicit requested format # 2. default format of storage $format = PVE::Storage::resolve_format_hint($storecfg, $storeid, $format); my $size = $drive->{size} / 1024; my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size); my $newdrive = $drive; $newdrive->{format} = $format; $newdrive->{file} = $newvolid; my $drivestr = print_drive($newdrive); $nbd->{$opt}->{drivestr} = $drivestr; $nbd->{$opt}->{volid} = $newvolid; } return $nbd; } my sub remove_left_over_vmstate_opts { my ($vmid, $conf) = @_; my $found; for my $opt (qw(running-nets-host-mtu runningmachine runningcpu)) { if (defined($conf->{$opt})) { print "No VM state set, removing left-over option '$opt'\n"; delete $conf->{$opt}; $found = 1; } } PVE::QemuConfig->write_config($vmid, $conf) if $found; } sub generate_storage_hints { my ($conf, $plugin_may_deactivate_volume) = @_; my $hints = {}; if (PVE::Storage::Plugin::is_hint_supported('guest-is-windows')) { $hints->{'guest-is-windows'} = int(!!windows_version($conf->{ostype})); } if (PVE::Storage::Plugin::is_hint_supported('plugin-may-deactivate-volume')) { $hints->{'plugin-may-deactivate-volume'} = $plugin_may_deactivate_volume || 0; } return $hints; } my sub check_efi_vars { my ($storecfg, $vmid, $conf) = @_; return if PVE::QemuConfig->is_template($conf); return if !$conf->{efidisk0}; my $efidisk = parse_drive('efidisk0', $conf->{efidisk0}); if (PVE::QemuServer::OVMF::should_enroll_ms_2023_cert($efidisk)) { # TODO: make the first print a log_warn with PVE 9.2 to make it more noticeable! print "EFI disk without 'ms-cert=2023k' option, suggesting that not all UEFI 2023\n"; print "certificates from Microsoft are enrolled yet. The UEFI 2011 certificates expire\n"; print "in June 2026! The new certificates are required for secure boot update for Windows\n"; print "and common Linux distributions. Use 'Disk Action > Enroll Updated Certificates'\n"; print "in the UI or, while the VM is shut down, run 'qm enroll-efi-keys $vmid' to enroll\n"; print "the new certificates.\n\n"; print "For Windows with BitLocker, run the following command inside Powershell:\n"; print " manage-bde -protectors -disable \n"; print "for each drive with BitLocker (for example, could be 'C:').\n"; print "This is required for each drive with BitLocker before proceeding with enrollment.\n"; print "Otherwise, you will be prompted for the BitLocker recovery key on the next boot.\n"; } return; } my $log_filter_catch_outdated_zen5_firmware = sub { my ($line) = @_; print "$line\n"; if ($line =~ m/host doesn't support requested feature:.*rdseed\s*\[bit 18\]/) { log_warn("On Zen 5 systems, the rdseed CPU flag might not be available when the CPU" . " firmware is outdated. See:\n" . "https://security-tracker.debian.org/tracker/CVE-2025-62626\n" . "https://pve.proxmox.com/pve-docs/chapter-sysadmin.html#sysadmin_firmware_cpu"); } }; # see vm_start_nolock for parameters, additionally: # migrate_opts: # storagemap = parsed storage map for allocating NBD disks sub vm_start { my ($storecfg, $vmid, $params, $migrate_opts) = @_; return PVE::QemuConfig->lock_config( $vmid, sub { my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom}); die "you can't start a vm if it's a template\n" if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf); my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended'); my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup'); my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom}); if ($has_backup_lock && $running) { # a backup is currently running, attempt to start the guest in the # existing QEMU instance return PVE::QemuServer::RunState::vm_resume($vmid); } PVE::QemuConfig->check_lock($conf) if !($params->{skiplock} || $has_suspended_lock); $params->{resume} = $has_suspended_lock || defined($conf->{vmstate}); die "VM $vmid already running\n" if $running; if (my $storagemap = $migrate_opts->{storagemap}) { my $replicated = $migrate_opts->{replicated_volumes}; my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated); $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap); foreach my $opt (keys %{ $migrate_opts->{nbd} }) { $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr}; } } return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts); }, ); } # params: # statefile => 'tcp', 'unix' for migration or path/volid for RAM state # skiplock => 0/1, skip checking for config lock # skiptemplate => 0/1, skip checking whether VM is template # forcemachine => to force QEMU machine (rollback/migration) # forcecpu => a QEMU '-cpu' argument string to override get_cpu_options # timeout => in seconds # paused => start VM in paused state (backup) # resume => resume from hibernation # live-restore-backing => { # sata0 => { # name => blockdev-name, # blockdev => "arg to the -blockdev command instantiating device named 'name'", # }, # virtio2 => ... # } # nets-host-mtu => Used for migration compat. List of VirtIO network devices and their effective # host_mtu setting according to the QEMU object model on the source side of the migration. # migrate_opts: # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks) # migratedfrom => source node # spice_ticket => used for spice migration, passed via tunnel/stdin # network => CIDR of migration network # type => secure/insecure - tunnel over encrypted connection or plain-text # nbd_proto_version => int, 0 for TCP, 1 for UNIX # replicated_volumes => which volids should be re-used with bitmaps for nbd migration # offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet # contained in config # with_conntrack_state => whether to start the dbus-vmstate helper for conntrack state migration sub vm_start_nolock { my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_; my $statefile = $params->{statefile}; my $resume = $params->{resume}; my $migratedfrom = $migrate_opts->{migratedfrom}; my $migration_type = $migrate_opts->{type}; my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0; # clean up leftover reboot request files eval { clear_reboot_request($vmid); }; warn $@ if $@; # terminate left-over storage daemon if still running if (my $pid = PVE::QemuServer::Helpers::qsd_running_locally($vmid)) { log_warn("left-over QEMU storage daemon for $vmid running with PID $pid - terminating now"); PVE::QemuServer::QSD::quit($vmid); } if (!$statefile && !$resume && scalar(keys %{ $conf->{pending} })) { vmconfig_apply_pending($vmid, $conf, $storecfg); $conf = PVE::QemuConfig->load_config($vmid); # update/reload } # don't regenerate the ISO if the VM is started as part of a live migration # this way we can reuse the old ISO with the correct config if (!$migratedfrom) { if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) { # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop # $conf->{'special-sections'}->{cloudinit}, so we could just not do this? # But we do it above, so for now let's be consistent. $conf = PVE::QemuConfig->load_config($vmid); # update/reload } } # override offline migrated volumes, conf is out of date still if (my $offline_volumes = $migrate_opts->{offline_volumes}) { for my $key (sort keys $offline_volumes->%*) { my $parsed = parse_drive($key, $conf->{$key}); $parsed->{file} = $offline_volumes->{$key}; $conf->{$key} = print_drive($parsed); } } my $defaults = load_defaults(); # set environment variable useful inside network script # for remote migration the config is available on the target node! if (!$migrate_opts->{remote_node}) { $ENV{PVE_MIGRATED_FROM} = $migratedfrom; } PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1); my $forcemachine = $params->{forcemachine}; my $forcecpu = $params->{forcecpu}; my $nets_host_mtu = $params->{'nets-host-mtu'}; if ($resume) { # enforce machine and CPU type on suspended vm to ensure HW compatibility $forcemachine = $conf->{runningmachine}; $forcecpu = $conf->{runningcpu}; $nets_host_mtu = $conf->{'running-nets-host-mtu'}; print "Resuming suspended VM\n"; } my $migration_ip; if ( ($statefile && $statefile eq 'tcp' && $migration_type eq 'insecure') || ($migrate_opts->{nbd} && ($nbd_protocol_version == 0 || $migration_type eq 'insecure')) ) { my $nodename = nodename(); $migration_ip = PVE::QemuServer::StateFile::get_migration_ip($nodename, $migrate_opts->{network}); } my $res = {}; my $statefile_is_a_volume; my $state_cmdline = []; if ($statefile) { ($state_cmdline, $res->{migrate}, $statefile_is_a_volume) = PVE::QemuServer::StateFile::statefile_cmdline_option( $storecfg, $vmid, $statefile, $migration_type, $migration_ip, ); } elsif ($params->{paused}) { $state_cmdline = ['-S']; } my $vollist = get_current_vm_volumes($storecfg, $conf); push $vollist->@*, $statefile if $statefile_is_a_volume; my ($cmd, $spice_port, $start_timeout); my $pci_reserve_list = []; eval { # With -blockdev, it is necessary to activate the volumes before generating the command line # Plugins can safely deactivate already-active volumes here if needed my $storage_hints = generate_storage_hints($conf, 1); PVE::Storage::activate_volumes($storecfg, $vollist, undef, $storage_hints); check_efi_vars($storecfg, $vmid, $conf) if $conf->{bios} && $conf->{bios} eq 'ovmf'; # Note that for certain cases like templates, the configuration is minimized, so need to ensure # the rest of the function here uses the same configuration that was used to build the command ($cmd, $spice_port, my $pci_devices, $conf) = config_to_command( $storecfg, $vmid, $conf, $defaults, { 'force-machine' => $forcemachine, 'force-cpu' => $forcecpu, 'live-restore-backing' => $params->{'live-restore-backing'}, 'nets-host-mtu' => $nets_host_mtu, }, ); my $memory = get_current_memory($conf->{memory}); $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume); push $cmd->@*, $state_cmdline->@*; for my $device (values $pci_devices->%*) { next if $device->{mdev}; # we don't reserve for mdev devices push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*; } # reserve all PCI IDs before actually doing anything with them PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout); my $uuid; for my $id (sort keys %$pci_devices) { my $d = $pci_devices->{$id}; my ($index) = ($id =~ m/^hostpci(\d+)$/); my $chosen_mdev; for my $dev ($d->{ids}->@*) { my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d) }; if ($d->{mdev} || $d->{nvidia}) { warn $@ if $@; $chosen_mdev = $info; last if $chosen_mdev; # if successful, we're done } else { die $@ if $@; } } next if !$d->{mdev} && !$d->{nvidia}; die "could not create mediated device\n" if !defined($chosen_mdev); # nvidia grid needs the uuid of the mdev as qemu parameter if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) { if (defined($conf->{smbios1})) { my $smbios_conf = parse_smbios1($conf->{smbios1}); $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid}); } $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid); } } push @$cmd, '-uuid', $uuid if defined($uuid); }; if (my $err = $@) { eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); }; warn $@ if $@; eval { cleanup_pci_devices($vmid, $conf) }; warn $@ if $@; die $err; } my %silence_std_outs = (outfunc => sub { }, errfunc => sub { }); eval { # See systemd GH #39141, need to reset failed PartOf units too, or scope might be blocked run_command( ['/bin/systemctl', 'reset-failed', "pve-dbus-vmstate\@$vmid.service"], %silence_std_outs, ); }; eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) }; eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) }; # Issues with the above 'stop' not being fully completed are extremely rare, a very low # timeout should be more than enough here... PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20); my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits}); my %run_params = ( timeout => $statefile ? undef : $start_timeout, umask => 0077, noerr => 1, ); # when migrating, prefix QEMU output so other side can pick up any # errors that might occur and show the user if ($migratedfrom) { $run_params{quiet} = 1; $run_params{logfunc} = sub { print "QEMU: $_[0]\n" }; } elsif ($cpuinfo->{vendor} eq 'AuthenticAMD' && $cpuinfo->{family} == 26) { $run_params{logfunc} = $log_filter_catch_outdated_zen5_firmware; } my %systemd_properties = ( Slice => 'qemu.slice', KillMode => 'process', SendSIGKILL => 0, TimeoutStopUSec => ULONG_MAX, # infinity After => ['dbus.service'], # The point is ordering the VMID.scope after these during stop. # During start-up, the slice/scopes are not enabled. Before => ['pve-ha-lrm.service', 'pve-guests.service'], ); if (PVE::CGroup::cgroup_mode() == 2) { $systemd_properties{CPUWeight} = $cpuunits; } else { $systemd_properties{CPUShares} = $cpuunits; } if (my $cpulimit = $conf->{cpulimit}) { $systemd_properties{CPUQuota} = int($cpulimit * 100); } $systemd_properties{timeout} = 10 if $statefile; # setting up the scope should be quick my $cleanup_qsd = sub { if (PVE::QemuServer::Helpers::qsd_running_locally($vmid)) { eval { PVE::QemuServer::QSD::quit($vmid); }; warn "stopping QEMU storage daemon failed - $@" if $@; } }; my $run_qemu = sub { PVE::Tools::run_fork sub { PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties); my $virtiofs_sockets = start_all_virtiofsd($conf, $vmid); my $tpmpid; if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) { # start the TPM emulator so QEMU can connect on start eval { $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom); }; if (my $err = $@) { $cleanup_qsd->(); die $err; } } my $exitcode = run_command($cmd, %run_params); eval { PVE::QemuServer::Virtiofs::close_sockets(@$virtiofs_sockets); }; log_warn("closing virtiofs sockets failed - $@") if $@; if ($exitcode) { if ($tpmpid) { warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n"; kill 'TERM', $tpmpid; } $cleanup_qsd->(); die "QEMU exited with code $exitcode\n"; } }; }; if ($conf->{hugepages}) { my $code = sub { my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1'); my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory}); my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology(); PVE::QemuServer::Memory::hugepages_mount(); PVE::QemuServer::Memory::hugepages_allocate( $hugepages_topology, $hugepages_host_topology, ); eval { $run_qemu->() }; if (my $err = $@) { PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology) if !$conf->{keephugepages}; die $err; } PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology) if !$conf->{keephugepages}; }; eval { PVE::QemuServer::Memory::hugepages_update_locked($code); }; } else { eval { $run_qemu->() }; } if (my $err = $@) { # deactivate volumes if start fails eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); }; warn $@ if $@; eval { cleanup_pci_devices($vmid, $conf) }; warn $@ if $@; die "start failed: $err"; } # re-reserve all PCI IDs now that we can know the actual VM PID my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid); eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) }; warn $@ if $@; syslog("info", "VM $vmid started with PID $pid."); if (defined(my $migrate = $res->{migrate})) { if ($migrate->{proto} eq 'tcp') { my $nodename = nodename(); my $pfamily = PVE::Tools::get_host_address_family($nodename); $migrate->{port} = PVE::Tools::next_migrate_port($pfamily); $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}"; mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri}); } print "migration listens on $migrate->{uri}\n"; } elsif ($statefile) { eval { mon_cmd($vmid, "cont"); }; warn $@ if $@; } #start nbd server for storage migration if (my $nbd = $migrate_opts->{nbd}) { my $migrate_storage_uri; # nbd_protocol_version > 0 for unix socket support if ( $nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket') ) { my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate"; mon_cmd( $vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } }, ); $migrate_storage_uri = "nbd:unix:$socket_path"; $res->{migrate}->{unix_sockets} = [$socket_path]; } else { my $nodename = nodename(); my $localip = $migration_ip // die "internal error - no migration IP"; my $pfamily = PVE::Tools::get_host_address_family($nodename); my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily); mon_cmd( $vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}", }, }, ); $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip); $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}"; } my $block_info = PVE::QemuServer::Blockdev::get_block_info($vmid); foreach my $opt (sort keys %$nbd) { my $drivestr = $nbd->{$opt}->{drivestr}; my $volid = $nbd->{$opt}->{volid}; my $block_node = $block_info->{$opt}->{inserted}->{'node-name'}; mon_cmd( $vmid, "block-export-add", id => "drive-$opt", 'node-name' => $block_node, writable => JSON::true, type => "nbd", name => "drive-$opt", # NBD export name ); my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt"; print "storage migration listens on $nbd_uri volume:$drivestr\n"; print "re-using replicated volume: $opt - $volid\n" if $nbd->{$opt}->{replicated}; $res->{drives}->{$opt} = $nbd->{$opt}; $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri; } } if ($migratedfrom) { eval { PVE::QemuMigrate::Helpers::set_migration_caps($vmid); }; warn $@ if $@; if ($spice_port) { print "spice listens on port $spice_port\n"; $res->{spice_port} = $spice_port; if ($migrate_opts->{spice_ticket}) { mon_cmd( $vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket}, ); mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30"); } } # conntrack migration is only supported for intra-cluster migrations if ($migrate_opts->{with_conntrack_state} && !$migrate_opts->{remote_node}) { PVE::QemuServer::DBusVMState::qemu_add_dbus_vmstate($vmid); } } else { mon_cmd($vmid, "balloon", value => $conf->{balloon} * 1024 * 1024) if !$statefile && $conf->{balloon}; foreach my $opt (keys %$conf) { next if $opt !~ m/^net\d+$/; my $nicconf = PVE::QemuServer::Network::parse_net($conf->{$opt}); qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down}; } PVE::QemuServer::Network::add_nets_bridge_fdb($conf, $vmid); } if (!defined($conf->{balloon}) || $conf->{balloon}) { eval { mon_cmd( $vmid, 'qom-set', path => "machine/peripheral/balloon0", property => "guest-stats-polling-interval", value => 2, ); }; log_warn("could not set polling interval for ballooning - $@") if $@; } if ($resume) { print "Resumed VM, removing state\n"; if (my $vmstate = $conf->{vmstate}) { PVE::Storage::deactivate_volumes($storecfg, [$vmstate]); PVE::Storage::vdisk_free($storecfg, $vmstate); } delete $conf->@{qw(lock vmstate running-nets-host-mtu runningmachine runningcpu)}; PVE::QemuConfig->write_config($vmid, $conf); } elsif (!$conf->{vmstate} && !$migratedfrom) { remove_left_over_vmstate_opts($vmid, $conf); } PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start'); my ($current_machine, $is_deprecated) = PVE::QemuServer::Machine::get_current_qemu_machine($vmid); if ($is_deprecated) { log_warn( "current machine version '$current_machine' is deprecated - see the documentation and " . "change to a newer one", ); } return $res; } sub vm_commandline { my ($storecfg, $vmid, $snapname) = @_; my $conf = PVE::QemuConfig->load_config($vmid); my $options = { 'dry-run' => 1 }; if ($snapname) { my $snapshot = $conf->{snapshots}->{$snapname}; die "snapshot '$snapname' does not exist\n" if !defined($snapshot); # check for machine or CPU overrides in snapshot $options->{'force-machine'} = $snapshot->{runningmachine}; $options->{'force-cpu'} = $snapshot->{runningcpu}; $options->{'nets-host-mtu'} = $snapshot->{'running-nets-host-mtu'}; $snapshot->{digest} = $conf->{digest}; # keep file digest for API $conf = $snapshot; } my $defaults = load_defaults(); my $running = PVE::QemuServer::Helpers::vm_running_locally($vmid); my $volumes = []; # With -blockdev, it is necessary to activate the volumes before generating the command line if (!$running) { $volumes = get_current_vm_volumes($storecfg, $conf); PVE::Storage::activate_volumes($storecfg, $volumes); } # There might be concurrent operations on the volumes, so do not deactivate. my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $options); return PVE::Tools::cmd2string($cmd); } sub vm_reset { my ($vmid, $skiplock) = @_; PVE::QemuConfig->lock_config( $vmid, sub { my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_lock($conf) if !$skiplock; mon_cmd($vmid, "system_reset"); }, ); } sub get_vm_volumes { my ($conf) = @_; my $vollist = []; foreach_volid( $conf, sub { my ($volid, $attr) = @_; return if $volid =~ m|^/|; my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1); return if !$sid; push @$vollist, $volid; }, ); return $vollist; } # Get volumes defined in the current VM configuration, including the VM state file. sub get_current_vm_volumes { my ($storecfg, $conf) = @_; my $volumes = []; PVE::QemuConfig->foreach_volume_full( $conf, { extra_keys => ['vmstate'] }, sub { my ($ds, $drive) = @_; if (PVE::Storage::parse_volume_id($drive->{file}, 1)) { check_volume_storage_type($storecfg, $drive->{file}); push $volumes->@*, $drive->{file}; } }, ); return $volumes; } sub cleanup_pci_devices { my ($vmid, $conf) = @_; # templates don't use pci devices return if $conf->{template}; my $reservations = PVE::QemuServer::PCI::get_reservations($vmid); # clean up nvidia devices for my $id ($reservations->@*) { $id = PVE::SysFSTools::normalize_pci_id($id); my $create_path = "/sys/bus/pci/devices/$id/nvidia/current_vgpu_type"; next if !-f $create_path; for (my $i = 0; $i < 10; $i++) { last if file_read_firstline($create_path) eq "0"; sleep 1; PVE::SysFSTools::file_write($create_path, "0"); } if (file_read_firstline($create_path) ne "0") { warn "could not cleanup nvidia vgpu for '$id'\n"; } } foreach my $key (keys %$conf) { next if $key !~ m/^hostpci(\d+)$/; my $hostpciindex = $1; my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex); my $d = parse_hostpci($conf->{$key}); if ($d->{mdev}) { # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we # don't want to break ABI just for this two liner my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid"; # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error # out when we do it first. so wait for up to 10 seconds and then try it manually if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/ && -e $dev_sysfs_dir) { my $count = 0; while (-e $dev_sysfs_dir && $count < 10) { sleep 1; $count++; } print "waited $count seconds for mediated device driver finishing clean up\n"; } if (-e $dev_sysfs_dir) { print "actively clean up mediated device with UUID $uuid\n"; PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1"); } } } PVE::QemuServer::PCI::remove_pci_reservation($vmid); } sub vm_stop_cleanup { my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes, $noerr) = @_; eval { PVE::QemuServer::QSD::quit($vmid) if PVE::QemuServer::Helpers::qsd_running_locally($vmid); # ensure that no dbus-vmstate helper is left running in any case # at this point, it should never be still running, so quiesce any warnings PVE::QemuServer::DBusVMState::qemu_del_dbus_vmstate($vmid, quiet => 1); if (!$keepActive) { my $vollist = get_vm_volumes($conf); PVE::Storage::deactivate_volumes($storecfg, $vollist); } foreach my $ext (qw(mon qmp pid vnc qga)) { unlink "/var/run/qemu-server/${vmid}.$ext"; } if ($conf->{ivshmem}) { my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem}); # just delete it for now, VMs which have this already open do not # are affected, but new VMs will get a separated one. If this # becomes an issue we either add some sort of ref-counting or just # add a "don't delete on stop" flag to the ivshmem format. unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid); } cleanup_pci_devices($vmid, $conf); vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes; }; if (my $err = $@) { die $err if !$noerr; warn $err; } } # call only in locked context sub _do_vm_stop { my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_; my $pid = check_running($vmid, $nocheck); return if !$pid; my $conf; if (!$nocheck) { $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_lock($conf) if !$skiplock; if (!defined($timeout) && $shutdown && $conf->{startup}) { my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup}); $timeout = $opts->{down} if $opts->{down}; } PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop'); } eval { if ($shutdown) { if (defined($conf) && get_qga_key($conf, 'enabled') && qga_check_running($vmid)) { mon_cmd($vmid, "guest-shutdown", timeout => $timeout); } else { mon_cmd($vmid, "system_powerdown"); } } else { mon_cmd($vmid, "quit"); } }; my $err = $@; if (!$err) { $timeout = 60 if !defined($timeout); my $count = 0; while (($count < $timeout) && check_running($vmid, $nocheck)) { $count++; sleep 1; } if ($count >= $timeout) { if ($force) { warn "VM still running - terminating now with SIGTERM\n"; kill 15, $pid; } else { die "VM quit/powerdown failed - got timeout\n"; } } else { vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1, 1) if $conf; return; } } else { if (!check_running($vmid, $nocheck)) { warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n"; return; } if ($force) { warn "VM quit/powerdown failed - terminating now with SIGTERM\n"; kill 15, $pid; } else { die "VM quit/powerdown failed\n"; } } # wait again $timeout = 10; my $count = 0; while (($count < $timeout) && check_running($vmid, $nocheck)) { $count++; sleep 1; } if ($count >= $timeout) { warn "VM still running - terminating now with SIGKILL\n"; kill 9, $pid; sleep 1; } vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1, 1) if $conf; } # Note: use $nocheck to skip tests if VM configuration file exists. # We need that when migration VMs to other nodes (files already moved) # Note: we set $keepActive in vzdump stop mode - volumes need to stay active sub vm_stop { my ( $storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom, ) = @_; $force = 1 if !defined($force) && !$shutdown; if ($migratedfrom) { my $pid = check_running($vmid, $nocheck, $migratedfrom); kill 15, $pid if $pid; my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom); vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0, 1); return; } PVE::QemuConfig->lock_config( $vmid, sub { _do_vm_stop( $storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, ); }, ); } sub vm_reboot { my ($vmid, $timeout) = @_; PVE::QemuConfig->lock_config( $vmid, sub { eval { # only reboot if running, as qmeventd starts it again on a stop event return if !check_running($vmid); create_reboot_request($vmid); my $storecfg = PVE::Storage::config(); _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1); }; if (my $err = $@) { # avoid that the next normal shutdown will be confused for a reboot clear_reboot_request($vmid); die $err; } }, ); } sub vm_sendkey { my ($vmid, $skiplock, $key) = @_; PVE::QemuConfig->lock_config( $vmid, sub { my $conf = PVE::QemuConfig->load_config($vmid); # there is no qmp command, so we use the human monitor command my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key"); die $res if $res ne ''; }, ); } sub check_bridge_access { my ($rpcenv, $authuser, $conf) = @_; return 1 if $authuser eq 'root@pam'; for my $opt (sort keys $conf->%*) { next if $opt !~ m/^net\d+$/; my $net = PVE::QemuServer::Network::parse_net($conf->{$opt}); my ($bridge, $tag, $trunks) = $net->@{ 'bridge', 'tag', 'trunks' }; next if !defined($bridge); # no vnet to check for PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks); } return 1; } sub check_mapping_access { my ($rpcenv, $user, $conf) = @_; return 1 if $user eq 'root@pam'; for my $opt (keys $conf->%*) { if ($opt =~ m/^usb\d+$/) { my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt}); if (my $host = $device->{host}) { die "only root can set '$opt' config for real devices\n" if $host !~ m/^spice$/i; } elsif ($device->{mapping}) { $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']); } else { die "either 'host' or 'mapping' must be set.\n"; } } elsif ($opt =~ m/^hostpci\d+$/) { my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt}); if ($device->{host}) { die "only root can set '$opt' config for non-mapped devices\n"; } elsif ($device->{mapping}) { $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']); } else { die "either 'host' or 'mapping' must be set.\n"; } } elsif ($opt =~ m/^rng\d+$/) { my $device = PVE::JSONSchema::parse_property_string('pve-qm-rng', $conf->{$opt}); if ($device->{source} && $device->{source} eq '/dev/hwrng') { $rpcenv->check_full($user, "/mapping/hwrng", ['Mapping.Use']); } } elsif ($opt =~ m/^virtiofs\d$/) { my $virtiofs = PVE::JSONSchema::parse_property_string('pve-qm-virtiofs', $conf->{$opt}); $rpcenv->check_full($user, "/mapping/dir/$virtiofs->{dirid}", ['Mapping.Use']); } } } sub check_restore_permissions { my ($rpcenv, $user, $conf) = @_; check_bridge_access($rpcenv, $user, $conf); check_mapping_access($rpcenv, $user, $conf); } # vzdump restore implementation sub tar_archive_read_firstfile { my $archive = shift; die "ERROR: file '$archive' does not exist\n" if !-f $archive; # try to detect archive type first my $pid = open(my $fh, '-|', 'tar', 'tf', $archive) || die "unable to open file '$archive'\n"; my $firstfile = <$fh>; kill 15, $pid; close $fh; die "ERROR: archive contaions no data\n" if !$firstfile; chomp $firstfile; return $firstfile; } sub tar_restore_cleanup { my ($storecfg, $statfile) = @_; print STDERR "starting cleanup\n"; if (my $fd = IO::File->new($statfile, "r")) { while (defined(my $line = <$fd>)) { if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) { my $volid = $2; eval { if ($volid =~ m|^/|) { unlink $volid || die 'unlink failed\n'; } else { PVE::Storage::vdisk_free($storecfg, $volid); } print STDERR "temporary volume '$volid' successfully removed\n"; }; print STDERR "unable to cleanup '$volid' - $@" if $@; } else { print STDERR "unable to parse line in statfile - $line"; } } $fd->close(); } } sub restore_file_archive { my ($archive, $vmid, $user, $opts) = @_; return restore_vma_archive($archive, $vmid, $user, $opts) if $archive eq '-'; my $info = PVE::Storage::archive_info($archive); my $format = $opts->{format} // $info->{format}; my $comp = $info->{compression}; # try to detect archive format if ($format eq 'tar') { return restore_tar_archive($archive, $vmid, $user, $opts); } else { return restore_vma_archive($archive, $vmid, $user, $opts, $comp); } } # helper to remove disks that will not be used after restore my $restore_cleanup_oldconf = sub { my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_; my $kept_disks = {}; PVE::QemuConfig->foreach_volume( $oldconf, sub { my ($ds, $drive) = @_; return if drive_is_cdrom($drive, 1); my $volid = $drive->{file}; return if !$volid || $volid =~ m|^/|; my ($path, $owner) = PVE::Storage::path($storecfg, $volid); return if !$path || !$owner || ($owner != $vmid); # Note: only delete disk we want to restore # other volumes will become unused if ($virtdev_hash->{$ds}) { eval { PVE::Storage::vdisk_free($storecfg, $volid); }; if (my $err = $@) { warn $err; } } else { $kept_disks->{$volid} = 1; } }, ); # after the restore we have no snapshots anymore for my $snapname (keys $oldconf->{snapshots}->%*) { my $snap = $oldconf->{snapshots}->{$snapname}; if ($snap->{vmstate}) { eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); }; if (my $err = $@) { warn $err; } } for my $volid (keys $kept_disks->%*) { eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); }; warn $@ if $@; } } }; # Helper to parse vzdump backup device hints # # $rpcenv: Environment, used to ckeck storage permissions # $user: User ID, to check storage permissions # $storecfg: Storage configuration # $fh: the file handle for reading the configuration # $devinfo: should contain device sizes for all backu-up'ed devices # $options: backup options (pool, default storage) # # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid) my $parse_backup_hints = sub { my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_; my $check_storage = sub { # assert if an image can be allocate my ($storeid, $scfg) = @_; die "Content type 'images' is not available on storage '$storeid'\n" if !$scfg->{content}->{images}; $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']) if $user ne 'root@pam'; }; my $virtdev_hash = {}; while (defined(my $line = <$fh>)) { if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) { my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4); die "archive does not contain data for drive '$virtdev'\n" if !$devinfo->{$devname}; if (defined($options->{storage})) { $storeid = $options->{storage} || 'local'; } elsif (!$storeid) { $storeid = 'local'; } $format = 'raw' if !$format; $devinfo->{$devname}->{devname} = $devname; $devinfo->{$devname}->{virtdev} = $virtdev; $devinfo->{$devname}->{format} = $format; $devinfo->{$devname}->{storeid} = $storeid; my $scfg = PVE::Storage::storage_config($storecfg, $storeid); $check_storage->($storeid, $scfg); # permission and content type check $virtdev_hash->{$virtdev} = $devinfo->{$devname}; } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) { my $virtdev = $1; my $drive = parse_drive($virtdev, $2); if (drive_is_cloudinit($drive)) { my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}); $storeid = $options->{storage} if defined($options->{storage}); my $scfg = PVE::Storage::storage_config($storecfg, $storeid); my $format = eval { checked_volume_format($storecfg, $drive->{file}) } // 'raw'; $check_storage->($storeid, $scfg); # permission and content type check $virtdev_hash->{$virtdev} = { format => $format, storeid => $storeid, size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE, is_cloudinit => 1, }; } } } return $virtdev_hash; }; # Helper to allocate and activate all volumes required for a restore # # $storecfg: Storage configuration # $virtdev_hash: as returned by parse_backup_hints() # # Returns: { $virtdev => $volid } my $restore_allocate_devices = sub { my ($storecfg, $virtdev_hash, $vmid) = @_; my $map = {}; foreach my $virtdev (sort keys %$virtdev_hash) { my $d = $virtdev_hash->{$virtdev}; die "got no size for '$virtdev'\n" if !defined($d->{size}); my $alloc_size = int(($d->{size} + 1024 - 1) / 1024); my $storeid = $d->{storeid}; my $scfg = PVE::Storage::storage_config($storecfg, $storeid); # falls back to default format if requested format is not supported $d->{format} = PVE::Storage::resolve_format_hint($storecfg, $storeid, $d->{format}); my $name; if ($d->{is_cloudinit}) { $name = "vm-$vmid-cloudinit"; my $scfg = PVE::Storage::storage_config($storecfg, $storeid); if ($scfg->{path}) { $name .= ".$d->{format}"; } } my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size); print STDERR "new volume ID is '$volid'\n"; $d->{volid} = $volid; PVE::Storage::activate_volumes($storecfg, [$volid]); $map->{$virtdev} = $volid; } return $map; }; sub restore_update_config_line { my ($cookie, $map, $line, $unique) = @_; return '' if $line =~ m/^\#qmdump\#/; return '' if $line =~ m/^\#vzdump\#/; return '' if $line =~ m/^lock:/; return '' if $line =~ m/^unused\d+:/; return '' if $line =~ m/^parent:/; my $res = ''; my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg'); if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) { # try to convert old 1.X settings my ($id, $ind, $ethcfg) = ($1, $2, $3); foreach my $devconfig (PVE::Tools::split_list($ethcfg)) { my ($model, $macaddr) = split(/\=/, $devconfig); $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique; my $net = { model => $model, bridge => "vmbr$ind", macaddr => $macaddr, }; my $netstr = PVE::QemuServer::Network::print_net($net); $res .= "net$cookie->{netcount}: $netstr\n"; $cookie->{netcount}++; } } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) { my ($id, $netstr) = ($1, $2); my $net = PVE::QemuServer::Network::parse_net($netstr); $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr}; $netstr = PVE::QemuServer::Network::print_net($net); $res .= "$id: $netstr\n"; } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) { my $virtdev = $1; my $value = $3; my $di = parse_drive($virtdev, $value); if (defined($di->{backup}) && !$di->{backup}) { $res .= "#$line"; } elsif ($map->{$virtdev}) { delete $di->{format}; # format can change on restore $di->{file} = $map->{$virtdev}; $value = print_drive($di); $res .= "$virtdev: $value\n"; } else { $res .= $line; } } elsif (($line =~ m/^vmgenid: (.*)/)) { my $vmgenid = $1; if ($vmgenid ne '0') { # always generate a new vmgenid if there was a valid one setup $vmgenid = generate_uuid(); } $res .= "vmgenid: $vmgenid\n"; } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) { my ($uuid, $uuid_str); UUID::generate($uuid); UUID::unparse($uuid, $uuid_str); my $smbios1 = parse_smbios1($2); $smbios1->{uuid} = $uuid_str; $res .= $1 . print_smbios1($smbios1) . "\n"; } else { $res .= $line; } return $res; } my $restore_deactivate_volumes = sub { my ($storecfg, $virtdev_hash) = @_; my $vollist = []; for my $dev (values $virtdev_hash->%*) { push $vollist->@*, $dev->{volid} if $dev->{volid}; } eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); }; print STDERR $@ if $@; }; my $restore_destroy_volumes = sub { my ($storecfg, $virtdev_hash) = @_; for my $dev (values $virtdev_hash->%*) { my $volid = $dev->{volid} or next; eval { PVE::Storage::vdisk_free($storecfg, $volid); print STDERR "temporary volume '$volid' successfully removed\n"; }; print STDERR "unable to cleanup '$volid' - $@" if $@; } }; sub restore_merge_config { my ($filename, $backup_conf_raw, $override_conf) = @_; my $backup_conf = parse_vm_config($filename, $backup_conf_raw); for my $key (keys $override_conf->%*) { $backup_conf->{$key} = $override_conf->{$key}; } return $backup_conf; } sub scan_volids { my ($cfg, $vmid) = @_; my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images'); my $volid_hash = {}; foreach my $storeid (keys %$info) { foreach my $item (@{ $info->{$storeid} }) { next if !($item->{volid} && $item->{size}); $item->{path} = PVE::Storage::path($cfg, $item->{volid}); $volid_hash->{ $item->{volid} } = $item; } } return $volid_hash; } sub update_disk_config { my ($vmid, $conf, $volid_hash) = @_; my $changes; my $prefix = "VM $vmid"; # used and unused disks my $referenced = {}; # Note: it is allowed to define multiple storages with same path (alias), so # we need to check both 'volid' and real 'path' (two different volid can point # to the same path). my $referencedpath = {}; # update size info PVE::QemuConfig->foreach_volume( $conf, sub { my ($opt, $drive) = @_; my $volid = $drive->{file}; return if !$volid; my $volume = $volid_hash->{$volid}; # mark volid as "in-use" for next step $referenced->{$volid} = 1; if ($volume && (my $path = $volume->{path})) { $referencedpath->{$path} = 1; } return if drive_is_cdrom($drive); return if !$volume; my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size}); if (defined($updated)) { $changes = 1; $conf->{$opt} = print_drive($updated); print "$prefix ($opt): $msg\n"; } }, ); # remove 'unusedX' entry if volume is used PVE::QemuConfig->foreach_unused_volume( $conf, sub { my ($opt, $drive) = @_; my $volid = $drive->{file}; return if !$volid; my $path; $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid}; if ($referenced->{$volid} || ($path && $referencedpath->{$path})) { print "$prefix remove entry '$opt', its volume '$volid' is in use\n"; $changes = 1; delete $conf->{$opt}; } $referenced->{$volid} = 1; $referencedpath->{$path} = 1 if $path; }, ); if (my $fleecing = $conf->{'special-sections'}->{fleecing}) { $referenced->{$_} = 1 for PVE::Tools::split_list($fleecing->{'fleecing-images'}); } foreach my $volid (sort keys %$volid_hash) { next if $volid =~ m/vm-$vmid-state-/; next if $referenced->{$volid}; my $path = $volid_hash->{$volid}->{path}; next if !$path; # just to be sure next if $referencedpath->{$path}; $changes = 1; my $key = PVE::QemuConfig->add_unused_volume($conf, $volid); print "$prefix add unreferenced volume '$volid' as '$key' to config\n"; $referencedpath->{$path} = 1; # avoid to add more than once (aliases) } return $changes; } sub rescan { my ($vmid, $nolock, $dryrun) = @_; my $cfg = PVE::Storage::config(); print "rescan volumes...\n"; my $volid_hash = scan_volids($cfg, $vmid); my $updatefn = sub { my ($vmid) = @_; my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_lock($conf); my $vm_volids = {}; foreach my $volid (keys %$volid_hash) { my $info = $volid_hash->{$volid}; $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid; } my $changes = update_disk_config($vmid, $conf, $vm_volids); PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun; }; if (defined($vmid)) { if ($nolock) { &$updatefn($vmid); } else { PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid); } } else { my $vmlist = config_list(); foreach my $vmid (keys %$vmlist) { if ($nolock) { &$updatefn($vmid); } else { PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid); } } } } sub restore_proxmox_backup_archive { my ($archive, $vmid, $user, $options) = @_; my $storecfg = PVE::Storage::config(); my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive); my $scfg = PVE::Storage::storage_config($storecfg, $storeid); my $fingerprint = $scfg->{fingerprint}; my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid); my $repo = PVE::PBSClient::get_repository($scfg); my $namespace = $scfg->{namespace}; # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore) my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid); local $ENV{PBS_PASSWORD} = $password; local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint); my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) = PVE::Storage::parse_volname($storecfg, $archive); die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup'; die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm'; my $tmpdir = "/var/tmp/vzdumptmp$$"; rmtree $tmpdir; mkpath $tmpdir; my $conffile = PVE::QemuConfig->config_file($vmid); # disable interrupts (always do cleanups) local $SIG{INT} = local $SIG{TERM} = local $SIG{QUIT} = local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; }; # Note: $oldconf is undef if VM does not exists my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid); my $oldconf = PVE::Cluster::cfs_read_file($cfs_path); my $new_conf_raw = ''; my $rpcenv = PVE::RPCEnvironment::get(); my $devinfo = {}; # info about drives included in backup my $virtdev_hash = {}; # info about allocated drives eval { # enable interrupts local $SIG{INT} = local $SIG{TERM} = local $SIG{QUIT} = local $SIG{HUP} = local $SIG{PIPE} = sub { die "interrupted by signal\n"; }; my $cfgfn = "$tmpdir/qemu-server.conf"; my $firewall_config_fn = "$tmpdir/fw.conf"; my $index_fn = "$tmpdir/index.json"; my $cmd = "restore"; my $param = [$pbs_backup_name, "index.json", $index_fn]; PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param); my $index = PVE::Tools::file_get_contents($index_fn); $index = decode_json($index); foreach my $info (@{ $index->{files} }) { if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) { my $devname = $1; if ($info->{size} =~ m/^(\d+)$/) { # untaint size $devinfo->{$devname}->{size} = $1; } else { die "unable to parse file size in 'index.json' - got '$info->{size}'\n"; } } } my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{ $index->{files} }); if (!$is_qemu_server_backup) { die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n"; } my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{ $index->{files} }); $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn]; PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param); if ($has_firewall_config) { $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn]; PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param); my $pve_firewall_dir = '/etc/pve/firewall'; mkdir $pve_firewall_dir; # make sure the dir exists PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw"); } my $fh = IO::File->new($cfgfn, "r") || die "unable to read qemu-server.conf - $!\n"; $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options); # fixme: rate limit? # create empty/temp config PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create"); $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf; # allocate volumes my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid); foreach my $virtdev (sort keys %$virtdev_hash) { my $d = $virtdev_hash->{$virtdev}; next if $d->{is_cloudinit}; # no need to restore cloudinit # this fails if storage is unavailable my $volid = $d->{volid}; my $path = PVE::Storage::path($storecfg, $volid); # for live-restore we only want to preload the efidisk and TPM state next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0'; my @ns_arg; if (defined(my $ns = $scfg->{namespace})) { @ns_arg = ('--ns', $ns); } my $pbs_restore_cmd = [ '/usr/bin/pbs-restore', '--repository', $repo, @ns_arg, $pbs_backup_name, "$d->{devname}.img.fidx", $path, '--verbose', ]; push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format}; push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile; if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) { push @$pbs_restore_cmd, '--skip-zero'; } my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd); print "restore proxmox backup image: $dbg_cmdstring\n"; run_command($pbs_restore_cmd); } $fh->seek(0, 0) || die "seek failed - $!\n"; my $cookie = { netcount => 0 }; while (defined(my $line = <$fh>)) { $new_conf_raw .= restore_update_config_line( $cookie, $map, $line, $options->{unique}, ); } $fh->close(); }; my $err = $@; if ($err || !$options->{live}) { $restore_deactivate_volumes->($storecfg, $virtdev_hash); } rmtree $tmpdir; if ($err) { $restore_destroy_volumes->($storecfg, $virtdev_hash); die $err; } if ($options->{live}) { # keep lock during live-restore $new_conf_raw .= "\nlock: create"; } my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf}); check_restore_permissions($rpcenv, $user, $new_conf); PVE::QemuConfig->write_config($vmid, $new_conf); eval { rescan($vmid, 1); }; warn $@ if $@; PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool}; if ($options->{live}) { # enable interrupts local $SIG{INT} = local $SIG{TERM} = local $SIG{QUIT} = local $SIG{HUP} = local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; }; my $conf = PVE::QemuConfig->load_config($vmid); die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf); # these special drives are already restored before start delete $devinfo->{'drive-efidisk0'}; delete $devinfo->{'drive-tpmstate0-backup'}; my $pbs_opts = { repo => $repo, keyfile => $keyfile, snapshot => $pbs_backup_name, namespace => $namespace, }; pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts); PVE::QemuConfig->remove_lock($vmid, "create"); } } sub restore_external_archive { my ($backup_provider, $archive, $vmid, $user, $options) = @_; die "live restore from backup provider is not implemented\n" if $options->{live}; my $storecfg = PVE::Storage::config(); my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive); my $scfg = PVE::Storage::storage_config($storecfg, $storeid); my $tmpdir = "/run/qemu-server/vzdumptmp$$"; rmtree($tmpdir); mkpath($tmpdir) or die "unable to create $tmpdir\n"; my $conffile = PVE::QemuConfig->config_file($vmid); # disable interrupts (always do cleanups) local $SIG{INT} = local $SIG{TERM} = local $SIG{QUIT} = local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; }; # Note: $oldconf is undef if VM does not exists my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid); my $oldconf = PVE::Cluster::cfs_read_file($cfs_path); my $new_conf_raw = ''; my $rpcenv = PVE::RPCEnvironment::get(); my $devinfo = {}; # info about drives included in backup my $virtdev_hash = {}; # info about allocated drives eval { # enable interrupts local $SIG{INT} = local $SIG{TERM} = local $SIG{QUIT} = local $SIG{HUP} = local $SIG{PIPE} = sub { die "interrupted by signal\n"; }; my $cfgfn = "$tmpdir/qemu-server.conf"; my $firewall_config_fn = "$tmpdir/fw.conf"; my $cmd = "restore"; my ($mechanism, $vmtype) = $backup_provider->restore_get_mechanism($volname); die "mechanism '$mechanism' requested by backup provider is not supported for VMs\n" if $mechanism ne 'qemu-img'; die "cannot restore non-VM guest of type '$vmtype'\n" if $vmtype ne 'qemu'; $devinfo = $backup_provider->restore_vm_init($volname); my $data = $backup_provider->archive_get_guest_config($volname) or die "backup provider failed to extract guest configuration\n"; PVE::Tools::file_set_contents($cfgfn, $data); if ($data = $backup_provider->archive_get_firewall_config($volname)) { PVE::Tools::file_set_contents($firewall_config_fn, $data); my $pve_firewall_dir = '/etc/pve/firewall'; mkdir $pve_firewall_dir; # make sure the dir exists PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw"); } my $fh = IO::File->new($cfgfn, "r") or die "unable to read qemu-server.conf - $!\n"; $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options); # create empty/temp config PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create"); $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf; # allocate volumes my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid); for my $virtdev (sort keys $virtdev_hash->%*) { my $d = $virtdev_hash->{$virtdev}; next if $d->{is_cloudinit}; # no need to restore cloudinit my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $d->{volid}); my $source_format = 'raw'; my $info = $backup_provider->restore_vm_volume_init($volname, $d->{devname}, {}); my $source_path = $info->{'qemu-img-path'} or die "did not get source image path from backup provider\n"; print "importing drive '$d->{devname}' from '$source_path'\n"; # safety check for untrusted source image PVE::Storage::file_size_info($source_path, undef, $source_format, 1); eval { my $convert_opts = { bwlimit => $options->{bwlimit}, 'is-zero-initialized' => $sparseinit, 'source-path-format' => $source_format, }; PVE::QemuServer::QemuImage::convert( $source_path, $d->{volid}, $d->{size}, $convert_opts, ); }; my $err = $@; eval { $backup_provider->restore_vm_volume_cleanup($volname, $d->{devname}, {}); }; if (my $cleanup_err = $@) { die $cleanup_err if !$err; warn $cleanup_err; } die $err if $err; } $fh->seek(0, 0) || die "seek failed - $!\n"; my $cookie = { netcount => 0 }; while (defined(my $line = <$fh>)) { $new_conf_raw .= restore_update_config_line( $cookie, $map, $line, $options->{unique}, ); } $fh->close(); }; my $err = $@; eval { $backup_provider->restore_vm_cleanup($volname); }; warn "backup provider cleanup after restore failed - $@" if $@; if ($err) { $restore_deactivate_volumes->($storecfg, $virtdev_hash); } rmtree($tmpdir); if ($err) { $restore_destroy_volumes->($storecfg, $virtdev_hash); die $err; } my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf}); check_restore_permissions($rpcenv, $user, $new_conf); PVE::QemuConfig->write_config($vmid, $new_conf); eval { rescan($vmid, 1); }; warn $@ if $@; PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool}; return; } sub pbs_live_restore { my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_; print "starting VM for live-restore\n"; print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n"; my $live_restore_backing = {}; for my $ds (keys %$restored_disks) { $ds =~ m/^drive-(.*)$/; my $confname = $1; my $pbs_conf = {}; $pbs_conf = { repository => $opts->{repo}, snapshot => $opts->{snapshot}, archive => "$ds.img.fidx", }; $pbs_conf->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile}; $pbs_conf->{namespace} = $opts->{namespace} if defined($opts->{namespace}); my $drive = parse_drive($confname, $conf->{$confname}); print "restoring '$ds' to '$drive->{file}'\n"; my $pbs_name = "drive-${confname}-pbs"; $live_restore_backing->{$confname} = { name => $pbs_name }; # add blockdev information my $machine_type = PVE::QemuServer::Machine::get_vm_machine($conf); my $machine_version = PVE::QemuServer::Machine::extract_version( $machine_type, PVE::QemuServer::Helpers::kvm_user_version(), ); if (min_version($machine_version, 10, 0)) { # for the switch to -blockdev $live_restore_backing->{$confname}->{blockdev} = PVE::QemuServer::Blockdev::generate_pbs_blockdev($pbs_conf, $pbs_name); } else { $live_restore_backing->{$confname}->{blockdev} = print_pbs_blockdev($pbs_conf, $pbs_name); } } my $drives_streamed = 0; eval { # make sure HA doesn't interrupt our restore by stopping the VM if (vm_is_ha_managed($vmid)) { run_command(['ha-manager', 'set', "vm:$vmid", '--state', 'started']); } # start VM with backing chain pointing to PBS backup, environment vars for PBS driver # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller vm_start_nolock( $storecfg, $vmid, $conf, { paused => 1, 'live-restore-backing' => $live_restore_backing }, {}, ); my $qmeventd_fd = register_qmeventd_handle($vmid); # begin streaming, i.e. data copy from PBS to target disk for every vol, # this will effectively collapse the backing image chain consisting of # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track # removes itself once all backing images vanish with 'auto-remove=on') my $jobs = {}; for my $ds (sort keys %$restored_disks) { my $node_name = PVE::QemuServer::Blockdev::get_node_name_below_throttle(vm_qmp_peer($vmid), $ds); my $job_id = "restore-$ds"; mon_cmd( $vmid, 'block-stream', 'job-id' => $job_id, device => "$node_name", 'auto-dismiss' => JSON::false, ); $jobs->{$job_id} = {}; } mon_cmd($vmid, 'cont'); PVE::QemuServer::BlockJob::monitor( vm_qmp_peer($vmid), undef, $jobs, 'auto', 0, 'stream', ); print "restore-drive jobs finished successfully, removing all tracking block devices" . " to disconnect from Proxmox Backup Server\n"; for my $ds (sort keys %$restored_disks) { PVE::QemuServer::Blockdev::detach(vm_qmp_peer($vmid), "$ds-pbs"); } close($qmeventd_fd); }; my $err = $@; if ($err) { warn "An error occurred during live-restore: $err\n"; _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1); die "live-restore failed\n"; } } # Inspired by pbs live-restore, this restores with the disks being available as files. # Theoretically this can also be used to quick-start a full-clone vm if the # disks are all available as files. # # The mapping should provide a path by config entry, such as # `{ scsi0 => { format => , path => "/path/to/file", sata1 => ... } }` # # This is used when doing a `create` call with the `--live-import` parameter, # where the disks get an `import-from=` property. The non-live part is # therefore already handled in the `$create_disks()` call happening in the # `create` api call sub live_import_from_files { my ($mapping, $vmid, $conf, $restore_options) = @_; my $storecfg = PVE::Storage::config(); my $live_restore_backing = {}; my $sources_to_remove = []; for my $dev (keys %$mapping) { die "disk not support for live-restoring: '$dev'\n" if !is_valid_drivename($dev) || $dev =~ /^(?:efidisk|tpmstate)/; die "mapping contains disk '$dev' which does not exist in the config\n" if !exists($conf->{$dev}); my $info = $mapping->{$dev}; my ($format, $path, $volid) = $info->@{qw(format path volid)}; die "missing path for '$dev' mapping\n" if !$path; die "missing volid for '$dev' mapping\n" if !$volid; die "missing format for '$dev' mapping\n" if !$format; die "invalid format '$format' for '$dev' mapping\n" if !grep { $format eq $_ } qw(raw qcow2 vmdk); $live_restore_backing->{$dev} = { name => "drive-$dev-restore" }; my $machine_type = PVE::QemuServer::Machine::get_vm_machine($conf); my $machine_version = PVE::QemuServer::Machine::extract_version( $machine_type, PVE::QemuServer::Helpers::kvm_user_version(), ); if (min_version($machine_version, 10, 0)) { # for the switch to -blockdev my ($interface, $index) = PVE::QemuServer::Drive::parse_drive_interface($dev); my $drive = { file => $volid, interface => $interface, index => $index }; my $blockdev = PVE::QemuServer::Blockdev::generate_drive_blockdev( $storecfg, $drive, $machine_version, { 'no-throttle' => 1 }, ); $live_restore_backing->{$dev}->{blockdev} = $blockdev; } else { $live_restore_backing->{$dev}->{blockdev} = "driver=$format,node-name=drive-$dev-restore" . ",read-only=on" . ",file.driver=file,file.filename=$path"; } my $source_volid = $info->{'delete-after-finish'}; push $sources_to_remove->@*, $source_volid if defined($source_volid); } eval { # make sure HA doesn't interrupt our restore by stopping the VM if (vm_is_ha_managed($vmid)) { run_command(['ha-manager', 'set', "vm:$vmid", '--state', 'started']); } vm_start_nolock( $storecfg, $vmid, $conf, { paused => 1, 'live-restore-backing' => $live_restore_backing }, {}, ); # prevent shutdowns from qmeventd when the VM powers off from the inside my $qmeventd_fd = register_qmeventd_handle($vmid); # begin streaming, i.e. data copy from PBS to target disk for every vol, # this will effectively collapse the backing image chain consisting of # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track # removes itself once all backing images vanish with 'auto-remove=on') my $jobs = {}; for my $ds (sort keys %$live_restore_backing) { my $node_name = PVE::QemuServer::Blockdev::get_node_name_below_throttle( vm_qmp_peer($vmid), "drive-$ds", ); my $job_id = "restore-$ds"; mon_cmd( $vmid, 'block-stream', 'job-id' => $job_id, device => "$node_name", 'auto-dismiss' => JSON::false, ); $jobs->{$job_id} = {}; } mon_cmd($vmid, 'cont'); PVE::QemuServer::BlockJob::monitor( vm_qmp_peer($vmid), undef, $jobs, 'auto', 0, 'stream', ); print "restore-drive jobs finished successfully, removing all tracking block devices\n"; for my $ds (sort keys %$live_restore_backing) { PVE::QemuServer::Blockdev::detach(vm_qmp_peer($vmid), "drive-$ds-restore"); } close($qmeventd_fd); }; my $err = $@; for my $volid ($sources_to_remove->@*) { eval { PVE::Storage::vdisk_free($storecfg, $volid); print "cleaned up extracted image $volid\n"; }; warn "An error occurred while cleaning up source images: $@\n" if $@; } if ($err) { warn "An error occurred during live-restore: $err\n"; _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1); die "live-restore failed\n"; } PVE::QemuConfig->remove_lock($vmid, "import"); } sub restore_vma_archive { my ($archive, $vmid, $user, $opts, $comp) = @_; my $readfrom = $archive; my $cfg = PVE::Storage::config(); my $commands = []; my $bwlimit = $opts->{bwlimit}; my $dbg_cmdstring = ''; my $add_pipe = sub { my ($cmd) = @_; push @$commands, $cmd; $dbg_cmdstring .= ' | ' if length($dbg_cmdstring); $dbg_cmdstring .= PVE::Tools::cmd2string($cmd); $readfrom = '-'; }; my $input = undef; if ($archive eq '-') { $input = '<&STDIN'; } else { # If we use a backup from a PVE defined storage we also consider that # storage's rate limit: my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive); if (defined($volid)) { my ($sid, undef) = PVE::Storage::parse_volume_id($volid); my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit); if ($readlimit) { print STDERR "applying read rate limit: $readlimit\n"; my $cstream = ['cstream', '-t', $readlimit * 1024, '--', $readfrom]; $add_pipe->($cstream); } } } if ($comp) { my $info = PVE::Storage::decompressor_info('vma', $comp); my $cmd = $info->{decompressor}; push @$cmd, $readfrom; $add_pipe->($cmd); } my $tmpdir = "/var/tmp/vzdumptmp$$"; rmtree $tmpdir; # disable interrupts (always do cleanups) local $SIG{INT} = local $SIG{TERM} = local $SIG{QUIT} = local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; }; my $mapfifo = "/var/tmp/vzdumptmp$$.fifo"; POSIX::mkfifo($mapfifo, 0600); my $fifofh; my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! }; $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]); my $devinfo = {}; # info about drives included in backup my $virtdev_hash = {}; # info about allocated drives my $rpcenv = PVE::RPCEnvironment::get(); my $conffile = PVE::QemuConfig->config_file($vmid); # Note: $oldconf is undef if VM does not exist my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid); my $oldconf = PVE::Cluster::cfs_read_file($cfs_path); my $new_conf_raw = ''; my %storage_limits; my $print_devmap = sub { my $cfgfn = "$tmpdir/qemu-server.conf"; # we can read the config - that is already extracted my $fh = IO::File->new($cfgfn, "r") || die "unable to read qemu-server.conf - $!\n"; my $fwcfgfn = "$tmpdir/qemu-server.fw"; if (-f $fwcfgfn) { my $pve_firewall_dir = '/etc/pve/firewall'; mkdir $pve_firewall_dir; # make sure the dir exists PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw"); } $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts); foreach my $info (values %{$virtdev_hash}) { my $storeid = $info->{storeid}; next if defined($storage_limits{$storeid}); my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0; print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit; $storage_limits{$storeid} = $limit * 1024; } foreach my $devname (keys %$devinfo) { die "found no device mapping information for device '$devname'\n" if !$devinfo->{$devname}->{virtdev}; } # create empty/temp config if ($oldconf) { PVE::Tools::file_set_contents($conffile, "memory: 128\n"); $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash); } # allocate volumes my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid); # print restore information to $fifofh foreach my $virtdev (sort keys %$virtdev_hash) { my $d = $virtdev_hash->{$virtdev}; next if $d->{is_cloudinit}; # no need to restore cloudinit my $storeid = $d->{storeid}; my $volid = $d->{volid}; my $map_opts = ''; if (my $limit = $storage_limits{$storeid}) { $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:"; } my $write_zeros = 1; if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) { $write_zeros = 0; } my $path = PVE::Storage::path($cfg, $volid); print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n"; print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n"; } $fh->seek(0, 0) || die "seek failed - $!\n"; my $cookie = { netcount => 0 }; while (defined(my $line = <$fh>)) { $new_conf_raw .= restore_update_config_line( $cookie, $map, $line, $opts->{unique}, ); } $fh->close(); }; my $oldtimeout; eval { my $timeout_message = "got timeout preparing VMA restore\n"; # enable interrupts local $SIG{INT} = local $SIG{TERM} = local $SIG{QUIT} = local $SIG{HUP} = local $SIG{PIPE} = sub { die "interrupted by signal\n"; }; local $SIG{ALRM} = sub { die $timeout_message; }; $oldtimeout = alarm(60); # for reading the VMA header - might hang with a corrupted one $timeout_message = "got timeout reading VMA header - corrupted?\n"; my $parser = sub { my $line = shift; print "$line\n"; if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) { my ($dev_id, $size, $devname) = ($1, $2, $3); $devinfo->{$devname} = { size => $size, dev_id => $dev_id }; } elsif ($line =~ m/^CTIME: /) { $timeout_message = "got timeout during VMA restore\n"; # we correctly received the vma config, so we can disable # the timeout now for disk allocation alarm($oldtimeout || 0); $oldtimeout = undef; &$print_devmap(); print $fifofh "done\n"; close($fifofh); $fifofh = undef; } }; print "restore vma archive: $dbg_cmdstring\n"; run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo); }; my $err = $@; alarm($oldtimeout) if $oldtimeout; $restore_deactivate_volumes->($cfg, $virtdev_hash); close($fifofh) if $fifofh; unlink $mapfifo; rmtree $tmpdir; if ($err) { $restore_destroy_volumes->($cfg, $virtdev_hash); die $err; } my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf}); check_restore_permissions($rpcenv, $user, $new_conf); PVE::QemuConfig->write_config($vmid, $new_conf); eval { rescan($vmid, 1); }; warn $@ if $@; PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool}; } sub restore_tar_archive { my ($archive, $vmid, $user, $opts) = @_; if (scalar(keys $opts->{override_conf}->%*) > 0) { my $keystring = join(' ', keys $opts->{override_conf}->%*); die "cannot pass along options ($keystring) when restoring from tar archive\n"; } if ($archive ne '-') { my $firstfile = tar_archive_read_firstfile($archive); die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n" if $firstfile ne 'qemu-server.conf'; } my $storecfg = PVE::Storage::config(); # avoid zombie disks when restoring over an existing VM -> cleanup first # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us # skiplock=1 because qmrestore has set the 'create' lock itself already my $vmcfgfn = PVE::QemuConfig->config_file($vmid); destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn; my $tocmd = "/usr/lib/qemu-server/qmextract"; $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage}; $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool}; $tocmd .= ' --prealloc' if $opts->{prealloc}; $tocmd .= ' --info' if $opts->{info}; # tar option "xf" does not autodetect compression when read from STDIN, # so we pipe to zcat my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " . PVE::Tools::shellquote("--to-command=$tocmd"); my $tmpdir = "/var/tmp/vzdumptmp$$"; mkpath $tmpdir; local $ENV{VZDUMP_TMPDIR} = $tmpdir; local $ENV{VZDUMP_VMID} = $vmid; local $ENV{VZDUMP_USER} = $user; my $conffile = PVE::QemuConfig->config_file($vmid); my $new_conf_raw = ''; # disable interrupts (always do cleanups) local $SIG{INT} = local $SIG{TERM} = local $SIG{QUIT} = local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; }; eval { # enable interrupts local $SIG{INT} = local $SIG{TERM} = local $SIG{QUIT} = local $SIG{HUP} = local $SIG{PIPE} = sub { die "interrupted by signal\n"; }; if ($archive eq '-') { print "extracting archive from STDIN\n"; run_command($cmd, input => "<&STDIN"); } else { print "extracting archive '$archive'\n"; run_command($cmd); } return if $opts->{info}; # read new mapping my $map = {}; my $statfile = "$tmpdir/qmrestore.stat"; if (my $fd = IO::File->new($statfile, "r")) { while (defined(my $line = <$fd>)) { if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) { $map->{$1} = $2 if $1; } else { print STDERR "unable to parse line in statfile - $line\n"; } } $fd->close(); } my $confsrc = "$tmpdir/qemu-server.conf"; my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n"; my $cookie = { netcount => 0 }; while (defined(my $line = <$srcfd>)) { $new_conf_raw .= restore_update_config_line( $cookie, $map, $line, $opts->{unique}, ); } $srcfd->close(); }; if (my $err = $@) { tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info}; die $err; } rmtree $tmpdir; PVE::Tools::file_set_contents($conffile, $new_conf_raw); PVE::Cluster::cfs_update(); # make sure we read new file eval { rescan($vmid, 1); }; warn $@ if $@; } sub do_snapshots_type { my ($storecfg, $volid, $deviceid, $running) = @_; #we use storage snapshot if vm is not running or if disk is unused; return 'storage' if !$running || !$deviceid; if (my $method = PVE::Storage::volume_qemu_snapshot_method($storecfg, $volid)) { return 'internal' if $method eq 'qemu'; return 'external' if $method eq 'mixed'; } return 'storage'; } =head3 template_create($vmid, $conf [, $disk]) Converts all used disk volumes for the VM with the identifier C<$vmid> and configuration C<$conf> to base images (e.g. for VM templates). If the optional C<$disk> parameter is set, it will only convert the disk volume at the specified drive name (e.g. "scsi0"). =cut sub template_create : prototype($$;$) { my ($vmid, $conf, $disk) = @_; my $storecfg = PVE::Storage::config(); PVE::QemuConfig->foreach_volume( $conf, sub { my ($ds, $drive) = @_; return if drive_is_cdrom($drive); return if $disk && $ds ne $disk; my $volid = $drive->{file}; return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid); my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid); $drive->{file} = $voliddst; $conf->{$ds} = print_drive($drive); # write vm config on every change in case this fails on subsequent iterations PVE::QemuConfig->write_config($vmid, $conf); }, ); } # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the # source, but some storages have problems with io_uring, sometimes even leading to crashes. my sub clone_disk_check_io_uring { my ($vmid, $src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_; return if !$use_drive_mirror; # Don't complain when not changing storage. # Assume if it works for the source, it'll work for the target too. return if $src_storeid eq $dst_storeid; my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid); my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid); my $cache_direct = PVE::QemuServer::Drive::drive_uses_cache_direct($src_drive, $src_scfg); my $src_uses_io_uring; if ($src_drive->{aio}) { $src_uses_io_uring = $src_drive->{aio} eq 'io_uring'; } else { # With the switch to -blockdev and blockdev-mirror, the aio setting will be changed on the # fly if not explicitly set. my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid); return if PVE::QemuServer::Machine::is_machine_version_at_least($machine_type, 10, 0); $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct); } die "target storage is known to cause issues with aio=io_uring (used by current drive)\n" if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct); } sub clone_disk { my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_; my ($vmid, $running) = $source->@{qw(vmid running)}; my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)}; my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)}; my ($storage, $format) = $dest->@{qw(storage format)}; my $unused = defined($src_drivename) && $src_drivename =~ /^unused/; my $use_drive_mirror = $full && $running && $src_drivename && !$snapname && !$unused; if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) { die "cloning from/to EFI disk requires EFI disk\n" if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0'; die "cloning from/to TPM state requires TPM state\n" if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0'; # This would lead to two device nodes in QEMU pointing to the same backing image! die "cannot change drive name when cloning disk from/to the same VM\n" if $use_drive_mirror && $vmid == $newvmid; } die "cannot move TPM state while VM is running\n" if $use_drive_mirror && $src_drivename eq 'tpmstate0'; my $newvolid; print "create " . ($full ? 'full' : 'linked') . " clone of drive "; print "$src_drivename " if $src_drivename; print "($drive->{file})\n"; if (!$full) { $newvolid = PVE::Storage::vdisk_clone($storecfg, $drive->{file}, $newvmid, $snapname); push @$newvollist, $newvolid; } else { my ($src_storeid) = PVE::Storage::parse_volume_id($drive->{file}); my $storeid = $storage || $src_storeid; my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $drive->{file}, $format); my $name = undef; my $size = undef; if (drive_is_cloudinit($drive)) { $name = "vm-$newvmid-cloudinit"; my $scfg = PVE::Storage::storage_config($storecfg, $storeid); if ($scfg->{path}) { $name .= ".$dst_format"; } $snapname = undef; $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE; } elsif ($dst_drivename eq 'efidisk0') { $size = $efisize or die "internal error - need to specify EFI disk size\n"; } elsif ($dst_drivename eq 'tpmstate0') { $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE; } else { clone_disk_check_io_uring( $vmid, $drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror, ); $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10); } $newvolid = PVE::Storage::vdisk_alloc( $storecfg, $storeid, $newvmid, $dst_format, $name, ($size / 1024), ); push @$newvollist, $newvolid; print("allocated target volume '$newvolid'\n"); PVE::Storage::activate_volumes($storecfg, [$newvolid]); if (drive_is_cloudinit($drive)) { # when cloning multiple disks (e.g. during clone_vm) it might be the last disk # if this is the case, we have to complete any block-jobs still there from # previous drive-mirrors if (($completion && $completion eq 'complete') && (scalar(keys %$jobs) > 0)) { PVE::QemuServer::BlockJob::monitor( vm_qmp_peer($vmid), $newvmid, $jobs, $completion, $qga, ); } goto no_data_clone; } my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid); if ($use_drive_mirror) { my $source_info = { vmid => $vmid, drive => $drive }; my $dest_info = { volid => $newvolid }; $dest_info->{'zero-initialized'} = 1 if $sparseinit; $dest_info->{vmid} = $newvmid if defined($newvmid); my $mirror_opts = {}; $mirror_opts->{'guest-agent'} = 1 if $qga; $mirror_opts->{bwlimit} = $bwlimit if defined($bwlimit); PVE::QemuServer::BlockJob::mirror( $source_info, $dest_info, $jobs, $completion, $mirror_opts, ); } else { if ($dst_drivename eq 'efidisk0') { # the relevant data on the efidisk may be smaller than the source # e.g. on RBD/ZFS, so we use dd to copy only the amount # that is given by the OVMF_VARS.fd my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname); my $dst_path = PVE::Storage::path($storecfg, $newvolid); my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6]; # better for Ceph if block size is not too small, see bug #3324 my $bs = 1024 * 1024; my $cmd = ['qemu-img', 'dd', '-n', '-f', $src_format, '-O', $dst_format]; if ($src_format eq 'qcow2' && $snapname) { die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n" if !min_version(kvm_user_version(), 6, 2); my $method = PVE::Storage::volume_qemu_snapshot_method($storecfg, $drive->{file}); # in case of snapshot-as-volume-chain, $src_path points to the snapshot volume push $cmd->@*, '-l', $snapname if $method eq 'qemu'; } push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path"; run_command($cmd); } else { my $opts = { bwlimit => $bwlimit, 'is-zero-initialized' => $sparseinit, snapname => $snapname, }; PVE::QemuServer::QemuImage::convert($drive->{file}, $newvolid, $size, $opts); } } } no_data_clone: my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) }; my $disk = dclone($drive); delete $disk->{format}; $disk->{file} = $newvolid; $disk->{size} = $size if defined($size) && !$unused; return $disk; } sub get_running_qemu_version { my ($vmid) = @_; my $res = mon_cmd($vmid, "query-version"); return "$res->{qemu}->{major}.$res->{qemu}->{minor}"; } sub qemu_use_old_bios_files { my ($machine_type) = @_; return if !$machine_type; my $use_old_bios_files = undef; if ($machine_type =~ m/^(\S+)\.pxe$/) { $machine_type = $1; $use_old_bios_files = 1; } else { my $version = extract_version($machine_type, kvm_user_version()); # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we # load new efi bios files on migration. So this hack is required to allow # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when # updrading from proxmox-ve-3.X to proxmox-ve 4.0 $use_old_bios_files = !min_version($version, 2, 4); } return ($use_old_bios_files, $machine_type); } sub get_efivars_size { my ($conf, $efidisk) = @_; my $arch = PVE::QemuServer::Helpers::get_vm_arch($conf); $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef; my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf); my $cvm_type = get_cvm_type($conf); return PVE::QemuServer::OVMF::get_efivars_size($arch, $efidisk, $smm, $cvm_type); } sub update_efidisk_size { my ($conf) = @_; return if !defined($conf->{efidisk0}); my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0}); $disk->{size} = get_efivars_size($conf); $conf->{efidisk0} = print_drive($disk); return; } sub update_tpmstate_size { my ($conf) = @_; my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0}); $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE; $conf->{tpmstate0} = print_drive($disk); } sub vm_iothreads_list { my ($vmid) = @_; my $res = mon_cmd($vmid, 'query-iothreads'); my $iothreads = {}; foreach my $iothread (@$res) { $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"}; } return $iothreads; } sub resolve_dst_disk_format { my ($storecfg, $storeid, $src_volid, $format) = @_; # if no target format is specified, use the source disk format as hint if (!$format && $src_volid) { $format = checked_volume_format($storecfg, $src_volid); } # falls back to default format if requested format is not supported return PVE::Storage::resolve_format_hint($storecfg, $storeid, $format); } sub generate_uuid { my ($uuid, $uuid_str); UUID::generate($uuid); UUID::unparse($uuid, $uuid_str); return $uuid_str; } sub generate_smbios1_uuid { return "uuid=" . generate_uuid(); } sub create_reboot_request { my ($vmid) = @_; open(my $fh, '>', "/run/qemu-server/$vmid.reboot") or die "failed to create reboot trigger file: $!\n"; close($fh); } sub clear_reboot_request { my ($vmid) = @_; my $path = "/run/qemu-server/$vmid.reboot"; my $res = 0; $res = unlink($path); die "could not remove reboot request for $vmid: $!" if !$res && $! != POSIX::ENOENT; return $res; } sub bootorder_from_legacy { my ($conf, $bootcfg) = @_; my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default}; my $bootindex_hash = {}; my $i = 1; foreach my $o (split(//, $boot)) { $bootindex_hash->{$o} = $i * 100; $i++; } my $bootorder = {}; PVE::QemuConfig->foreach_volume( $conf, sub { my ($ds, $drive) = @_; if (drive_is_cdrom($drive, 1)) { if ($bootindex_hash->{d}) { $bootorder->{$ds} = $bootindex_hash->{d}; $bootindex_hash->{d} += 1; } } elsif ($bootindex_hash->{c}) { $bootorder->{$ds} = $bootindex_hash->{c} if $conf->{bootdisk} && $conf->{bootdisk} eq $ds; $bootindex_hash->{c} += 1; } }, ); if ($bootindex_hash->{n}) { for (my $i = 0; $i < $MAX_NETS; $i++) { my $netname = "net$i"; next if !$conf->{$netname}; $bootorder->{$netname} = $bootindex_hash->{n}; $bootindex_hash->{n} += 1; } } return $bootorder; } # Generate default device list for 'boot: order=' property. Matches legacy # default boot order, but with explicit device names. This is important, since # the fallback for when neither 'order' nor the old format is specified relies # on 'bootorder_from_legacy' above, and it would be confusing if this diverges. sub get_default_bootdevices { my ($conf) = @_; my @ret = (); # harddisk my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0); push @ret, $first if $first; # cdrom $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1); push @ret, $first if $first; # network for (my $i = 0; $i < $MAX_NETS; $i++) { my $netname = "net$i"; next if !$conf->{$netname}; push @ret, $netname; last; } return \@ret; } sub device_bootorder { my ($conf) = @_; return bootorder_from_legacy($conf) if !defined($conf->{boot}); my $boot = parse_property_string($boot_fmt, $conf->{boot}); my $bootorder = {}; if (!defined($boot) || $boot->{legacy}) { $bootorder = bootorder_from_legacy($conf, $boot); } elsif ($boot->{order}) { my $i = 100; # start at 100 to allow user to insert devices before us with -args for my $dev (PVE::Tools::split_list($boot->{order})) { $bootorder->{$dev} = $i++; } } return $bootorder; } sub register_qmeventd_handle { my ($vmid) = @_; my $fh; my $peer = "/var/run/qmeventd.sock"; my $count = 0; for (;;) { $count++; $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1); last if $fh; if ($! != EINTR && $! != EAGAIN) { die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n"; } if ($count > 4) { die "unable to connect to qmeventd socket (vmid: $vmid) - timeout " . "after $count retries\n"; } usleep(25000); } # send handshake to mark VM as backing up print $fh to_json({ vzdump => { vmid => "$vmid" } }); # return handle to be closed later when inhibit is no longer required return $fh; } # bash completion helper sub complete_backup_archives { my ($cmdname, $pname, $cvalue) = @_; my $cfg = PVE::Storage::config(); my $storeid; if ($cvalue =~ m/^([^:]+):/) { $storeid = $1; } my $data = PVE::Storage::template_list($cfg, $storeid, 'backup'); my $res = []; foreach my $id (keys %$data) { foreach my $item (@{ $data->{$id} }) { next if ($item->{subtype} // '') ne 'qemu'; push @$res, $item->{volid} if defined($item->{volid}); } } return $res; } my $complete_vmid_full = sub { my ($running) = @_; my $idlist = vmstatus(); my $res = []; foreach my $id (keys %$idlist) { my $d = $idlist->{$id}; if (defined($running)) { next if $d->{template}; next if $running && $d->{status} ne 'running'; next if !$running && $d->{status} eq 'running'; } push @$res, $id; } return $res; }; sub complete_vmid { return &$complete_vmid_full(); } sub complete_vmid_stopped { return &$complete_vmid_full(0); } sub complete_vmid_running { return &$complete_vmid_full(1); } sub complete_storage { my $cfg = PVE::Storage::config(); my $ids = $cfg->{ids}; my $res = []; foreach my $sid (keys %$ids) { next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1); next if !$ids->{$sid}->{content}->{images}; push @$res, $sid; } return $res; } sub complete_migration_storage { my ($cmd, $param, $current_value, $all_args) = @_; my $targetnode = @$all_args[1]; my $cfg = PVE::Storage::config(); my $ids = $cfg->{ids}; my $res = []; foreach my $sid (keys %$ids) { next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1); next if !$ids->{$sid}->{content}->{images}; push @$res, $sid; } return $res; } sub vm_is_paused { my ($vmid, $include_suspended) = @_; my $qmpstatus = eval { PVE::QemuConfig::assert_config_exists_on_node($vmid); mon_cmd($vmid, "query-status"); }; warn "$@\n" if $@; return $qmpstatus && ($qmpstatus->{status} eq "paused" || $qmpstatus->{status} eq "prelaunch" || ($include_suspended && $qmpstatus->{status} eq "suspended")); } sub check_volume_storage_type { my ($storecfg, $vol) = @_; my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol); my $scfg = PVE::Storage::storage_config($storecfg, $storeid); my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol); die "storage '$storeid' does not support content-type '$vtype'\n" if !$scfg->{content}->{$vtype}; return 1; } 1; ================================================ FILE: src/PVE/VZDump/Makefile ================================================ DESTDIR= PREFIX=/usr PERLDIR=$(PREFIX)/share/perl5 .PHONY: install install: install -D -m 0644 QemuServer.pm $(DESTDIR)$(PERLDIR)/PVE/VZDump/QemuServer.pm ================================================ FILE: src/PVE/VZDump/QemuServer.pm ================================================ package PVE::VZDump::QemuServer; use strict; use warnings; use Fcntl qw(:mode); use File::Basename; use File::Path qw(make_path remove_tree); use File::stat qw(); use IO::File; use IPC::Open3; use JSON; use POSIX qw(EINTR EAGAIN); use Time::HiRes qw(usleep); use PVE::Cluster qw(cfs_read_file); use PVE::INotify; use PVE::IPCC; use PVE::JSONSchema; use PVE::PBSClient; use PVE::RESTEnvironment qw(log_warn); use PVE::QMPClient; use PVE::Storage::Plugin; use PVE::Storage::PBSPlugin; use PVE::Storage; use PVE::Tools qw(run_command); use PVE::VZDump; use PVE::Format qw(render_duration render_bytes); use PVE::QemuConfig; use PVE::QemuServer; use PVE::QemuServer::Agent; use PVE::QemuServer::Blockdev; use PVE::QemuServer::Drive qw(checked_volume_format); use PVE::QemuServer::Helpers; use PVE::QemuServer::Machine; use PVE::QemuServer::Monitor qw(mon_cmd vm_qmp_peer); use PVE::QemuServer::QMPHelpers; use base qw (PVE::VZDump::Plugin); sub new { my ($class, $vzdump) = @_; PVE::VZDump::check_bin('qm'); my $self = bless { vzdump => $vzdump }, $class; $self->{vmlist} = PVE::QemuServer::vzlist(); $self->{storecfg} = PVE::Storage::config(); return $self; } sub type { return 'qemu'; } sub vmlist { my ($self) = @_; return [keys %{ $self->{vmlist} }]; } sub prepare { my ($self, $task, $vmid, $mode) = @_; my $running = PVE::QemuServer::Helpers::vm_running_locally($vmid); if ($running && (my $status = mon_cmd($vmid, 'query-backup'))) { if ($status->{status} && $status->{status} eq 'active') { $self->log('warn', "left-over backup job still running inside QEMU - canceling now"); mon_cmd($vmid, 'backup-cancel'); } } $task->{disks} = []; my $conf = $self->{vmlist}->{$vmid} = PVE::QemuConfig->load_config($vmid); $self->loginfo("VM Name: $conf->{name}") if defined($conf->{name}); $self->{vm_was_running} = $running ? 1 : 0; $self->{vm_was_paused} = 0; if ($running && PVE::QemuServer::vm_is_paused($vmid, 0)) { # Do not treat a suspended VM as paused, as it would cause us to skip # fs-freeze even if the VM wakes up before we reach qga_fs_freeze. $self->{vm_was_paused} = 1; } $task->{hostname} = $conf->{name}; my $hostname = PVE::INotify::nodename(); my $vollist = []; my $drivehash = {}; my $backup_volumes = PVE::QemuConfig->get_backup_volumes($conf); foreach my $volume (@{$backup_volumes}) { my $name = $volume->{key}; my $volume_config = $volume->{volume_config}; my $volid = $volume_config->{file}; if (!$volume->{included}) { $self->loginfo("exclude disk '$name' '$volid' ($volume->{reason})"); next; } else { my $log = "include disk '$name' '$volid'"; if (defined(my $size = $volume_config->{size})) { my $readable_size = PVE::JSONSchema::format_size($size); $log .= " $readable_size"; } $self->loginfo($log); } my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); push @$vollist, $volid if $storeid; $drivehash->{$name} = $volume->{volume_config}; } PVE::Storage::activate_volumes($self->{storecfg}, $vollist); foreach my $ds (sort keys %$drivehash) { my $drive = $drivehash->{$ds}; my $volid = $drive->{file}; my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); my $path = $volid; if ($storeid) { $path = PVE::Storage::path($self->{storecfg}, $volid); } next if !$path; my ($size, $format); if ($storeid) { # The call in list context can be expensive for certain plugins like RBD, just get size $size = eval { PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5) }; die "cannot determine size of volume '$volid' - $@\n" if $@; $format = checked_volume_format($self->{storecfg}, $volid); } else { ($size, $format) = eval { PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5); }; die "cannot determine size and format of volume '$volid' - $@\n" if $@; } my $diskinfo = { path => $path, volid => $volid, storeid => $storeid, size => $size, format => $format, virtdev => $ds, qmdevice => "drive-$ds", }; if ($ds eq 'tpmstate0') { # TPM drive only exists for backup, which is reflected in the name $diskinfo->{qmdevice} = 'drive-tpmstate0-backup'; $task->{'tpm-volid'} = $volid; } if (-b $path) { $diskinfo->{type} = 'block'; } else { $diskinfo->{type} = 'file'; } push @{ $task->{disks} }, $diskinfo; } } sub vm_status { my ($self, $vmid) = @_; my $running = PVE::QemuServer::check_running($vmid) ? 1 : 0; return wantarray ? ($running, $running ? 'running' : 'stopped') : $running; } sub lock_vm { my ($self, $vmid) = @_; PVE::QemuConfig->set_lock($vmid, 'backup'); } sub unlock_vm { my ($self, $vmid) = @_; PVE::QemuConfig->remove_lock($vmid, 'backup'); } sub stop_vm { my ($self, $task, $vmid) = @_; my $opts = $self->{vzdump}->{opts}; my $wait = $opts->{stopwait} * 60; # send shutdown and wait $self->cmd("qm shutdown $vmid --skiplock --keepActive --timeout $wait"); } sub start_vm { my ($self, $task, $vmid) = @_; $self->cmd("qm start $vmid --skiplock"); } sub suspend_vm { my ($self, $task, $vmid) = @_; return if $self->{vm_was_paused}; $self->cmd("qm suspend $vmid --skiplock"); } sub resume_vm { my ($self, $task, $vmid) = @_; return if $self->{vm_was_paused}; $self->cmd("qm resume $vmid --skiplock"); } sub assemble { my ($self, $task, $vmid) = @_; my $conffile = PVE::QemuConfig->config_file($vmid); my $outfile = "$task->{tmpdir}/qemu-server.conf"; my $firewall_src = "/etc/pve/firewall/$vmid.fw"; my $firewall_dest = "$task->{tmpdir}/qemu-server.fw"; my $outfd = IO::File->new(">$outfile") or die "unable to open '$outfile' - $!\n"; my $conffd = IO::File->new($conffile, 'r') or die "unable to open '$conffile' - $!\n"; my $found_snapshot; my $found_pending; my $found_special; while (defined(my $line = <$conffd>)) { next if $line =~ m/^\#vzdump\#/; # just to be sure next if $line =~ m/^\#qmdump\#/; # just to be sure if ($line =~ m/^\[(.*)\]\s*$/) { if ($1 =~ m/^PENDING$/i) { $found_pending = 1; } elsif ($1 =~ m/^special:.*$/) { $found_special = 1; } else { $found_snapshot = 1; } } # skip all snapshots, pending changes and special sections next if $found_snapshot || $found_pending || $found_special; if ($line =~ m/^unused\d+:\s*(\S+)\s*/) { $self->loginfo("skip unused drive '$1' (not included into backup)"); next; } next if $line =~ m/^lock:/ || $line =~ m/^parent:/; print $outfd $line; } foreach my $di (@{ $task->{disks} }) { if ($di->{type} eq 'block' || $di->{type} eq 'file') { my $storeid = $di->{storeid} || ''; my $format = $di->{format} || ''; print $outfd "#qmdump#map:$di->{virtdev}:$di->{qmdevice}:$storeid:$format:\n"; } else { die "internal error"; } } if ($found_special) { $self->loginfo("special config section found (not included into backup)"); } if ($found_snapshot) { $self->loginfo("snapshots found (not included into backup)"); } if ($found_pending) { $self->loginfo("pending configuration changes found (not included into backup)"); } PVE::Tools::file_copy($firewall_src, $firewall_dest) if -f $firewall_src; } sub archive { my ($self, $task, $vmid, $filename, $comp) = @_; my $opts = $self->{vzdump}->{opts}; my $scfg = $opts->{scfg}; if ($self->{vzdump}->{opts}->{pbs}) { $self->archive_pbs($task, $vmid); } elsif ($self->{vzdump}->{'backup-provider'}) { $self->archive_external($task, $vmid); } else { $self->archive_vma($task, $vmid, $filename, $comp); } } my $bitmap_action_to_human = sub { my ($self, $info) = @_; my $action = $info->{action}; if ($action eq "not-used") { return "disabled (no support)"; } elsif ($action eq "not-used-removed") { return "disabled (old bitmap cleared)"; } elsif ($action eq "new") { return "created new"; } elsif ($action eq "used") { if ($info->{dirty} == 0) { return "OK (drive clean)"; } else { my $size = render_bytes($info->{size}, 1); my $dirty = render_bytes($info->{dirty}, 1); return "OK ($dirty of $size dirty)"; } } elsif ($action eq "invalid") { return "existing bitmap was invalid and has been cleared"; } elsif ($action eq "missing-recreated") { # Lie about the TPM state, because it is newly attached each time. return "created new" if $info->{drive} eq 'drive-tpmstate0-backup'; return "expected bitmap was missing and has been recreated"; } else { return "unknown"; } }; my $query_backup_status_loop = sub { my ($self, $vmid, $job_uuid, $qemu_support) = @_; my $starttime = time(); my $last_time = $starttime; my ($last_percent, $last_total, $last_target, $last_zero, $last_transferred) = (-1, 0, 0, 0, 0); my ($transferred, $reused); my $get_mbps = sub { my ($mb, $delta) = @_; return "0 B/s" if $mb <= 0; my $bw = int(($mb / $delta)); return render_bytes($bw, 1) . "/s"; }; my $target = 0; my $last_reused = 0; my $has_query_bitmap = $qemu_support && $qemu_support->{'query-bitmap-info'}; my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); if ($has_query_bitmap) { my $total = 0; my $bitmap_info = mon_cmd($vmid, 'query-pbs-bitmap-info'); for my $info (sort { $a->{drive} cmp $b->{drive} } @$bitmap_info) { if (!$is_template) { my $text = $bitmap_action_to_human->($self, $info); my $drive = $info->{drive}; $drive =~ s/^drive-//; # for consistency $self->loginfo("$drive: dirty-bitmap status: $text"); } $target += $info->{dirty}; $total += $info->{size}; $last_reused += $info->{size} - $info->{dirty}; } if ($target < $total) { my $total_h = render_bytes($total, 1); my $target_h = render_bytes($target, 1); $self->loginfo( "using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total" ); } } my $last_finishing = 0; while (1) { my $status = mon_cmd($vmid, 'query-backup'); my $total = $status->{total} || 0; my $dirty = $status->{dirty}; $target = (defined($dirty) && $dirty < $total) ? $dirty : $total if !$has_query_bitmap; $transferred = $status->{transferred} || 0; $reused = $status->{reused}; my $percent = $target ? int(($transferred * 100) / $target) : 100; my $zero = $status->{'zero-bytes'} || 0; die "got unexpected uuid\n" if !$status->{uuid} || ($status->{uuid} ne $job_uuid); my $ctime = time(); my $duration = $ctime - $starttime; my $rbytes = $transferred - $last_transferred; my $wbytes; if ($reused) { # reused includes zero bytes for PBS $wbytes = $rbytes - ($reused - $last_reused); } else { $wbytes = $rbytes - ($zero - $last_zero); } my $timediff = ($ctime - $last_time) || 1; # fixme my $mbps_read = $get_mbps->($rbytes, $timediff); my $mbps_write = $get_mbps->($wbytes, $timediff); my $target_h = render_bytes($target, 1); my $transferred_h = render_bytes($transferred, 1); my $statusline = sprintf( "%3d%% ($transferred_h of $target_h) in %s" . ", read: $mbps_read, write: $mbps_write", $percent, render_duration($duration), ); my $res = $status->{status} || 'unknown'; if ($res ne 'active') { if ($last_percent < 100) { $self->loginfo($statusline); } if ($res ne 'done') { die(($status->{errmsg} || "unknown error") . "\n") if $res eq 'error'; die "got unexpected status '$res'\n"; } $last_target = $target if $target; $last_total = $total if $total; $last_zero = $zero if $zero; $last_transferred = $transferred if $transferred; last; } if ($percent != $last_percent && ($timediff > 2)) { $self->loginfo($statusline); $last_percent = $percent; $last_target = $target if $target; $last_total = $total if $total; $last_zero = $zero if $zero; $last_transferred = $transferred if $transferred; $last_time = $ctime; $last_reused = $reused; if (!$last_finishing && $status->{finishing}) { $self->loginfo("Waiting for server to finish backup validation..."); } $last_finishing = $status->{finishing}; } sleep(1); } my $duration = time() - $starttime; if ($last_zero) { my $zero_per = $last_target ? int(($last_zero * 100) / $last_target) : 0; my $zero_h = render_bytes($last_zero); $self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data"); } if ($reused) { my $reused_h = render_bytes($reused); my $reuse_per = int($reused * 100 / $last_total); $self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)"); } if ($transferred) { my $transferred_h = render_bytes($transferred); if ($duration) { my $mbps = $get_mbps->($transferred, $duration); $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)"); } else { $self->loginfo("transferred $transferred_h in <1 seconds"); } } return { total => $last_total, reused => $reused, }; }; my $attach_tpmstate_drive = sub { my ($self, $task, $vmid) = @_; return if !$task->{'tpm-volid'}; # unconditionally try to remove the tpmstate-named drive - it only exists # for backing up, and avoids errors if left over from some previous event eval { PVE::QemuServer::Blockdev::detach_tpm_backup_node($vmid); }; $self->loginfo('attaching TPM drive to QEMU for backup'); my $drive = { file => $task->{'tpm-volid'}, interface => 'tpmstate', index => 0 }; my $extra_options = { 'tpm-backup' => 1, 'read-only' => 1 }; PVE::QemuServer::Blockdev::attach($self->{storecfg}, $vmid, $drive, $extra_options); }; my $detach_tpmstate_drive = sub { my ($task, $vmid) = @_; return if !$task->{'tpm-volid'} || !PVE::QemuServer::Helpers::vm_running_locally($vmid); eval { PVE::QemuServer::Blockdev::detach_tpm_backup_node($vmid); }; }; my sub add_backup_performance_options { my ($qmp_param, $perf, $qemu_support) = @_; return if !$perf || scalar(keys $perf->%*) == 0; if (!$qemu_support) { my $settings_string = join(', ', sort keys $perf->%*); log_warn("ignoring setting(s): $settings_string - issue checking if supported"); return; } if (defined($perf->{'max-workers'})) { if ($qemu_support->{'backup-max-workers'}) { $qmp_param->{'max-workers'} = int($perf->{'max-workers'}); } else { log_warn("ignoring 'max-workers' setting - not supported by running QEMU"); } } } sub get_and_check_pbs_encryption_config { my ($self) = @_; my $opts = $self->{vzdump}->{opts}; my $scfg = $opts->{scfg}; my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($scfg, $opts->{storage}); my $master_keyfile = PVE::Storage::PBSPlugin::pbs_master_pubkey_file_name($scfg, $opts->{storage}); if (-e $keyfile) { if (-e $master_keyfile) { $self->loginfo("enabling encryption with master key feature"); return ($keyfile, $master_keyfile); } elsif ($scfg->{'master-pubkey'}) { die "master public key configured but no key file found\n"; } else { $self->loginfo("enabling encryption"); return ($keyfile, undef); } } else { my $encryption_fp = $scfg->{'encryption-key'}; die "encryption configured ('$encryption_fp') but no encryption key file found!\n" if $encryption_fp; if (-e $master_keyfile) { $self->log( 'warn', "backup target storage is configured with master-key, but no encryption key set!" . " Ignoring master key settings and creating unencrypted backup.", ); } return (undef, undef); } die "internal error - unhandled case for getting & checking PBS encryption ($keyfile, $master_keyfile)!"; } # Helper is intended to be called from allocate_fleecing_images() only. Otherwise, fleecing volids # have already been recorded in the configuration and PVE::QemuConfig::cleanup_fleecing_images() # should be used instead. my sub cleanup_fleecing_images { my ($self, $vmid, $disks) = @_; my $failed = []; for my $di ($disks->@*) { if (my $volid = $di->{'fleece-volid'}) { eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); }; if (my $err = $@) { $self->log('warn', "error removing fleecing image '$volid' - $err"); push $failed->@*, $volid; } } } PVE::QemuConfig::record_fleecing_images($vmid, $failed); } my sub allocate_fleecing_images { my ($self, $disks, $vmid, $fleecing_storeid, $format, $all_images) = @_; die "internal error - no fleecing storage specified\n" if !$fleecing_storeid; my $fleece_volids = []; eval { my $n = 0; # counter for fleecing image names for my $di ($disks->@*) { # EFI/TPM are usually too small to be worth it, but it's required for external providers next if !$all_images && $di->{virtdev} =~ m/^(?:tpmstate|efidisk)\d$/; if ($di->{type} eq 'block' || $di->{type} eq 'file') { my $scfg = PVE::Storage::storage_config($self->{storecfg}, $fleecing_storeid); my $name = "vm-$vmid-fleece-$n"; $name .= ".$format" if $scfg->{path}; my $size; if ($format ne 'raw') { # Since non-raw images cannot be attached with an explicit 'size' parameter to # QEMU later, pass the exact size to the storage layer. This makes qcow2 # fleecing images work for non-1KiB-aligned source images. $size = $di->{'block-node-size'} / 1024; } else { $size = PVE::Tools::convert_size($di->{'block-node-size'}, 'b' => 'kb'); } $di->{'fleece-volid'} = PVE::Storage::vdisk_alloc( $self->{storecfg}, $fleecing_storeid, $vmid, $format, $name, $size, ); push $fleece_volids->@*, $di->{'fleece-volid'}; $n++; } else { die "implement me (type '$di->{type}')"; } } }; if (my $err = $@) { cleanup_fleecing_images($self, $vmid, $disks); die $err; } PVE::QemuConfig::record_fleecing_images($vmid, $fleece_volids); } my sub detach_fleecing_images { my ($disks, $vmid) = @_; return if !PVE::QemuServer::Helpers::vm_running_locally($vmid); for my $di ($disks->@*) { if (my $volid = $di->{'fleece-volid'}) { my $node_name = "$di->{qmdevice}-fleecing"; eval { PVE::QemuServer::Blockdev::detach(vm_qmp_peer($vmid), $node_name) }; } } } my sub attach_fleecing_images { my ($self, $disks, $vmid, $format) = @_; # unconditionally try to remove potential left-overs from a previous backup detach_fleecing_images($disks, $vmid); my $vollist = [map { $_->{'fleece-volid'} } grep { $_->{'fleece-volid'} } $disks->@*]; PVE::Storage::activate_volumes($self->{storecfg}, $vollist); for my $di ($disks->@*) { if (my $volid = $di->{'fleece-volid'}) { $self->loginfo("$di->{qmdevice}: attaching fleecing image $volid to QEMU"); my ($interface, $index) = PVE::QemuServer::Drive::parse_drive_interface($di->{virtdev}); my $drive = { file => $volid, interface => $interface, index => $index, format => $format, discard => 'on', }; my $options = { 'fleecing' => 1 }; $options->{'tpm-backup'} = 1 if $interface eq 'tpmstate'; # Specify size explicitly, to make it work if storage backend rounded up size for # fleecing image when allocating. $options->{size} = $di->{'block-node-size'} if $format eq 'raw'; PVE::QemuServer::Blockdev::attach($self->{storecfg}, $vmid, $drive, $options); } } } my sub check_and_prepare_fleecing { my ($self, $vmid, $fleecing_opts, $disks, $is_template, $qemu_support, $all_images) = @_; # Even if the VM was started specifically for fleecing, it's possible that the VM is resumed and # then starts doing IO. For VMs that are not resumed the fleecing images will just stay empty, # so there is no big cost. my $use_fleecing = $fleecing_opts && $fleecing_opts->{enabled} && !$is_template; if ($use_fleecing && !$qemu_support->{'backup-fleecing'}) { $self->log( 'warn', "running QEMU version does not support backup fleecing - continuing without", ); $use_fleecing = 0; } # clean up potential left-overs from a previous attempt eval { PVE::QemuConfig::cleanup_fleecing_images( $vmid, $self->{storecfg}, sub { $self->log($_[0], $_[1]); }, ); }; $self->log('warn', "attempt to clean up left-over fleecing images failed - $@") if $@; if ($use_fleecing) { $self->query_block_node_sizes($vmid, $disks); my $format = PVE::Storage::resolve_format_hint($self->{storecfg}, $fleecing_opts->{storage}, 'qcow2'); allocate_fleecing_images( $self, $disks, $vmid, $fleecing_opts->{storage}, $format, $all_images, ); attach_fleecing_images($self, $disks, $vmid, $format); } return $use_fleecing; } sub archive_pbs { my ($self, $task, $vmid) = @_; my $conffile = "$task->{tmpdir}/qemu-server.conf"; my $firewall = "$task->{tmpdir}/qemu-server.fw"; my $opts = $self->{vzdump}->{opts}; my $scfg = $opts->{scfg}; my $starttime = time(); my $fingerprint = $scfg->{fingerprint}; my $repo = PVE::PBSClient::get_repository($scfg); my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $opts->{storage}); my ($keyfile, $master_keyfile) = $self->get_and_check_pbs_encryption_config(); my $diskcount = scalar(@{ $task->{disks} }); # proxmox-backup-client can only handle raw files and block devs, so only use it (directly) for # disk-less VMs if (!$diskcount) { $self->loginfo("backup contains no disks"); local $ENV{PBS_PASSWORD} = $password; local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint); my $cmd = [ '/usr/bin/proxmox-backup-client', 'backup', '--repository', $repo, '--backup-type', 'vm', '--backup-id', "$vmid", '--backup-time', $task->{backup_time}, ]; if (defined(my $ns = $scfg->{namespace})) { push @$cmd, '--ns', $ns; } if (defined($keyfile)) { push @$cmd, '--keyfile', $keyfile; push @$cmd, '--master-pubkey-file', $master_keyfile if defined($master_keyfile); } push @$cmd, "qemu-server.conf:$conffile"; push @$cmd, "fw.conf:$firewall" if -e $firewall; $self->loginfo("starting diskless backup"); $self->loginfo(join(' ', @$cmd)); $self->cmd($cmd); return; } # get list early so we die on unknown drive types before doing anything my $devlist = _get_task_devlist($task); my $backup_job_uuid; eval { $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub { die "interrupted by signal\n"; }; $self->enforce_vm_running_for_backup($vmid); $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") }; my $err = $@; if (!$qemu_support || $err) { die "query-proxmox-support returned empty value\n" if !$err; if ($err =~ m/The command query-proxmox-support has not been found/) { die "PBS backups are not supported by the running QEMU version. Please make " . "sure you've installed the latest version and the VM has been restarted.\n"; } else { die "QMP command query-proxmox-support failed - $err\n"; } } # pve-qemu supports it since 5.2.0-1 (PVE 6.4), so safe to die since PVE 8 die "master key configured but running QEMU version does not support master keys\n" if !$qemu_support->{'pbs-masterkey'} && defined($master_keyfile); $attach_tpmstate_drive->($self, $task, $vmid); my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); $task->{'use-fleecing'} = check_and_prepare_fleecing( $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support, 0, ); my $fs_frozen = $self->qga_fs_freeze($task, $vmid); my $params = { format => "pbs", 'backup-file' => $repo, 'backup-id' => "$vmid", 'backup-time' => $task->{backup_time}, password => $password, devlist => $devlist, 'config-file' => $conffile, }; $params->{fleecing} = JSON::true if $task->{'use-fleecing'}; if (defined(my $ns = $scfg->{namespace})) { $params->{'backup-ns'} = $ns; } $params->{speed} = $opts->{bwlimit} * 1024 if $opts->{bwlimit}; add_backup_performance_options($params, $opts->{performance}, $qemu_support); $params->{fingerprint} = $fingerprint if defined($fingerprint); $params->{'firewall-file'} = $firewall if -e $firewall; $params->{encrypt} = defined($keyfile) ? JSON::true : JSON::false; if (defined($keyfile)) { $params->{keyfile} = $keyfile; $params->{"master-keyfile"} = $master_keyfile if defined($master_keyfile); } $params->{'use-dirty-bitmap'} = JSON::true if $qemu_support->{'pbs-dirty-bitmap'} && !$is_template; $params->{timeout} = 125; # give some time to connect to the backup server $self->loginfo("starting backup via QMP command"); my $res = eval { mon_cmd($vmid, "backup", %$params) }; my $qmperr = $@; $backup_job_uuid = $res->{UUID} if $res; if ($fs_frozen) { $self->qga_fs_thaw($vmid); } die $qmperr if $qmperr; die "got no uuid for backup task\n" if !defined($backup_job_uuid); $self->loginfo("started backup task '$backup_job_uuid'"); $self->resume_vm_after_job_start($task, $vmid); my $stat = $query_backup_status_loop->($self, $vmid, $backup_job_uuid, $qemu_support); $task->{size} = $stat->{total}; }; my $err = $@; if ($err) { $self->logerr($err); $self->mon_backup_cancel($vmid); $self->resume_vm_after_job_start($task, $vmid); } $self->restore_vm_power_state($vmid); die $err if $err; } my $fork_compressor_pipe = sub { my ($self, $comp, $outfileno) = @_; my @pipefd = POSIX::pipe(); my $cpid = fork(); die "unable to fork worker - $!" if !defined($cpid) || $cpid < 0; if ($cpid == 0) { eval { POSIX::close($pipefd[1]); # redirect STDIN my $fd = fileno(STDIN); close STDIN; POSIX::close(0) if $fd != 0; die "unable to redirect STDIN - $!" if !open(STDIN, "<&", $pipefd[0]); # redirect STDOUT $fd = fileno(STDOUT); close STDOUT; POSIX::close(1) if $fd != 1; die "unable to redirect STDOUT - $!" if !open(STDOUT, ">&", $outfileno); exec($comp); die "fork compressor '$comp' failed\n"; }; if (my $err = $@) { $self->logerr($err); POSIX::_exit(1); } POSIX::_exit(0); kill(-9, $$); } else { POSIX::close($pipefd[0]); $outfileno = $pipefd[1]; } return ($cpid, $outfileno); }; sub archive_vma { my ($self, $task, $vmid, $filename, $comp) = @_; my $conffile = "$task->{tmpdir}/qemu-server.conf"; my $firewall = "$task->{tmpdir}/qemu-server.fw"; my $opts = $self->{vzdump}->{opts}; my $starttime = time(); my $speed = 0; if ($opts->{bwlimit}) { $speed = $opts->{bwlimit} * 1024; } my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); my $diskcount = scalar(@{ $task->{disks} }); if (!$diskcount) { $self->loginfo("backup contains no disks"); my $outcmd; if ($comp) { $outcmd = "exec:$comp"; } else { $outcmd = "exec:cat"; } $outcmd .= " > $filename" if !$opts->{stdout}; my $cmd = ['/usr/bin/vma', 'create', '-v', '-c', $conffile]; push @$cmd, '-c', $firewall if -e $firewall; push @$cmd, $outcmd; $self->loginfo("starting diskless backup"); $self->loginfo(join(' ', @$cmd)); if ($opts->{stdout}) { $self->cmd($cmd, output => ">&" . fileno($opts->{stdout})); } else { $self->cmd($cmd); } return; } my $devlist = _get_task_devlist($task); my $cpid; my $backup_job_uuid; eval { $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub { die "interrupted by signal\n"; }; $self->enforce_vm_running_for_backup($vmid); $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); # Currently, failing to determine Proxmox support is not critical here, because it's only # used for performance settings like 'max-workers'. my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") }; log_warn($@) if $@; $attach_tpmstate_drive->($self, $task, $vmid); $task->{'use-fleecing'} = check_and_prepare_fleecing( $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support, 0, ); my $outfh; if ($opts->{stdout}) { $outfh = $opts->{stdout}; } else { $outfh = IO::File->new($filename, "w") || die "unable to open file '$filename' - $!\n"; } my $outfileno = fileno($outfh); if ($comp) { ($cpid, $outfileno) = $fork_compressor_pipe->($self, $comp, $outfileno); } my $qmpclient = PVE::QMPClient->new(); my $qmp_peer = vm_qmp_peer($vmid); my $backup_cb = sub { my ($vmid, $resp) = @_; $backup_job_uuid = $resp->{return}->{UUID}; }; my $add_fd_cb = sub { my ($vmid, $resp) = @_; my $params = { 'backup-file' => "/dev/fdname/backup", speed => $speed, 'config-file' => $conffile, devlist => $devlist, }; $params->{'firewall-file'} = $firewall if -e $firewall; $params->{fleecing} = JSON::true if $task->{'use-fleecing'}; add_backup_performance_options($params, $opts->{performance}, $qemu_support); $qmpclient->queue_cmd($qmp_peer, $backup_cb, 'backup', %$params); }; $qmpclient->queue_cmd( $qmp_peer, $add_fd_cb, 'getfd', fd => $outfileno, fdname => "backup", ); my $fs_frozen = $self->qga_fs_freeze($task, $vmid); $self->loginfo("starting backup via QMP command"); eval { $qmpclient->queue_execute(30) }; my $qmperr = $@; if ($fs_frozen) { $self->qga_fs_thaw($vmid); } die "executing QMP command to start backup failed - $qmperr" if $qmperr; die $qmpclient->{errors}->{$vmid} if $qmpclient->{errors}->{$vmid}; if ($cpid) { POSIX::close($outfileno) == 0 || die "close output file handle failed\n"; } die "got no uuid for backup task\n" if !defined($backup_job_uuid); $self->loginfo("started backup task '$backup_job_uuid'"); $self->resume_vm_after_job_start($task, $vmid); $query_backup_status_loop->($self, $vmid, $backup_job_uuid); }; my $err = $@; if ($err) { $self->logerr($err); $self->mon_backup_cancel($vmid); $self->resume_vm_after_job_start($task, $vmid); } $self->restore_vm_power_state($vmid); if ($err) { if ($cpid) { kill(9, $cpid); waitpid($cpid, 0); } die $err; } if ($cpid && (waitpid($cpid, 0) > 0)) { my $stat = $?; my $ec = $stat >> 8; my $signal = $stat & 127; if ($ec || $signal) { die "$comp failed - wrong exit status $ec" . ($signal ? " (signal $signal)\n" : "\n"); } } } sub _get_task_devlist { my ($task) = @_; my $devlist = ''; foreach my $di (@{ $task->{disks} }) { if ($di->{type} eq 'block' || $di->{type} eq 'file') { $devlist .= ',' if $devlist; $devlist .= $di->{qmdevice}; } else { die "implement me (type '$di->{type}')"; } } return $devlist; } sub qga_fs_freeze { my ($self, $task, $vmid) = @_; return if $task->{mode} eq 'stop' || !$self->{vm_was_running} || $self->{vm_was_paused}; my $conf = $self->{vmlist}->{$vmid}; if (!PVE::QemuServer::Agent::get_qga_key($conf, 'enabled')) { $self->loginfo("skipping guest-agent 'fs-freeze', agent not configured in VM options"); return; } if (!PVE::QemuServer::Agent::qga_check_running($vmid, 1)) { $self->loginfo("skipping guest-agent 'fs-freeze', agent configured but not running?"); return; } if (!PVE::QemuServer::Agent::should_fs_freeze($conf)) { $self->loginfo("skipping guest-agent 'fs-freeze', disabled in VM options"); return; } $self->loginfo("issuing guest-agent 'fs-freeze' command"); eval { PVE::QemuServer::Agent::guest_fsfreeze($vmid); }; $self->logerr($@) if $@; return 1; # even on error, ensure we always thaw again } # only call if fs_freeze return 1 sub qga_fs_thaw { my ($self, $vmid) = @_; $self->loginfo("issuing guest-agent 'fs-thaw' command"); eval { PVE::QemuServer::Agent::guest_fsthaw($vmid); }; $self->logerr($@) if $@; } # The size for fleecing images needs to be exactly the same size as QEMU sees. E.g. EFI disk can bex # attached with a smaller size then the underyling image on the storage. sub query_block_node_sizes { my ($self, $vmid, $disks) = @_; my $block_info = PVE::QemuServer::Blockdev::get_block_info($vmid); for my $diskinfo ($disks->@*) { my $drive_key = $diskinfo->{virtdev}; my $block_node_size; if ($drive_key eq 'tpmstate0') { # There is no front-end device for TPM state, so it's not included in the result of # get_block_info(). Note that it is always attached with the same explicit node name. my $named_block_node_info = mon_cmd($vmid, 'query-named-block-nodes'); for my $info ($named_block_node_info->@*) { next if $info->{'node-name'} ne 'drive-tpmstate0-backup'; $block_node_size = $info->{image}->{'virtual-size'}; last; } } else { $block_node_size = eval { $block_info->{$drive_key}->{inserted}->{image}->{'virtual-size'}; }; } if (!$block_node_size) { $self->loginfo( "could not determine block node size of drive '$drive_key' - using fallback"); $block_node_size = $diskinfo->{size} or die "could not determine size of drive '$drive_key'\n"; } $diskinfo->{'block-node-size'} = $block_node_size; } return; } # we need a running QEMU/KVM process for backup, starts a paused (prelaunch) # one if VM isn't already running sub enforce_vm_running_for_backup { my ($self, $vmid) = @_; if (PVE::QemuServer::check_running($vmid)) { $self->{vm_was_running} = 1; return; } eval { $self->loginfo("starting kvm to execute backup task"); # start with skiplock my $params = { skiplock => 1, skiptemplate => 1, paused => 1, }; PVE::QemuServer::vm_start($self->{storecfg}, $vmid, $params); }; die $@ if $@; } # resume VM again once in a clear state (stop mode backup of running VM) sub resume_vm_after_job_start { my ($self, $task, $vmid) = @_; return if !$self->{vm_was_running} || $self->{vm_was_paused}; if (my $stoptime = $task->{vmstoptime}) { my $delay = time() - $task->{vmstoptime}; $task->{vmstoptime} = undef; # avoid printing 'online after ..' twice $self->loginfo("resuming VM again after $delay seconds"); } else { $self->loginfo("resuming VM again"); } mon_cmd($vmid, 'cont', timeout => 45); } # stop again if VM was not running before sub restore_vm_power_state { my ($self, $vmid) = @_; # we always let VMs keep running return if $self->{vm_was_running}; eval { my $resp = mon_cmd($vmid, 'query-status'); my $status = $resp && $resp->{status} ? $resp->{status} : 'unknown'; if ($status eq 'prelaunch') { $self->loginfo("stopping kvm after backup task"); PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1); } else { $self->loginfo("kvm status changed after backup ('$status') - keep VM running"); } }; warn $@ if $@; } sub mon_backup_cancel { my ($self, $vmid) = @_; $self->loginfo("aborting backup job"); eval { mon_cmd($vmid, 'backup-cancel') }; $self->logerr($@) if $@; } sub snapshot { my ($self, $task, $vmid) = @_; # nothing to do } my sub cleanup_file_handles { my ($self, $file_handles) = @_; for my $file_handle ($file_handles->@*) { close($file_handle) or $self->log('warn', "unable to close file handle - $!"); } } my sub cleanup_nbd_mounts { my ($self, $info) = @_; for my $mount_point (keys $info->%*) { my $pid_file = delete($info->{$mount_point}->{'pid-file'}); unlink($pid_file) or $self->log('warn', "unable to unlink '$pid_file' - $!"); # Do a lazy unmount, because the target might still be busy even if the file handle was # already closed. eval { run_command(['fusermount', '-z', '-u', $mount_point]); }; if (my $err = $@) { delete $info->{$mount_point}; $self->log('warn', "unable to unmount NBD backup source '$mount_point' - $err"); } } # Wait for the unmount before cleaning up child PIDs to avoid 'nbdfuse' processes being # interrupted by the signals issued there. my $waited; my $wait_limit = 50; # 5 seconds for ($waited = 0; $waited < $wait_limit && scalar(keys $info->%*); $waited++) { for my $mount_point (keys $info->%*) { delete($info->{$mount_point}) if !-e $info->{$mount_point}->{'virtual-file'}; eval { remove_tree($mount_point); }; } usleep(100_000); } # just informational, remaining child processes will be killed afterwards $self->loginfo("unable to gracefully cleanup NBD fuse mounts") if scalar(keys $info->%*) != 0; } my sub cleanup_child_processes { my ($self, $cpids) = @_; my $waited; my $wait_limit = 5; for ($waited = 0; $waited < $wait_limit && scalar(keys $cpids->%*); $waited++) { for my $cpid (keys $cpids->%*) { delete($cpids->{$cpid}) if waitpid($cpid, POSIX::WNOHANG) > 0; } if ($waited == 0) { kill 15, $_ for keys $cpids->%*; } sleep 1; } if ($waited == $wait_limit && scalar(keys $cpids->%*)) { kill 9, $_ for keys $cpids->%*; sleep 1; for my $cpid (keys $cpids->%*) { delete($cpids->{$cpid}) if waitpid($cpid, POSIX::WNOHANG) > 0; } $self->log('warn', "unable to collect child process '$_'") for keys $cpids->%*; } } sub cleanup { my ($self, $task, $vmid) = @_; # If VM was started only for backup, it is already stopped now. if (PVE::QemuServer::Helpers::vm_running_locally($vmid)) { if ($task->{cleanup}->{'nbd-stop'}) { eval { PVE::QemuServer::QMPHelpers::nbd_stop($vmid); }; $self->logerr($@) if $@; } if (my $info = $task->{cleanup}->{'backup-access-teardown'}) { my $params = { 'target-id' => $info->{'target-id'}, timeout => 60, success => $info->{success} ? JSON::true : JSON::false, }; $self->loginfo("tearing down backup-access"); eval { mon_cmd($vmid, "backup-access-teardown", $params->%*) }; $self->logerr($@) if $@; } $detach_tpmstate_drive->($task, $vmid); } if ($task->{'use-fleecing'}) { eval { detach_fleecing_images($task->{disks}, $vmid); PVE::QemuConfig::cleanup_fleecing_images( $vmid, $self->{storecfg}, sub { $self->log($_[0], $_[1]); }, ); }; $self->log('warn', "attempt to clean up fleecing images failed - $@") if $@; } if ($self->{qmeventd_fh}) { close($self->{qmeventd_fh}); } cleanup_file_handles($self, $task->{cleanup}->{'file-handles'}) if $task->{cleanup}->{'file-handles'}; cleanup_nbd_mounts($self, $task->{cleanup}->{'nbd-mounts'}) if $task->{cleanup}->{'nbd-mounts'}; cleanup_child_processes($self, $task->{cleanup}->{'child-pids'}) if $task->{cleanup}->{'child-pids'}; if (my $dir = $task->{'backup-access-root-dir'}) { eval { remove_tree($dir) }; $self->log('warn', "unable to cleanup directory $dir - $@") if $@; } } my sub virtual_file_backup_prepare { my ($self, $vmid, $task, $device_name, $size, $nbd_path, $bitmap_name) = @_; my $cleanup = $task->{cleanup}; my $nbd_uri = "nbd+unix:///${device_name}?socket=${nbd_path}"; my $error_fh; my $next_dirty_region; # If there is no dirty bitmap, it can be treated as if there's a full dirty one. The output of # nbdinfo is a list of tuples with offset, length, type, description. The first bit of 'type' is # set when the bitmap is dirty, see QEMU's docs/interop/nbd.txt my $dirty_bitmap = []; if ($bitmap_name) { my $input = IO::File->new(); my $info = IO::File->new(); $error_fh = IO::File->new(); my $nbdinfo_cmd = ["nbdinfo", $nbd_uri, "--map=qemu:dirty-bitmap:${bitmap_name}"]; my $cpid = open3($input, $info, $error_fh, $nbdinfo_cmd->@*) or die "failed to spawn nbdinfo child - $!\n"; $cleanup->{'child-pids'}->{$cpid} = 1; $next_dirty_region = sub { my ($offset, $length, $type); do { my $line = <$info>; return if !$line; die "unexpected output from nbdinfo - $line\n" if $line !~ m/^\s*(\d+)\s*(\d+)\s*(\d+)/; # also untaints ($offset, $length, $type) = ($1, $2, $3); } while (($type & 0x1) == 0); # not dirty return ($offset, $length); }; } else { my $done = 0; $next_dirty_region = sub { return if $done; $done = 1; return (0, $size); }; } my $mount_point = $task->{'backup-access-root-dir'} . "/${vmid}-nbd.backup-access.${device_name}.$$"; make_path($mount_point) or die "unable to create directory $mount_point\n"; $cleanup->{'nbd-mounts'}->{$mount_point} = {}; # Note that nbdfuse requires "$dir/$file". A single name would be treated as a dir and the file # would be named "$dir/nbd" then my $virtual_file = "${mount_point}/${device_name}"; $cleanup->{'nbd-mounts'}->{$mount_point}->{'virtual-file'} = $virtual_file; my $pid_file = "${mount_point}.pid"; PVE::Tools::file_set_contents($pid_file, '', 0600); $cleanup->{'nbd-mounts'}->{$mount_point}->{'pid-file'} = $pid_file; my $cpid = fork() // die "fork failed: $!\n"; if (!$cpid) { # By default, access will be restricted to the current user, because the allow_other fuse # mount option is not used. eval { run_command( ["nbdfuse", '--pidfile', $pid_file, $virtual_file, $nbd_uri], logfunc => sub { $self->loginfo("nbdfuse '$virtual_file': $_[0]") }, ); }; if (my $err = $@) { eval { $self->loginfo($err); }; POSIX::_exit(1); } POSIX::_exit(0); } $cleanup->{'child-pids'}->{$cpid} = 1; my ($virtual_file_ready, $waited) = (0, 0); while (!$virtual_file_ready && $waited < 30) { # 3 seconds my $pid = PVE::Tools::file_read_firstline($pid_file); if ($pid) { $virtual_file_ready = 1; } else { usleep(100_000); $waited++; } } die "timeout setting up virtual file '$virtual_file'" if !$virtual_file_ready; $self->loginfo("provided NBD export as a virtual file '$virtual_file'"); # NOTE O_DIRECT, because each block should be read exactly once and also because fuse will try # to read ahead otherwise, which would produce warning messages if the next block is not # mapped/allocated for the NBD export in case of incremental backup. Open as writable to support # discard. my $fh = IO::File->new($virtual_file, O_RDWR | O_DIRECT) or die "unable to open backup source '$virtual_file' - $!\n"; push $cleanup->{'file-handles'}->@*, $fh; return ($fh, $next_dirty_region); } my sub backup_access_to_volume_info { my ($self, $vmid, $task, $backup_access_info, $mechanism, $nbd_path) = @_; my $bitmap_action_to_status = { 'not-used' => 'none', 'not-used-removed' => 'none', 'new' => 'new', 'used' => 'reuse', 'invalid' => 'new', 'missing-recreated' => 'new', }; my $volumes = {}; for my $info ($backup_access_info->@*) { my $bitmap_status = 'none'; my $bitmap_name; if (my $bitmap_action = $info->{'bitmap-action'}) { $bitmap_status = $bitmap_action_to_status->{$bitmap_action} or die "got unexpected bitmap action '$bitmap_action'\n"; $bitmap_name = $info->{'bitmap-name'} or die "bitmap-name is not present\n"; } my ($device, $size) = $info->@{qw(device size)}; $volumes->{$device}->{'bitmap-mode'} = $bitmap_status; $volumes->{$device}->{size} = $size; if ($mechanism eq 'file-handle') { my ($fh, $next_dirty_region) = virtual_file_backup_prepare( $self, $vmid, $task, $device, $size, $nbd_path, $bitmap_name, ); $volumes->{$device}->{'file-handle'} = $fh; $volumes->{$device}->{'next-dirty-region'} = $next_dirty_region; } elsif ($mechanism eq 'nbd') { $volumes->{$device}->{'nbd-path'} = $nbd_path; $volumes->{$device}->{'bitmap-name'} = $bitmap_name; } else { die "internal error - unknown mechanism '$mechanism'"; } } return $volumes; } sub archive_external { my ($self, $task, $vmid) = @_; $task->{'backup-access-root-dir'} = "/run/qemu-server/${vmid}.backup-access.$$/"; make_path($task->{'backup-access-root-dir'}) or die "unable to create directory $task->{'backup-access-root-dir'}\n"; chmod(0700, $task->{'backup-access-root-dir'}) or die "unable to chmod directory $task->{'backup-access-root-dir'}\n"; my $guest_config = PVE::Tools::file_get_contents("$task->{tmpdir}/qemu-server.conf"); my $firewall_file = "$task->{tmpdir}/qemu-server.fw"; my $opts = $self->{vzdump}->{opts}; my $backup_provider = $self->{vzdump}->{'backup-provider'}; $self->loginfo("starting external backup via " . $backup_provider->provider_name()); my $starttime = time(); $self->enforce_vm_running_for_backup($vmid); $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); eval { $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub { die "interrupted by signal\n"; }; my $qemu_support = mon_cmd($vmid, "query-proxmox-support"); if (!$qemu_support->{'backup-access-api'}) { die "backups access API required for external provider backup is not supported by" . " the running QEMU version. Please make sure you've installed the latest " . " version and the VM has been restarted.\n"; } $attach_tpmstate_drive->($self, $task, $vmid); my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); my $fleecing = check_and_prepare_fleecing( $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support, 1, ); die "cannot setup backup access without fleecing\n" if !$fleecing; $task->{'use-fleecing'} = 1; my $target_id = "snapshot-access:$opts->{storage}"; my $mechanism = $backup_provider->backup_get_mechanism($vmid, 'qemu'); die "mechanism '$mechanism' requested by backup provider is not supported for VMs\n" if $mechanism ne 'file-handle' && $mechanism ne 'nbd'; $self->loginfo("using backup mechanism '$mechanism'"); if ($mechanism eq 'file-handle') { # For mechanism 'file-handle', the nbdfuse binary is required. Also, the bitmap needs # to be passed to the provider. The bitmap cannot be dumped via QMP and doing it via # qemu-img is experimental, so use nbdinfo. Both are in libnbd-bin. die "need 'nbdfuse' binary from package libnbd-bin\n" if !-e "/usr/bin/nbdfuse"; } my $devices = {}; for my $di ($task->{disks}->@*) { my $device_name = $di->{qmdevice}; die "implement me (type '$di->{type}')" if $di->{type} ne 'block' && $di->{type} ne 'file'; $devices->{$device_name}->{size} = $di->{'block-node-size'}; } my $incremental_info = $backup_provider->backup_vm_query_incremental($vmid, $devices); my $qmp_devices = []; for my $device (sort keys $devices->%*) { my $qmp_device = { device => $device }; if (defined(my $mode = $incremental_info->{$device})) { if ($mode eq 'new' || $mode eq 'use' || $mode eq 'none') { $qmp_device->{'bitmap-mode'} = $mode; } else { die "invalid incremental mode '$mode' returned by backup provider plugin\n"; } } push($qmp_devices->@*, $qmp_device); } my $params = { 'target-id' => $target_id, devices => $qmp_devices, timeout => 60, }; my $fs_frozen = $self->qga_fs_freeze($task, $vmid); $self->loginfo("setting up snapshot-access for backup"); $task->{cleanup}->{'backup-access-teardown'} = { 'target-id' => $target_id, success => 0 }; my $backup_access_info = eval { mon_cmd($vmid, "backup-access-setup", $params->%*) }; my $qmperr = $@; if ($fs_frozen) { $self->qga_fs_thaw($vmid); } die $qmperr if $qmperr; $self->resume_vm_after_job_start($task, $vmid); my $bitmap_info = mon_cmd($vmid, 'query-pbs-bitmap-info'); for my $info (sort { $a->{drive} cmp $b->{drive} } $bitmap_info->@*) { my $text = $bitmap_action_to_human->($self, $info); my $drive = $info->{drive}; $drive =~ s/^drive-//; # for consistency $self->loginfo("$drive: dirty-bitmap status: $text"); } $self->loginfo("starting NBD server"); my $nbd_path = "$task->{'backup-access-root-dir'}/${vmid}-nbd.backup-access"; mon_cmd( $vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $nbd_path } }, ); $task->{cleanup}->{'nbd-stop'} = 1; for my $info ($backup_access_info->@*) { $self->loginfo("adding NBD export for $info->{device}"); my $export_params = { id => $info->{device}, 'node-name' => $info->{'node-name'}, writable => JSON::true, # for discard type => "nbd", name => $info->{device}, # NBD export name }; if ($info->{'bitmap-name'}) { $export_params->{bitmaps} = [{ node => $info->{'bitmap-node-name'}, name => $info->{'bitmap-name'}, }]; } mon_cmd($vmid, "block-export-add", $export_params->%*); } my $volumes = backup_access_to_volume_info( $self, $vmid, $task, $backup_access_info, $mechanism, $nbd_path, ); my $param = {}; $param->{'bandwidth-limit'} = $opts->{bwlimit} * 1024 if $opts->{bwlimit}; $param->{'firewall-config'} = PVE::Tools::file_get_contents($firewall_file) if -e $firewall_file; $backup_provider->backup_vm($vmid, $guest_config, $volumes, $param); }; my $err = $@; if ($err) { $self->logerr($err); $self->resume_vm_after_job_start($task, $vmid); } else { $task->{cleanup}->{'backup-access-teardown'}->{success} = 1; } $self->restore_vm_power_state($vmid); die $err if $err; } 1; ================================================ FILE: src/bin/Makefile ================================================ PACKAGE ?= qemu-server DESTDIR= PREFIX=/usr SBINDIR=$(PREFIX)/sbin LIBDIR=$(PREFIX)/lib/$(PACKAGE) MANDIR=$(PREFIX)/share/man MAN1DIR=$(MANDIR)/man1/ MAN5DIR=$(MANDIR)/man5/ BASHCOMPLDIR=$(PREFIX)/share/bash-completion/completions/ ZSHCOMPLDIR=$(PREFIX)/share/zsh/vendor-completions/ PERL_DOC_INC_DIRS=.. -include /usr/share/pve-doc-generator/pve-doc-generator.mk all: qm.bash-completion: PVE_GENERATING_DOCS=1 perl -I.. -T -e "use PVE::CLI::qm; PVE::CLI::qm->generate_bash_completions();" >$@.tmp mv $@.tmp $@ qmrestore.bash-completion: PVE_GENERATING_DOCS=1 perl -I.. -T -e "use PVE::CLI::qmrestore; PVE::CLI::qmrestore->generate_bash_completions();" >$@.tmp mv $@.tmp $@ qm.zsh-completion: PVE_GENERATING_DOCS=1 perl -I.. -T -e "use PVE::CLI::qm; PVE::CLI::qm->generate_zsh_completions();" >$@.tmp mv $@.tmp $@ qmrestore.zsh-completion: PVE_GENERATING_DOCS=1 perl -I.. -T -e "use PVE::CLI::qmrestore; PVE::CLI::qmrestore->generate_zsh_completions();" >$@.tmp mv $@.tmp $@ PKGSOURCES=qm qm.1 qmrestore qmrestore.1 qmextract qm.conf.5 qm.bash-completion qmrestore.bash-completion \ qm.zsh-completion qmrestore.zsh-completion cpu-models.conf.5 .PHONY: install install: $(PKGSOURCES) install -d $(DESTDIR)/$(SBINDIR) install -d $(DESTDIR)$(LIBDIR) install -d $(DESTDIR)/$(MAN1DIR) install -d $(DESTDIR)/$(MAN5DIR) install -d $(DESTDIR)/usr/share/$(PACKAGE) install -m 0644 -D qm.bash-completion $(DESTDIR)/$(BASHCOMPLDIR)/qm install -m 0644 -D qmrestore.bash-completion $(DESTDIR)/$(BASHCOMPLDIR)/qmrestore install -m 0644 -D qm.zsh-completion $(DESTDIR)/$(ZSHCOMPLDIR)/_qm install -m 0644 -D qmrestore.zsh-completion $(DESTDIR)/$(ZSHCOMPLDIR)/_qmrestore install -m 0755 qm $(DESTDIR)$(SBINDIR) install -m 0755 qmrestore $(DESTDIR)$(SBINDIR) install -m 0755 qmextract $(DESTDIR)$(LIBDIR) install -m 0644 qm.1 $(DESTDIR)/$(MAN1DIR) install -m 0644 qmrestore.1 $(DESTDIR)/$(MAN1DIR) install -m 0644 cpu-models.conf.5 $(DESTDIR)/$(MAN5DIR) install -m 0644 qm.conf.5 $(DESTDIR)/$(MAN5DIR) cd $(DESTDIR)/$(MAN5DIR); ln -s -f qm.conf.5.gz vm.conf.5.gz .PHONY: test test: PVE_GENERATING_DOCS=1 perl -I.. ./qm verifyapi .PHONY: clean clean: $(MAKE) cleanup-docgen .PHONY: distclean distclean: clean ================================================ FILE: src/bin/qm ================================================ #!/usr/bin/perl use strict; use warnings; use PVE::CLI::qm; PVE::CLI::qm->run_cli_handler(); ================================================ FILE: src/bin/qmextract ================================================ #!/usr/bin/perl use strict; use warnings; use Getopt::Long; use File::Path; use IO::File; use PVE::INotify; use PVE::JSONSchema; use PVE::Tools; use PVE::Cluster; use PVE::RPCEnvironment; use PVE::Storage; use PVE::QemuServer; $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; die "please run as root\n" if $> != 0; my @std_opts = ('storage=s', 'pool=s', 'info', 'prealloc'); sub print_usage { print STDERR "usage: $0 [--storage=] [--pool=] [--info] [--prealloc] \n\n"; } my $opts = {}; if (!GetOptions($opts, @std_opts)) { print_usage(); exit(-1); } PVE::INotify::inotify_init(); my $rpcenv = PVE::RPCEnvironment->init('cli'); $rpcenv->init_request(); $rpcenv->set_language($ENV{LANG}); $rpcenv->set_user('root@pam'); sub extract_archive { # NOTE: this is run as tar subprocess (--to-command) $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub { die "interrupted by signal\n"; }; my $filename = $ENV{TAR_FILENAME}; die "got strange environment - no TAR_FILENAME\n" if !$filename; my $filesize = $ENV{TAR_SIZE}; die "got strange file size '$filesize'\n" if !$filesize; my $tmpdir = $ENV{VZDUMP_TMPDIR}; die "got strange environment - no VZDUMP_TMPDIR\n" if !$tmpdir; my $filetype = $ENV{TAR_FILETYPE} || 'none'; die "got strange filetype '$filetype'\n" if $filetype ne 'f'; my $vmid = $ENV{VZDUMP_VMID}; PVE::JSONSchema::pve_verify_vmid($vmid); my $user = $ENV{VZDUMP_USER}; $rpcenv->check_user_enabled($user); if ($opts->{info}) { print STDERR "reading archive member '$filename'\n"; } else { print STDERR "extracting '$filename' from archive\n"; } my $conffile = "$tmpdir/qemu-server.conf"; my $statfile = "$tmpdir/qmrestore.stat"; if ($filename eq 'qemu-server.conf') { my $outfd = IO::File->new($conffile, "w") || die "unable to write file '$conffile'\n"; while (defined(my $line = <>)) { print $outfd $line; print STDERR "CONFIG: $line" if $opts->{info}; } $outfd->close(); exit(0); } if ($opts->{info}) { exec 'dd', 'bs=256K', "of=/dev/null"; die "couldn't exec dd: $!\n"; } my $conffd = IO::File->new($conffile, "r") || die "unable to read file '$conffile'\n"; my $map; while (defined(my $line = <$conffd>)) { if ($line =~ m/^\#vzdump\#map:(\S+):(\S+):(\d+):(\S*):$/) { $map->{$2} = { virtdev => $1, size => $3, storeid => $4 }; } } close($conffd); my $statfd = IO::File->new($statfile, "a") || die "unable to open file '$statfile'\n"; if ($filename !~ m/^.*\.([^\.]+)$/) { die "got strange filename '$filename'\n"; } my $format = $1; my $path; if (!$map) { print STDERR "restoring old style vzdump archive - " . "no device map inside archive\n"; die "can't restore old style archive to storage '$opts->{storage}'\n" if defined($opts->{storage}) && $opts->{storage} ne 'local'; my $dir = "/var/lib/vz/images/$vmid"; mkpath $dir; $path = "$dir/$filename"; print $statfd "vzdump::$path\n"; $statfd->close(); } else { my $info = $map->{$filename}; die "no vzdump info for '$filename'\n" if !$info; if ($filename !~ m/^vm-disk-$info->{virtdev}\.([^\.]+)$/) { die "got strange filename '$filename'\n"; } if ($filesize != $info->{size}) { die "detected size difference for '$filename' " . "($filesize != $info->{size})\n"; } # check permission for all used storages my $pool = $opts->{pool}; if ($user ne 'root@pam') { if (defined($opts->{storage})) { my $sid = $opts->{storage} || 'local'; $rpcenv->check($user, "/storage/$sid", ['Datastore.AllocateSpace']); } else { foreach my $fn (keys %$map) { my $fi = $map->{$fn}; my $sid = $fi->{storeid} || 'local'; $rpcenv->check($user, "/storage/$sid", ['Datastore.AllocateSpace']); } } } my $storeid; if (defined($opts->{storage})) { $storeid = $opts->{storage} || 'local'; } else { $storeid = $info->{storeid} || 'local'; } my $cfg = PVE::Storage::config(); my $scfg = PVE::Storage::storage_config($cfg, $storeid); my $alloc_size = int(($filesize + 1024 - 1) / 1024); if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') { # hack: we just alloc a small file (32K) - we overwrite it anyways $alloc_size = 32; } else { die "unable to restore '$filename' to storage '$storeid'\n" . "storage type '$scfg->{type}' does not support format '$format\n" if $format ne 'raw'; } my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $format, undef, $alloc_size); print STDERR "new volume ID is '$volid'\n"; print $statfd "vzdump:$info->{virtdev}:$volid\n"; $statfd->close(); $path = PVE::Storage::path($cfg, $volid); } print STDERR "restore data to '$path' ($filesize bytes)\n"; if ($opts->{prealloc} || $format ne 'raw' || (-b $path)) { exec 'dd', 'ibs=256K', 'obs=256K', "of=$path"; die "couldn't exec dd: $!\n"; } else { exec '/bin/cp', '--sparse=always', '/dev/stdin', $path; die "couldn't exec cp: $!\n"; } } if (scalar(@ARGV) == 2) { my $archive = shift; my $vmid = shift; # fixme: use API call PVE::JSONSchema::pve_verify_vmid($vmid); PVE::Cluster::check_cfs_quorum(); PVE::QemuServer::restore_archive($archive, $vmid, 'root@pam', $opts); } elsif (scalar(@ARGV) == 0 && $ENV{TAR_FILENAME}) { extract_archive(); } else { print_usage(); exit(-1); } exit(0); ================================================ FILE: src/bin/qmrestore ================================================ #!/usr/bin/perl use strict; use warnings; use PVE::CLI::qmrestore; PVE::CLI::qmrestore->run_cli_handler(); ================================================ FILE: src/qmeventd/.clang-format ================================================ BasedOnStyle: LLVM ColumnLimit: 100 IndentWidth: 4 AlignAfterOpenBracket: BlockIndent BinPackParameters: false # TODO: evaluate using OnePerLine (needs clang 20+) for a balanance? AlwaysBreakBeforeMultilineStrings: true AllowShortIfStatementsOnASingleLine: false InsertBraces: true AllowShortEnumsOnASingleLine: false ================================================ FILE: src/qmeventd/Makefile ================================================ DESTDIR= PREFIX=/usr SBINDIR=$(PREFIX)/sbin SERVICEDIR=$(PREFIX)/lib/systemd/system MANDIR=$(PREFIX)/share/man -include /usr/share/pve-doc-generator/pve-doc-generator.mk CC ?= gcc CFLAGS += -O2 -Werror -Wall -Wextra -Wpedantic -Wtype-limits -Wl,-z,relro -std=gnu11 CFLAGS += $(shell pkgconf --cflags json-c glib-2.0) LDFLAGS += $(shell pkgconf --libs json-c glib-2.0) qmeventd: qmeventd.c $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) docs: qmeventd.8 .PHONY: install install: qmeventd docs install -d $(DESTDIR)/$(SBINDIR) install -m 0755 qmeventd $(DESTDIR)/$(SBINDIR) install -d $(DESTDIR)/$(SERVICEDIR) install -m 0644 qmeventd.service $(DESTDIR)/$(SERVICEDIR) install -d $(DESTDIR)/$(MANDIR)/man8 install -m 0644 qmeventd.8 $(DESTDIR)/$(MANDIR)/man8 .PHONY: clean clean: $(MAKE) cleanup-docgen rm -rf qmeventd ================================================ FILE: src/qmeventd/qmeventd.c ================================================ // SPDX-License-Identifier: AGPL-3.0-or-later /* Copyright (C) 2018 - 2021 Proxmox Server Solutions GmbH Author: Dominik Csapak Author: Stefan Reiter Description: qmeventd listens on a given socket, and waits for qemu processes to connect. After accepting a connection qmeventd waits for shutdown events followed by the closing of the socket. Once that happens `qm cleanup` will be executed with following three arguments: VMID Where `graceful` can be `1` or `0` depending if shutdown event was observed before the socket got closed. The second parameter `guest` is also boolean `1` or `0` depending if the shutdown was requested from the guest OS (i.e., the "inside"). */ #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "qmeventd.h" #define DEFAULT_KILL_TIMEOUT 60 static int verbose = 0; static int kill_timeout = DEFAULT_KILL_TIMEOUT; static int epoll_fd = 0; static const char *progname; GHashTable *vm_clients; // key=vmid (freed on remove), value=*Client (free manually) GSList *forced_cleanups; static int needs_cleanup = 0; /* * Helper functions */ static void usage() { fprintf(stderr, "Usage: %s [-f] [-v] PATH\n", progname); fprintf(stderr, " -f run in foreground (default: false)\n"); fprintf(stderr, " -v verbose (default: false)\n"); fprintf(stderr, " -t kill timeout (default: %ds)\n", DEFAULT_KILL_TIMEOUT); fprintf(stderr, " PATH use PATH for socket\n"); } static pid_t get_pid_from_fd(int fd) { struct ucred credentials = {.pid = 0, .uid = 0, .gid = 0}; socklen_t len = sizeof(struct ucred); log_neg(getsockopt(fd, SOL_SOCKET, SO_PEERCRED, &credentials, &len), "getsockopt"); return credentials.pid; } /* * parses the vmid from the qemu.slice entry of /proc//cgroup */ static unsigned long get_vmid_from_pid(pid_t pid) { char filename[32] = {0}; int len = snprintf(filename, sizeof(filename), "/proc/%d/cmdline", pid); if (len < 0) { fprintf(stderr, "error during snprintf for %d: %s\n", pid, strerror(errno)); return 0; } if ((size_t)len >= sizeof(filename)) { fprintf(stderr, "error: pid %d too long\n", pid); return 0; } FILE *fp = fopen(filename, "re"); if (fp == NULL) { fprintf(stderr, "error opening %s: %s\n", filename, strerror(errno)); return 0; } unsigned long vmid = 0; char *buf = NULL; size_t buflen = 0; while (getdelim(&buf, &buflen, '\0', fp) >= 0) { if (strcmp(buf, "-pidfile")) { continue; } if (getdelim(&buf, &buflen, '\0', fp) < 0) { break; } char *vmid_start = strrchr(buf, '/'); if (!vmid_start) { fprintf(stderr, "unexpected pidfile option %s\n", buf); break; } vmid_start++; if (vmid_start[0] == '-' || vmid_start[0] == '\0') { fprintf(stderr, "invalid vmid in pidfile option %s\n", buf); break; } errno = 0; char *endptr = NULL; vmid = strtoul(vmid_start, &endptr, 10); if (!endptr || strcmp(endptr, ".pid")) { fprintf(stderr, "unexpected pidfile option %s\n", buf); vmid = 0; break; } if (errno != 0) { vmid = 0; } break; } if (errno) { fprintf(stderr, "error parsing vmid for %d: %s\n", pid, strerror(errno)); } else if (!vmid) { fprintf(stderr, "error parsing vmid for %d: no matching pidfile option\n", pid); } free(buf); fclose(fp); return vmid; } static bool must_write(int fd, const char *buf, size_t len) { ssize_t wlen; do { wlen = write(fd, buf, len); } while (wlen < 0 && errno == EINTR); return (wlen == (ssize_t)len); } /* * qmp handling functions */ static void send_qmp_cmd(struct Client *client, const char *buf, size_t len) { if (!must_write(client->fd, buf, len - 1)) { fprintf(stderr, "%s: cannot send QMP message\n", client->qemu.vmid); cleanup_client(client); } } void handle_qmp_handshake(struct Client *client) { VERBOSE_PRINT("pid%d: got QMP handshake, assuming QEMU client\n", client->pid); // extract vmid from cmdline, now that we know it's a QEMU process unsigned long vmid = get_vmid_from_pid(client->pid); int res = snprintf(client->qemu.vmid, sizeof(client->qemu.vmid), "%lu", vmid); if (vmid == 0 || res < 0 || res >= (int)sizeof(client->qemu.vmid)) { fprintf(stderr, "could not get vmid from pid %d\n", client->pid); cleanup_client(client); return; } VERBOSE_PRINT("pid%d: assigned VMID: %s\n", client->pid, client->qemu.vmid); client->type = CLIENT_QEMU; if (!g_hash_table_insert(vm_clients, strdup(client->qemu.vmid), client)) { // not fatal, just means backup handling won't work fprintf(stderr, "%s: could not insert client into VMID->client table\n", client->qemu.vmid); } static const char qmp_answer[] = "{\"execute\":\"qmp_capabilities\"}\n"; send_qmp_cmd(client, qmp_answer, sizeof(qmp_answer)); } void handle_qmp_event(struct Client *client, struct json_object *obj) { struct json_object *event; if (!json_object_object_get_ex(obj, "event", &event)) { return; } VERBOSE_PRINT("%s: got QMP event: %s\n", client->qemu.vmid, json_object_get_string(event)); if (client->state == STATE_TERMINATING) { // QEMU sometimes sends a second SHUTDOWN after SIGTERM, ignore VERBOSE_PRINT("%s: event was after termination, ignoring\n", client->qemu.vmid); return; } // event, check if shutdown and get guest parameter if (!strcmp(json_object_get_string(event), "SHUTDOWN")) { client->qemu.graceful = 1; struct json_object *data; struct json_object *guest; if (json_object_object_get_ex(obj, "data", &data) && json_object_object_get_ex(data, "guest", &guest)) { client->qemu.guest = (unsigned short)json_object_get_boolean(guest); } // check if a backup is running and kill QEMU process if not terminate_check(client); } } void terminate_check(struct Client *client) { if (client->state != STATE_IDLE) { // if we're already in a request, queue this one until after VERBOSE_PRINT("%s: terminate_check queued\n", client->qemu.vmid); client->qemu.term_check_queued = true; return; } client->qemu.term_check_queued = false; VERBOSE_PRINT("%s: query-status\n", client->qemu.vmid); client->state = STATE_EXPECT_STATUS_RESP; static const char qmp_req[] = "{\"execute\":\"query-status\"}\n"; send_qmp_cmd(client, qmp_req, sizeof(qmp_req)); } void handle_qmp_return(struct Client *client, struct json_object *data, bool error) { if (error) { const char *msg = "n/a"; struct json_object *desc; if (json_object_object_get_ex(data, "desc", &desc)) { msg = json_object_get_string(desc); } fprintf(stderr, "%s: received error from QMP: %s\n", client->qemu.vmid, msg); client->state = STATE_IDLE; goto out; } struct json_object *status; json_bool has_status = data && json_object_object_get_ex(data, "status", &status); bool active = false; if (has_status) { const char *status_str = json_object_get_string(status); active = status_str && (!strcmp(status_str, "running") || !strcmp(status_str, "paused") || !strcmp(status_str, "suspended") || !strcmp(status_str, "prelaunch")); } switch (client->state) { case STATE_EXPECT_STATUS_RESP: client->state = STATE_IDLE; if (active) { VERBOSE_PRINT("%s: got status: VM is active\n", client->qemu.vmid); } else if (!client->qemu.backup) { terminate_client(client); } else { // if we're in a backup, don't do anything, vzdump will notify // us when the backup finishes VERBOSE_PRINT("%s: not active, but running backup - keep alive\n", client->qemu.vmid); } break; // this means we received the empty return from our handshake answer case STATE_HANDSHAKE: client->state = STATE_IDLE; VERBOSE_PRINT("%s: QMP handshake complete\n", client->qemu.vmid); break; // we expect an empty return object after sending quit case STATE_TERMINATING: break; case STATE_IDLE: VERBOSE_PRINT("%s: spurious return value received\n", client->qemu.vmid); break; } out: if (client->qemu.term_check_queued) { terminate_check(client); } } /* * VZDump specific client functions */ void handle_vzdump_handshake(struct Client *client, struct json_object *data) { client->state = STATE_IDLE; struct json_object *vmid_obj; json_bool has_vmid = data && json_object_object_get_ex(data, "vmid", &vmid_obj); if (!has_vmid) { VERBOSE_PRINT("pid%d: invalid vzdump handshake: no vmid\n", client->pid); return; } const char *vmid_str = json_object_get_string(vmid_obj); if (!vmid_str) { VERBOSE_PRINT("pid%d: invalid vzdump handshake: vmid is not a string\n", client->pid); return; } int res = snprintf(client->vzdump.vmid, sizeof(client->vzdump.vmid), "%s", vmid_str); if (res < 0 || res >= (int)sizeof(client->vzdump.vmid)) { VERBOSE_PRINT("pid%d: invalid vzdump handshake: vmid too long or invalid\n", client->pid); return; } struct Client *vmc = (struct Client *)g_hash_table_lookup(vm_clients, client->vzdump.vmid); if (vmc) { vmc->qemu.backup = true; // only mark as VZDUMP once we have set everything up, otherwise 'cleanup' // might try to access an invalid value client->type = CLIENT_VZDUMP; VERBOSE_PRINT("%s: vzdump backup started\n", client->vzdump.vmid); } else { VERBOSE_PRINT( "%s: vzdump requested backup start for unregistered VM\n", client->vzdump.vmid ); } } /* * client management functions */ void add_new_client(int client_fd) { struct Client *client = calloc(1, sizeof(struct Client)); if (client == NULL) { fprintf(stderr, "could not add new client - allocation failed!\n"); fflush(stderr); return; } client->state = STATE_HANDSHAKE; client->type = CLIENT_NONE; client->fd = client_fd; client->pid = get_pid_from_fd(client_fd); if (client->pid == 0) { fprintf(stderr, "could not get pid from client\n"); goto err; } struct epoll_event ev; ev.events = EPOLLIN; ev.data.ptr = client; int res = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_fd, &ev); if (res < 0) { perror("epoll_ctl client add"); goto err; } VERBOSE_PRINT("added new client, pid: %d\n", client->pid); return; err: (void)close(client_fd); free(client); } static void cleanup_qemu_client(struct Client *client) { unsigned short graceful = client->qemu.graceful; unsigned short guest = client->qemu.guest; char vmid[sizeof(client->qemu.vmid)]; strncpy(vmid, client->qemu.vmid, sizeof(vmid)); g_hash_table_remove(vm_clients, &vmid); // frees key, ignore errors VERBOSE_PRINT("%s: executing cleanup (graceful: %d, guest: %d)\n", vmid, graceful, guest); int pid = fork(); if (pid < 0) { fprintf(stderr, "fork failed: %s\n", strerror(errno)); return; } if (pid == 0) { char *script = "/usr/sbin/qm"; char *args[] = {script, "cleanup", vmid, graceful ? "1" : "0", guest ? "1" : "0", NULL}; execvp(script, args); perror("execvp"); _exit(1); } } void cleanup_client(struct Client *client) { log_neg(epoll_ctl(epoll_fd, EPOLL_CTL_DEL, client->fd, NULL), "epoll del"); (void)close(client->fd); struct Client *vmc; switch (client->type) { case CLIENT_QEMU: cleanup_qemu_client(client); break; case CLIENT_VZDUMP: vmc = (struct Client *)g_hash_table_lookup(vm_clients, client->vzdump.vmid); if (vmc) { VERBOSE_PRINT("%s: backup ended\n", client->vzdump.vmid); vmc->qemu.backup = false; terminate_check(vmc); } break; case CLIENT_NONE: // do nothing, only close socket break; } if (client->pidfd > 0) { (void)close(client->pidfd); } VERBOSE_PRINT("removing %s from forced cleanups\n", client->qemu.vmid); forced_cleanups = g_slist_remove(forced_cleanups, client); free(client); } void terminate_client(struct Client *client) { VERBOSE_PRINT("%s: terminating client (pid %d)\n", client->qemu.vmid, client->pid); client->state = STATE_TERMINATING; // open a pidfd before kill for later cleanup int pidfd = pidfd_open(client->pid, 0); if (pidfd < 0) { switch (errno) { case ESRCH: // process already dead for some reason, cleanup done VERBOSE_PRINT( "%s: failed to open pidfd, process already dead (pid %d)\n", client->qemu.vmid, client->pid ); return; // otherwise fall back to just using the PID directly, but don't // print if we only failed because we're running on an older kernel case ENOSYS: break; default: perror("failed to open QEMU pidfd for cleanup"); break; } } // try to send a 'quit' command first, fallback to SIGTERM of the pid static const char qmp_quit_command[] = "{\"execute\":\"quit\"}\n"; VERBOSE_PRINT("%s: sending 'quit' via QMP\n", client->qemu.vmid); if (!must_write(client->fd, qmp_quit_command, sizeof(qmp_quit_command) - 1)) { VERBOSE_PRINT("%s: sending 'SIGTERM' to pid %d\n", client->qemu.vmid, client->pid); int err = kill(client->pid, SIGTERM); log_neg(err, "kill"); } time_t timeout = time(NULL) + kill_timeout; client->pidfd = pidfd; client->timeout = timeout; forced_cleanups = g_slist_prepend(forced_cleanups, (void *)client); needs_cleanup = 1; } void handle_client(struct Client *client) { VERBOSE_PRINT("pid%d: entering handle\n", client->pid); ssize_t len; do { len = read(client->fd, (client->buf + client->buflen), sizeof(client->buf) - client->buflen); } while (len < 0 && errno == EINTR); if (len < 0) { if (!(errno == EAGAIN || errno == EWOULDBLOCK)) { log_neg((int)len, "read"); cleanup_client(client); } return; } else if (len == 0) { VERBOSE_PRINT("pid%d: got EOF\n", client->pid); cleanup_client(client); return; } VERBOSE_PRINT("pid%d: read %ld bytes\n", client->pid, len); client->buflen += len; struct json_tokener *tok = json_tokener_new(); struct json_object *jobj = NULL; enum json_tokener_error jerr = json_tokener_success; while (jerr == json_tokener_success && client->buflen != 0) { jobj = json_tokener_parse_ex(tok, client->buf, (int)client->buflen); jerr = json_tokener_get_error(tok); unsigned int offset = (unsigned int)tok->char_offset; switch (jerr) { case json_tokener_success: // move rest from buffer to front memmove(client->buf, client->buf + offset, client->buflen - offset); client->buflen -= offset; if (json_object_is_type(jobj, json_type_object)) { struct json_object *obj; if (json_object_object_get_ex(jobj, "QMP", &obj)) { handle_qmp_handshake(client); } else if (json_object_object_get_ex(jobj, "event", &obj)) { handle_qmp_event(client, jobj); } else if (json_object_object_get_ex(jobj, "return", &obj)) { handle_qmp_return(client, obj, false); } else if (json_object_object_get_ex(jobj, "error", &obj)) { handle_qmp_return(client, obj, true); } else if (json_object_object_get_ex(jobj, "vzdump", &obj)) { handle_vzdump_handshake(client, obj); } // else ignore message } break; case json_tokener_continue: if (client->buflen >= sizeof(client->buf)) { VERBOSE_PRINT("pid%d: msg too large, discarding buffer\n", client->pid); memset(client->buf, 0, sizeof(client->buf)); client->buflen = 0; } // else we have enough space try again after next read break; default: VERBOSE_PRINT("pid%d: parse error: %d, discarding buffer\n", client->pid, jerr); memset(client->buf, 0, client->buflen); client->buflen = 0; break; } json_object_put(jobj); } json_tokener_free(tok); } static void sigkill(void *ptr, void *time_ptr) { struct Client *data = ptr; int err; if (data->timeout != 0 && data->timeout > *(time_t *)time_ptr) { return; } if (data->pidfd > 0) { err = pidfd_send_signal(data->pidfd, SIGKILL, NULL, 0); (void)close(data->pidfd); data->pidfd = -1; } else { err = kill(data->pid, SIGKILL); } if (err < 0) { if (errno != ESRCH) { fprintf( stderr, "SIGKILL cleanup of pid '%d' failed - %s\n", data->pid, strerror(errno) ); } } else { fprintf(stderr, "cleanup failed, terminating pid '%d' with SIGKILL\n", data->pid); } data->timeout = 0; // remove ourselves from the list forced_cleanups = g_slist_remove(forced_cleanups, ptr); } static void handle_forced_cleanup() { if (g_slist_length(forced_cleanups) > 0) { VERBOSE_PRINT("clearing forced cleanup backlog\n"); time_t cur_time = time(NULL); g_slist_foreach(forced_cleanups, sigkill, &cur_time); } needs_cleanup = g_slist_length(forced_cleanups) > 0; } int main(int argc, char *argv[]) { int opt; int daemonize = 1; char *socket_path = NULL; progname = argv[0]; while ((opt = getopt(argc, argv, "hfvt:")) != -1) { switch (opt) { case 'f': daemonize = 0; break; case 'v': verbose = 1; break; case 't': errno = 0; char *endptr = NULL; kill_timeout = strtoul(optarg, &endptr, 10); if (errno != 0 || *endptr != '\0' || kill_timeout == 0) { usage(); exit(EXIT_FAILURE); } break; case 'h': usage(); exit(EXIT_SUCCESS); break; default: usage(); exit(EXIT_FAILURE); } } if (optind >= argc) { usage(); exit(EXIT_FAILURE); } signal(SIGCHLD, SIG_IGN); socket_path = argv[optind]; int sock = socket(AF_UNIX, SOCK_STREAM, 0); bail_neg(sock, "socket"); struct sockaddr_un addr; memset(&addr, 0, sizeof(addr)); addr.sun_family = AF_UNIX; strncpy(addr.sun_path, socket_path, sizeof(addr.sun_path) - 1); unlink(socket_path); bail_neg(bind(sock, (struct sockaddr *)&addr, sizeof(addr)), "bind"); struct epoll_event ev, events[1]; epoll_fd = epoll_create1(EPOLL_CLOEXEC); bail_neg(epoll_fd, "epoll_create1"); ev.events = EPOLLIN; ev.data.fd = sock; bail_neg(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sock, &ev), "epoll_ctl"); bail_neg(listen(sock, 10), "listen"); if (daemonize) { bail_neg(daemon(0, 1), "daemon"); } vm_clients = g_hash_table_new_full(g_str_hash, g_str_equal, free, NULL); int nevents; for (;;) { nevents = epoll_wait(epoll_fd, events, 1, needs_cleanup ? 10 * 1000 : -1); if (nevents < 0 && errno == EINTR) { continue; } bail_neg(nevents, "epoll_wait"); for (int n = 0; n < nevents; n++) { if (events[n].data.fd == sock) { int conn_sock = accept4(sock, NULL, NULL, SOCK_NONBLOCK | SOCK_CLOEXEC); log_neg(conn_sock, "accept"); if (conn_sock > -1) { add_new_client(conn_sock); } } else { handle_client((struct Client *)events[n].data.ptr); } } handle_forced_cleanup(); } } ================================================ FILE: src/qmeventd/qmeventd.h ================================================ // SPDX-License-Identifier: AGPL-3.0-or-later /* Copyright (C) 2018 - 2021 Proxmox Server Solutions GmbH Author: Dominik Csapak Author: Stefan Reiter */ #include #include #ifndef __NR_pidfd_open #define __NR_pidfd_open 434 #endif #ifndef __NR_pidfd_send_signal #define __NR_pidfd_send_signal 424 #endif #define VERBOSE_PRINT(...) \ do { \ if (verbose) { \ printf(__VA_ARGS__); \ fflush(stdout); \ } \ } while (0) static inline void log_neg(int errval, const char *msg) { if (errval < 0) { perror(msg); } } static inline void bail_neg(int errval, const char *msg) { if (errval < 0) { perror(msg); exit(EXIT_FAILURE); } } static inline int pidfd_open(pid_t pid, unsigned int flags) { return syscall(__NR_pidfd_open, pid, flags); } static inline int pidfd_send_signal(int pidfd, int sig, siginfo_t *info, unsigned int flags) { return syscall(__NR_pidfd_send_signal, pidfd, sig, info, flags); } typedef enum { CLIENT_NONE, CLIENT_QEMU, CLIENT_VZDUMP } ClientType; typedef enum { STATE_HANDSHAKE, STATE_IDLE, STATE_EXPECT_STATUS_RESP, STATE_TERMINATING } ClientState; struct Client { char buf[4096]; unsigned int buflen; int fd; pid_t pid; int pidfd; time_t timeout; ClientType type; ClientState state; // only relevant for type=CLIENT_QEMU struct { char vmid[16]; unsigned short graceful; unsigned short guest; bool term_check_queued; bool backup; } qemu; // only relevant for type=CLIENT_VZDUMP struct { // vmid of referenced backup char vmid[16]; } vzdump; }; void handle_qmp_handshake(struct Client *client); void handle_qmp_event(struct Client *client, struct json_object *obj); void handle_qmp_return(struct Client *client, struct json_object *data, bool error); void handle_vzdump_handshake(struct Client *client, struct json_object *data); void handle_client(struct Client *client); void add_new_client(int client_fd); void cleanup_client(struct Client *client); void terminate_client(struct Client *client); void terminate_check(struct Client *client); ================================================ FILE: src/qmeventd/qmeventd.service ================================================ [Unit] Description=PVE Qemu Event Daemon RequiresMountsFor=/var/run Before=pve-ha-lrm.service Before=pve-guests.service [Service] ExecStart=/usr/sbin/qmeventd /var/run/qmeventd.sock Type=forking [Install] WantedBy=multi-user.target ================================================ FILE: src/query-machine-capabilities/Makefile ================================================ DESTDIR= PREFIX=/usr BINDIR=$(PREFIX)/libexec/qemu-server SERVICEDIR=$(PREFIX)/lib/systemd/system CC ?= gcc CFLAGS += -O2 -fanalyzer -Werror -Wall -Wextra -Wpedantic -Wtype-limits -Wl,-z,relro -std=gnu11 query-machine-capabilities: query-machine-capabilities.c $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) .PHONY: install install: query-machine-capabilities install -d $(DESTDIR)/$(BINDIR) install -m 0755 query-machine-capabilities $(DESTDIR)$(BINDIR) install -d $(DESTDIR)/$(SERVICEDIR) install -m 0644 pve-query-machine-capabilities.service $(DESTDIR)$(SERVICEDIR) .PHONY: clean clean: rm -f query-machine-capabilities ================================================ FILE: src/query-machine-capabilities/pve-query-machine-capabilities.service ================================================ [Unit] Description=PVE Query Machine Capabilities RequiresMountsFor=/run Before=pve-ha-lrm.service Before=pve-guests.service [Service] ExecStart=/usr/libexec/qemu-server/query-machine-capabilities Type=oneshot RemainAfterExit=yes [Install] WantedBy=multi-user.target ================================================ FILE: src/query-machine-capabilities/query-machine-capabilities.c ================================================ #include #include #include #include #include #include #include #include #include #ifdef __aarch64__ #include #ifndef HWCAP_AES #define HWCAP_AES (1 << 3) #endif #ifndef HWCAP_SHA2 #define HWCAP_SHA2 (1 << 6) #endif #endif // __aarch64__ #define eprintf(...) fprintf(stderr, __VA_ARGS__) #define OUTPUT_DIR "/run/qemu-server" #define OUTPUT_FILENAME "host-hw-capabilities.json" #define OUTPUT_PATH OUTPUT_DIR "/" OUTPUT_FILENAME typedef struct { bool sev_support; bool sev_es_support; bool sev_snp_support; uint8_t cbitpos; uint8_t reduced_phys_bits; } cpu_caps_amd_sev_t; typedef struct { bool tdx_support; } cpu_caps_intel_tdx_t; typedef struct { bool aes; bool sha2; } cpu_caps_arm_t; static inline void cpu_vendor(char vendor[13]) { #ifdef __x86_64__ uint32_t eax; uint32_t *vp = (uint32_t *)vendor; asm volatile("cpuid" : "=a"(eax), "=b"(*vp), "=c"(*(vp+2)), "=d"(*(vp+1)) : "a"(0) ); #elif defined(__aarch64__) // just parse /proc/cpuinfo as the MIDR_EL1 mrs might not be available to read from user space FILE *f = fopen("/proc/cpuinfo", "r"); int implementer = 0; if (f) { char line[256]; while (fgets(line, sizeof(line), f)) { if (strncmp(line, "CPU implementer", 15) == 0) { char *p = strchr(line, ':'); if (p) implementer = (int)strtol(p + 1, NULL, 0); break; } } fclose(f); } // mapping taken from arch/arm64/include/asm/cputype.h (e.g. ARM_CPU_IMP_ARM) switch(implementer) { case 0x41: strcpy(vendor, "ARM Limited"); break; case 0x42: strcpy(vendor, "Broadcom"); break; case 0x43: strcpy(vendor, "Cavium"); break; case 0x48: strcpy(vendor, "HiSilicon"); break; case 0x4E: strcpy(vendor, "NVIDIA"); break; case 0x51: strcpy(vendor, "Qualcomm"); break; case 0x53: strcpy(vendor, "Samsung"); break; case 0x61: strcpy(vendor, "Apple"); break; case 0xC0: strcpy(vendor, "Ampere"); break; default: snprintf(vendor, 13, "ARM64:%02x", implementer); break; } #else strcpy(vendor, "Unknown"); #endif vendor[12] = '\0'; } int read_msr(uint32_t msr_index, uint64_t *value) { uint64_t data; char* msr_file_name = "/dev/cpu/0/msr"; int fd; fd = open(msr_file_name, O_RDONLY); if (fd < 0) { if (errno == ENXIO) { eprintf("rdmsr: No CPU 0\n"); return -1; } else if (errno == EIO) { eprintf("rdmsr: CPU doesn't support MSRs\n"); return -1; } else { perror("rdmsr: failed to open MSR"); return -1; } } if (pread(fd, &data, sizeof(data), msr_index) != sizeof(data)) { if (errno == EIO) { eprintf("rdmsr: CPU cannot read MSR 0x%08x\n", msr_index); return -1; } else { perror("rdmsr: pread"); return -1; } } *value = data; close(fd); return 0; } void query_cpu_capabilities_sev(cpu_caps_amd_sev_t *res) { #ifdef __x86_64__ uint32_t eax, ebx, ecx, edx; // query Encrypted Memory Capabilities, see: // https://en.wikipedia.org/wiki/CPUID#EAX=8000001Fh:_Encrypted_Memory_Capabilities uint32_t query_function = 0x8000001F; asm volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "0"(query_function) ); res->sev_support = (eax & (1<<1)) != 0; res->sev_es_support = (eax & (1<<3)) != 0; res->sev_snp_support = (eax & (1<<4)) != 0; res->cbitpos = ebx & 0x3f; res->reduced_phys_bits = (ebx >> 6) & 0x3f; #else memset(res, 0, sizeof(*res)); #endif } int query_cpu_capabilities_tdx(cpu_caps_intel_tdx_t *res) { uint64_t tme_value, sgx_value, tdx_value; if (read_msr(0x982, &tme_value) == 0 && read_msr(0xa0, &sgx_value) == 0 && read_msr(0x1401, &tdx_value) == 0) { res->tdx_support = ((tme_value >> 1) & 1ULL) & (!sgx_value) & ((tdx_value >> 11) & 1ULL); } else { eprintf("Intel TDX support undetermined\n"); return -1; } return 0; } #ifdef __aarch64__ void query_cpu_capabilities_arm(cpu_caps_arm_t *res) { unsigned long hwcaps = getauxval(AT_HWCAP); res->aes = (hwcaps & HWCAP_AES); res->sha2 = (hwcaps & HWCAP_SHA2); } #endif int prepare_output_directory() { // Check that the directory exists and create it if it does not. struct stat statbuf; int ret = stat(OUTPUT_DIR, &statbuf); if (ret == 0) { if (!S_ISDIR(statbuf.st_mode)) { eprintf("Path '" OUTPUT_DIR "' already exists but is not a directory.\n"); return 0; } } else if (errno == ENOENT) { if (mkdir(OUTPUT_DIR, 0755) != 0) { eprintf("Error creating directory '" OUTPUT_DIR "': %s\n", strerror(errno)); return 0; } } else { eprintf("Error checking path '" OUTPUT_DIR "': %s\n", strerror(errno)); return 0; } return 1; } int main() { if (!prepare_output_directory()) { return 1; } FILE *file = fopen(OUTPUT_PATH, "w"); if (file == NULL) { eprintf("Error opening to file '" OUTPUT_PATH "': %s\n", strerror(errno)); return 1; } char vendor[13]; cpu_vendor(vendor); int ret = fprintf(file, "{"); if (ret < 0) { eprintf("Error writing to file '" OUTPUT_PATH "': %s\n", strerror(errno)); } if (strncmp(vendor, "AuthenticAMD", 12) == 0) { cpu_caps_amd_sev_t caps_sev; query_cpu_capabilities_sev(&caps_sev); ret = fprintf(file, " \"amd-sev\": {" " \"cbitpos\": %u," " \"reduced-phys-bits\": %u," " \"sev-support\": %s," " \"sev-support-es\": %s," " \"sev-support-snp\": %s" " }", caps_sev.cbitpos, caps_sev.reduced_phys_bits, caps_sev.sev_support ? "true" : "false", caps_sev.sev_es_support ? "true" : "false", caps_sev.sev_snp_support ? "true" : "false" ); } else if (strncmp(vendor, "GenuineIntel", 12) == 0) { cpu_caps_intel_tdx_t caps_tdx; if (query_cpu_capabilities_tdx(&caps_tdx) == 0) { ret = fprintf(file, " \"intel-tdx\": {" " \"tdx-support\": %s" " }", caps_tdx.tdx_support ? "true" : "false" ); } } #ifdef __aarch64__ else { cpu_caps_arm_t caps_arm; query_cpu_capabilities_arm(&caps_arm); ret = fprintf(file, " \"arm-caps\": {" " \"vendor\": \"%s\"," " \"aes\": %s," " \"sha2\": %s" " }", vendor, caps_arm.aes ? "true" : "false", caps_arm.sha2 ? "true" : "false" ); } #endif if (ret < 0) { eprintf("Error writing to file '" OUTPUT_PATH "': %s\n", strerror(errno)); } ret = fprintf(file, " }\n"); if (ret < 0) { eprintf("Error writing to file '" OUTPUT_PATH "': %s\n", strerror(errno)); } ret = fclose(file); if (ret != 0) { eprintf("Error closing file '" OUTPUT_PATH "': %s\n", strerror(errno)); } return 0; } ================================================ FILE: src/test/Makefile ================================================ all: test test: test_snapshot test_cfg_to_cmd test_cfg_to_cmd_aarch64 test_pci_addr_conflicts test_pci_reservation test_qemu_img_convert test_migration test_restore_config test_parse_config test_snapshot: run_snapshot_tests.pl ./run_snapshot_tests.pl ./test_get_replicatable_volumes.pl test_cfg_to_cmd: run_config2command_tests.pl cfg2cmd/*.conf perl -I../ ./run_config2command_tests.pl test_cfg_to_cmd_aarch64: run_config2command_tests.pl cfg2cmd/aarch64/*.conf perl -I../ ./run_config2command_tests.pl cfg2cmd/aarch64 test_qemu_img_convert: run_qemu_img_convert_tests.pl perl -I../ ./run_qemu_img_convert_tests.pl test_pci_addr_conflicts: run_pci_addr_checks.pl ./run_pci_addr_checks.pl test_pci_reservation: run_pci_reservation_tests.pl ./run_pci_reservation_tests.pl MIGRATION_TEST_TARGETS := $(addprefix test_migration_,$(shell perl -ne 'print "$$1 " if /^\s*name\s*=>\s*["'\'']([^\s"'\'']+)["'\'']\s*,\s*$$/; END { print "\n" }' run_qemu_migrate_tests.pl)) test_migration: run_qemu_migrate_tests.pl MigrationTest/*.pm $(MIGRATION_TEST_TARGETS) $(MIGRATION_TEST_TARGETS): ./run_qemu_migrate_tests.pl $(@:test_migration_%=%) test_restore_config: run_qemu_restore_config_tests.pl ./run_qemu_restore_config_tests.pl test_parse_config: run_parse_config_tests.pl ./run_parse_config_tests.pl .PHONY: clean clean: rm -rf MigrationTest/run parse-config-output ================================================ FILE: src/test/MigrationTest/QemuMigrateMock.pm ================================================ package MigrationTest::QemuMigrateMock; use strict; use warnings; use JSON; use Test::MockModule; use MigrationTest::Shared; use PVE::API2::Qemu; use PVE::QemuServer::Drive; use PVE::Storage; use PVE::Tools qw(file_set_contents file_get_contents); use PVE::CLIHandler; use base qw(PVE::CLIHandler); my $RUN_DIR_PATH = $ENV{RUN_DIR_PATH} or die "no RUN_DIR_PATH set\n"; my $QM_LIB_PATH = $ENV{QM_LIB_PATH} or die "no QM_LIB_PATH set\n"; my $source_volids = decode_json(file_get_contents("${RUN_DIR_PATH}/source_volids")); my $source_vdisks = decode_json(file_get_contents("${RUN_DIR_PATH}/source_vdisks")); my $vm_status = decode_json(file_get_contents("${RUN_DIR_PATH}/vm_status")); my $expected_calls = decode_json(file_get_contents("${RUN_DIR_PATH}/expected_calls")); my $fail_config = decode_json(file_get_contents("${RUN_DIR_PATH}/fail_config")); my $storage_migrate_map = decode_json(file_get_contents("${RUN_DIR_PATH}/storage_migrate_map")); my $migrate_params = decode_json(file_get_contents("${RUN_DIR_PATH}/migrate_params")); my $test_vmid = $migrate_params->{vmid}; my $test_target = $migrate_params->{target}; my $test_opts = $migrate_params->{opts}; my $current_log = ''; my $vm_stop_executed = 0; # mocked modules my $inotify_module = Test::MockModule->new("PVE::INotify"); $inotify_module->mock( nodename => sub { return 'pve0'; }, ); $MigrationTest::Shared::qemu_config_module->mock( move_config_to_node => sub { my ($self, $vmid, $target) = @_; die "moving wrong config: '$vmid'\n" if $vmid ne $test_vmid; die "moving config to wrong node: '$target'\n" if $target ne $test_target; delete $expected_calls->{move_config_to_node}; }, ); my $tunnel_module = Test::MockModule->new("PVE::Tunnel"); $tunnel_module->mock( finish_tunnel => sub { delete $expected_calls->{'finish_tunnel'}; return; }, write_tunnel => sub { my ($tunnel, $timeout, $command) = @_; if ($command =~ m/^resume (\d+)$/) { my $vmid = $1; die "resuming wrong VM '$vmid'\n" if $vmid ne $test_vmid; return; } die "write_tunnel (mocked) - implement me: $command\n"; }, ); my $qemu_migrate_module = Test::MockModule->new("PVE::QemuMigrate"); $qemu_migrate_module->mock( fork_tunnel => sub { die "fork_tunnel (mocked) - implement me\n"; # currently no call should lead here }, read_tunnel => sub { die "read_tunnel (mocked) - implement me\n"; # currently no call should lead here }, start_remote_tunnel => sub { my ($self, $raddr, $rport, $ruri, $unix_socket_info) = @_; $expected_calls->{'finish_tunnel'} = 1; $self->{tunnel} = { writer => "mocked", reader => "mocked", pid => 123456, version => 1, }; }, log => sub { my ($self, $level, $message) = @_; $current_log .= "$level: $message\n"; }, mon_cmd => sub { my ($vmid, $command, %params) = @_; if ($command eq 'nbd-server-start') { return; } elsif ($command eq 'block-dirty-bitmap-add') { my $drive = $params{node}; delete $expected_calls->{"block-dirty-bitmap-add-${drive}"}; return; } elsif ($command eq 'block-dirty-bitmap-remove') { return; } elsif ($command eq 'query-migrate') { return { status => 'failed' } if $fail_config->{'query-migrate'}; return { status => 'completed' }; } elsif ($command eq 'migrate') { return; } elsif ($command eq 'migrate-set-parameters') { return; } elsif ($command eq 'migrate_cancel') { return; } die "mon_cmd (mocked) - implement me: $command"; }, transfer_replication_state => sub { delete $expected_calls->{transfer_replication_state}; }, switch_replication_job_target => sub { delete $expected_calls->{switch_replication_job_target}; }, ); $MigrationTest::Shared::qemu_server_module->mock( kvm_user_version => sub { return "5.0.0"; }, vm_stop => sub { $vm_stop_executed = 1; delete $expected_calls->{'vm_stop'}; }, ); my sub common_mirror_mock { my ($vmid, $drive_id) = @_; die "drive_mirror with wrong vmid: '$vmid'\n" if $vmid ne $test_vmid; die "qemu_drive_mirror '$drive_id' error\n" if $fail_config->{qemu_drive_mirror} && $fail_config->{qemu_drive_mirror} eq $drive_id; my $nbd_info = decode_json(file_get_contents("${RUN_DIR_PATH}/nbd_info")); die "target does not expect drive mirror for '$drive_id'\n" if !defined($nbd_info->{$drive_id}); delete $nbd_info->{$drive_id}; file_set_contents("${RUN_DIR_PATH}/nbd_info", to_json($nbd_info)); } my $qemu_server_blockjob_module = Test::MockModule->new("PVE::QemuServer::BlockJob"); $qemu_server_blockjob_module->mock( qemu_blockjobs_cancel => sub { return; }, qemu_drive_mirror => sub { my ( $vmid, $drive_id, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap, ) = @_; common_mirror_mock($vmid, $drive_id); }, blockdev_mirror => sub { my ($source, $dest, $jobs, $completion, $options) = @_; my $drive_id = PVE::QemuServer::Drive::get_drive_id($source->{drive}); common_mirror_mock($source->{vmid}, $drive_id); }, monitor => sub { my ($qmp_peer, $vmiddst, $jobs, $completion, $qga, $op) = @_; if ( $fail_config->{block_job_monitor} && $fail_config->{block_job_monitor} eq $completion ) { die "block_job_monitor '$completion' error\n"; } return; }, qemu_drive_mirror_switch_to_active_mode => sub { return; }, ); my $qemu_server_cpuconfig_module = Test::MockModule->new("PVE::QemuServer::CPUConfig"); $qemu_server_cpuconfig_module->mock( get_cpu_from_running_vm => sub { die "invalid test: if you specify a custom CPU model you need to " . "specify runningcpu as well\n" if !defined($vm_status->{runningcpu}); return $vm_status->{runningcpu}; }, ); my $qemu_server_helpers_module = Test::MockModule->new("PVE::QemuServer::Helpers"); $qemu_server_helpers_module->mock( vm_running_locally => sub { return $vm_status->{running} && !$vm_stop_executed; }, ); my $qemu_server_machine_module = Test::MockModule->new("PVE::QemuServer::Machine"); $qemu_server_machine_module->mock( qemu_machine_pxe => sub { die "invalid test: no runningmachine specified\n" if !defined($vm_status->{runningmachine}); return $vm_status->{runningmachine}; }, get_current_qemu_machine => sub { die "invalid test: no runningmachine specified\n" if !defined($vm_status->{runningmachine}); return $vm_status->{runningmachine}; }, ); my $qemu_server_network_module = Test::MockModule->new("PVE::QemuServer::Network"); $qemu_server_network_module->mock( del_nets_bridge_fdb => sub { return; }, mon_cmd => sub { my ($vmid, $command, %params) = @_; if ($command eq 'qom-get') { if ( $params{path} =~ m|^/machine/peripheral/net\d+$| && $params{property} eq 'host_mtu' ) { return 1500; } die "mon_cmd (mocked) - implement me: $command for path '$params{path}' property" . " '$params{property}'"; } die "mon_cmd (mocked) - implement me: $command"; }, ); my $qemu_server_qmphelpers_module = Test::MockModule->new("PVE::QemuServer::QMPHelpers"); $qemu_server_qmphelpers_module->mock( runs_at_least_qemu_version => sub { return 1; }, ); my $ssh_info_module = Test::MockModule->new("PVE::SSHInfo"); $ssh_info_module->mock( get_ssh_info => sub { my ($node, $network_cidr) = @_; return { ip => '1.2.3.4', name => $node, network => $network_cidr, }; }, ); $MigrationTest::Shared::storage_module->mock( storage_migrate => sub { my ($cfg, $volid, $target_sshinfo, $target_storeid, $opts, $logfunc) = @_; die "storage_migrate '$volid' error\n" if $fail_config->{storage_migrate} && $fail_config->{storage_migrate} eq $volid; my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); die "invalid test: need to add entry for '$volid' to storage_migrate_map\n" if $storeid ne $target_storeid && !defined($storage_migrate_map->{$volid}); my $target_volname = $storage_migrate_map->{$volid} // $opts->{target_volname} // $volname; my $target_volid = "${target_storeid}:${target_volname}"; MigrationTest::Shared::add_target_volid($target_volid); return $target_volid; }, vdisk_list => sub { # expects vmid to be set my ($cfg, $storeid, $vmid, $vollist) = @_; my @storeids = defined($storeid) ? ($storeid) : keys %{$source_vdisks}; my $res = {}; foreach my $storeid (@storeids) { my $list_for_storeid = $source_vdisks->{$storeid}; my @list_for_vm = grep { $_->{vmid} eq $vmid } @{$list_for_storeid}; $res->{$storeid} = \@list_for_vm; } return $res; }, vdisk_free => sub { my ($scfg, $volid) = @_; PVE::Storage::parse_volume_id($volid); die "vdisk_free '$volid' error\n" if defined($fail_config->{vdisk_free}) && $fail_config->{vdisk_free} eq $volid; delete $source_volids->{$volid}; }, volume_size_info => sub { my ($scfg, $volid) = @_; my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); for my $v ($source_vdisks->{$storeid}->@*) { return wantarray ? ($v->{size}, $v->{format}, $v->{used}, $v->{parent}) : $v->{size} if $v->{volid} eq $volid; } die "could not find '$volid' in mock 'source_vdisks'\n"; }, ); $MigrationTest::Shared::tools_module->mock( get_host_address_family => sub { die "get_host_address_family (mocked) - implement me\n"; # currently no call should lead here }, next_migrate_port => sub { die "next_migrate_port (mocked) - implement me\n"; # currently no call should lead here }, run_command => sub { my ($cmd_tail, %param) = @_; my $cmd_msg = to_json($cmd_tail); my $cmd = shift @{$cmd_tail}; if ($cmd =~ m@^(?:/usr/bin/)?ssh$@) { while (scalar(@{$cmd_tail})) { $cmd = shift @{$cmd_tail}; if ($cmd eq '/bin/true') { return 0; } elsif ($cmd eq 'qm') { $cmd = shift @{$cmd_tail}; if ($cmd eq 'start') { delete $expected_calls->{ssh_qm_start}; delete $vm_status->{runningmachine}; delete $vm_status->{runningcpu}; my @options = (@{$cmd_tail}); while (scalar(@options)) { my $opt = shift @options; if ($opt eq '--machine') { $vm_status->{runningmachine} = shift @options; } elsif ($opt eq '--force-cpu') { $vm_status->{runningcpu} = shift @options; } } return $MigrationTest::Shared::tools_module->original('run_command')->( [ '/usr/bin/perl', "-I${QM_LIB_PATH}", "-I${QM_LIB_PATH}/test", "${QM_LIB_PATH}/test/MigrationTest/QmMock.pm", 'start', @{$cmd_tail}, ], %param, ); } elsif ($cmd eq 'nbdstop') { delete $expected_calls->{ssh_nbdstop}; return 0; } elsif ($cmd eq 'resume') { return 0; } elsif ($cmd eq 'unlock') { my $vmid = shift @{$cmd_tail}; die "unlocking wrong vmid: $vmid\n" if $vmid ne $test_vmid; PVE::QemuConfig->remove_lock($vmid); return 0; } elsif ($cmd eq 'stop') { return 0; } die "run_command (mocked) ssh qm command - implement me: ${cmd_msg}"; } elsif ($cmd eq 'pvesm') { $cmd = shift @{$cmd_tail}; if ($cmd eq 'free') { my $volid = shift @{$cmd_tail}; PVE::Storage::parse_volume_id($volid); return 1 if $fail_config->{ssh_pvesm_free} && $fail_config->{ssh_pvesm_free} eq $volid; MigrationTest::Shared::remove_target_volid($volid); return 0; } die "run_command (mocked) ssh pvesm command - implement me: ${cmd_msg}"; } } die "run_command (mocked) ssh command - implement me: ${cmd_msg}"; } die "run_command (mocked) - implement me: ${cmd_msg}"; }, ); eval { PVE::QemuMigrate->migrate($test_target, undef, $test_vmid, $test_opts) }; my $error = $@; file_set_contents("${RUN_DIR_PATH}/source_volids", to_json($source_volids)); file_set_contents("${RUN_DIR_PATH}/vm_status", to_json($vm_status)); file_set_contents("${RUN_DIR_PATH}/expected_calls", to_json($expected_calls)); file_set_contents("${RUN_DIR_PATH}/log", $current_log); die $error if $error; 1; ================================================ FILE: src/test/MigrationTest/QmMock.pm ================================================ package MigrationTest::QmMock; use strict; use warnings; use JSON; use Test::MockModule; use MigrationTest::Shared; use PVE::API2::Qemu; use PVE::Storage; use PVE::Tools qw(file_set_contents file_get_contents); use PVE::CLIHandler; use base qw(PVE::CLIHandler); my $RUN_DIR_PATH = $ENV{RUN_DIR_PATH} or die "no RUN_DIR_PATH set\n"; my $target_volids = decode_json(file_get_contents("${RUN_DIR_PATH}/target_volids")); my $fail_config = decode_json(file_get_contents("${RUN_DIR_PATH}/fail_config")); my $migrate_params = decode_json(file_get_contents("${RUN_DIR_PATH}/migrate_params")); my $nodename = $migrate_params->{target}; my $kvm_exectued = 0; my $forcemachine; sub setup_environment { my $rpcenv = PVE::RPCEnvironment::init('MigrationTest::QmMock', 'cli'); } # mock RPCEnvironment directly sub get_user { return 'root@pam'; } sub fork_worker { my ($self, $dtype, $id, $user, $function, $background) = @_; $function->(123456); return '123456'; } # mocked modules my sub mocked_mon_cmd { my ($vmid, $command, %params) = @_; if ($command eq 'nbd-server-start') { return; } elsif ($command eq 'block-export-add') { return; } elsif ($command eq 'query-block') { return []; } elsif ($command eq 'qom-set') { return; } die "mon_cmd (mocked) - implement me: $command"; } my $inotify_module = Test::MockModule->new("PVE::INotify"); $inotify_module->mock( nodename => sub { return $nodename; }, ); my $qemu_server_blockdev_module = Test::MockModule->new("PVE::QemuServer::Blockdev"); $qemu_server_blockdev_module->mock( mon_cmd => \&mocked_mon_cmd, ); my $qemu_server_helpers_module = Test::MockModule->new("PVE::QemuServer::Helpers"); $qemu_server_helpers_module->mock( vm_running_locally => sub { return $kvm_exectued; }, ); our $qemu_server_machine_module = Test::MockModule->new("PVE::QemuServer::Machine"); $qemu_server_machine_module->mock( get_current_qemu_machine => sub { return wantarray ? ($forcemachine, 0) : $forcemachine; }, ); # to make sure we get valid and predictable names my $disk_counter = 10; $MigrationTest::Shared::storage_module->mock( vdisk_alloc => sub { my ($cfg, $storeid, $vmid, $fmt, $name, $size) = @_; die "vdisk_alloc (mocked) - name is not expected to be set - implement me\n" if defined($name); my $name_without_extension = "vm-${vmid}-disk-${disk_counter}"; $disk_counter++; my $volid; my $scfg = PVE::Storage::storage_config($cfg, $storeid); if ($scfg->{path}) { $volid = "${storeid}:${vmid}/${name_without_extension}.${fmt}"; } else { $volid = "${storeid}:${name_without_extension}"; } PVE::Storage::parse_volume_id($volid); die "vdisk_alloc '$volid' error\n" if $fail_config->{vdisk_alloc} && $fail_config->{vdisk_alloc} eq $volid; MigrationTest::Shared::add_target_volid($volid); return $volid; }, ); $MigrationTest::Shared::qemu_server_module->mock( config_to_command => sub { return ['mocked_kvm_command']; }, mon_cmd => \&mocked_mon_cmd, nodename => sub { return $nodename; }, run_command => sub { my ($cmd_full, %param) = @_; my $cmd_msg = to_json($cmd_full); my $cmd = shift @{$cmd_full}; if ($cmd eq '/bin/systemctl') { return; } elsif ($cmd eq 'mocked_kvm_command') { $kvm_exectued = 1; return 0; } die "run_command (mocked) - implement me: ${cmd_msg}"; }, vm_migrate_alloc_nbd_disks => sub { my $nbd = $MigrationTest::Shared::qemu_server_module->original('vm_migrate_alloc_nbd_disks') ->(@_); file_set_contents("${RUN_DIR_PATH}/nbd_info", to_json($nbd)); return $nbd; }, vm_start_nolock => sub { my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_; $forcemachine = $params->{forcemachine} or die "mocked vm_start_nolock - expected 'forcemachine' parameter\n"; $MigrationTest::Shared::qemu_server_module->original('vm_start_nolock')->(@_); }, ); our $cmddef = { start => ["PVE::API2::Qemu", 'vm_start', ['vmid'], { node => $nodename }], }; MigrationTest::QmMock->run_cli_handler(); 1; ================================================ FILE: src/test/MigrationTest/Shared.pm ================================================ package MigrationTest::Shared; use strict; use warnings; use JSON; use Test::MockModule; use Socket qw(AF_INET); use PVE::QemuConfig; use PVE::Tools qw(file_set_contents file_get_contents lock_file_full); my $RUN_DIR_PATH = $ENV{RUN_DIR_PATH} or die "no RUN_DIR_PATH set\n"; my $storage_config = decode_json(file_get_contents("${RUN_DIR_PATH}/storage_config")); my $replication_config = decode_json(file_get_contents("${RUN_DIR_PATH}/replication_config")); my $fail_config = decode_json(file_get_contents("${RUN_DIR_PATH}/fail_config")); my $migrate_params = decode_json(file_get_contents("${RUN_DIR_PATH}/migrate_params")); my $test_vmid = $migrate_params->{vmid}; # helpers sub add_target_volid { my ($volid) = @_; PVE::Storage::parse_volume_id($volid); lock_file_full( "${RUN_DIR_PATH}/target_volids.lock", undef, 0, sub { my $target_volids = decode_json(file_get_contents("${RUN_DIR_PATH}/target_volids")); die "target volid already present " if defined($target_volids->{$volid}); $target_volids->{$volid} = 1; file_set_contents("${RUN_DIR_PATH}/target_volids", to_json($target_volids)); }, ); die $@ if $@; } sub remove_target_volid { my ($volid) = @_; PVE::Storage::parse_volume_id($volid); lock_file_full( "${RUN_DIR_PATH}/target_volids.lock", undef, 0, sub { my $target_volids = decode_json(file_get_contents("${RUN_DIR_PATH}/target_volids")); die "target volid does not exist " if !defined($target_volids->{$volid}); delete $target_volids->{$volid}; file_set_contents("${RUN_DIR_PATH}/target_volids", to_json($target_volids)); }, ); die $@ if $@; } my $mocked_cfs_read_file = sub { my ($file) = @_; if ($file eq 'datacenter.cfg') { return {}; } elsif ($file eq 'replication.cfg') { return $replication_config; } die "cfs_read_file (mocked) - implement me: $file\n"; }; # mocked modules our $cgroup_module = Test::MockModule->new("PVE::CGroup"); $cgroup_module->mock( cgroup_mode => sub { return 2; }, ); our $cluster_module = Test::MockModule->new("PVE::Cluster"); $cluster_module->mock( cfs_read_file => $mocked_cfs_read_file, check_cfs_quorum => sub { return 1; }, ); our $mapping_usb_module = Test::MockModule->new("PVE::Mapping::USB"); $mapping_usb_module->mock( config => sub { return {}; }, ); our $mapping_pci_module = Test::MockModule->new("PVE::Mapping::PCI"); $mapping_pci_module->mock( config => sub { return {}; }, ); our $mapping_dir_module = Test::MockModule->new("PVE::Mapping::Dir"); $mapping_dir_module->mock( config => sub { return {}; }, ); our $ha_config_module = Test::MockModule->new("PVE::HA::Config"); $ha_config_module->mock( vm_is_ha_managed => sub { return 0; }, ); our $qemu_config_module = Test::MockModule->new("PVE::QemuConfig"); $qemu_config_module->mock( assert_config_exists_on_node => sub { return; }, load_config => sub { my ($class, $vmid, $node) = @_; die "trying to load wrong config: '$vmid'\n" if $vmid ne $test_vmid; return decode_json(file_get_contents("${RUN_DIR_PATH}/vm_config")); }, lock_config => sub { # no use locking here because lock is local to node my ($self, $vmid, $code, @param) = @_; return $code->(@param); }, write_config => sub { my ($class, $vmid, $conf) = @_; die "trying to write wrong config: '$vmid'\n" if $vmid ne $test_vmid; file_set_contents("${RUN_DIR_PATH}/vm_config", to_json($conf)); }, ); our $qemu_migrate_helpers_module = Test::MockModule->new("PVE::QemuMigrate::Helpers"); $qemu_migrate_helpers_module->mock( set_migration_caps => sub { return; }, ); our $qemu_server_cloudinit_module = Test::MockModule->new("PVE::QemuServer::Cloudinit"); $qemu_server_cloudinit_module->mock( generate_cloudinitconfig => sub { return; }, ); our $qemu_server_module = Test::MockModule->new("PVE::QemuServer"); $qemu_server_module->mock( clear_reboot_request => sub { return 1; }, vm_stop_cleanup => sub { return; }, ); our $qemu_server_ovmf_module = Test::MockModule->new("PVE::QemuServer::OVMF"); $qemu_server_ovmf_module->mock( get_efivars_size => sub { return 128 * 1024; }, ); our $replication_module = Test::MockModule->new("PVE::Replication"); $replication_module->mock( run_replication => sub { die "run_replication error" if $fail_config->{run_replication}; my $vm_config = PVE::QemuConfig->load_config($test_vmid); return PVE::QemuConfig->get_replicatable_volumes( $storage_config, $test_vmid, $vm_config, ); }, ); our $replication_config_module = Test::MockModule->new("PVE::ReplicationConfig"); $replication_config_module->mock( cfs_read_file => $mocked_cfs_read_file, ); our $safe_syslog_module = Test::MockModule->new("PVE::SafeSyslog"); $safe_syslog_module->mock( initlog => sub { }, syslog => sub { }, ); our $storage_module = Test::MockModule->new("PVE::Storage"); $storage_module->mock( activate_volumes => sub { return 1; }, deactivate_volumes => sub { return 1; }, config => sub { return $storage_config; }, get_bandwidth_limit => sub { return 123456; }, cfs_read_file => $mocked_cfs_read_file, ); our $storage_plugin_module = Test::MockModule->new("PVE::Storage::Plugin"); $storage_plugin_module->mock( cluster_lock_storage => sub { my ($class, $storeid, $shared, $timeout, $func, @param) = @_; mkdir "${RUN_DIR_PATH}/lock"; my $path = "${RUN_DIR_PATH}/lock/pve-storage-${storeid}"; return PVE::Tools::lock_file($path, $timeout, $func, @param); }, ); our $systemd_module = Test::MockModule->new("PVE::Systemd"); $systemd_module->mock( wait_for_unit_removed => sub { return; }, enter_systemd_scope => sub { return; }, ); my $migrate_port_counter = 60000; our $tools_module = Test::MockModule->new("PVE::Tools"); $tools_module->mock( get_host_address_family => sub { return AF_INET; }, next_migrate_port => sub { return $migrate_port_counter++; }, ); 1; ================================================ FILE: src/test/cfg2cmd/README.adoc ================================================ QemuServer Config 2 Command Test ================================ Thomas Lamprecht Overview -------- This is a relatively simple configuration to command test program. It's main goals are to better enforce stability of commands, thus reducing the likelihood that, for example, a migration breaking change which forgot to bump/check the KVM/QEMU version, slips through Further you get a certain regression and functional test coverage. You get a safety net against breaking older or not often (manual) tested setups and features. NOTE: The safety net is only as good as the test count *and* quality. Test Specification ------------------ A single test consists of two files, the input VM config `FILE.conf` and the expected output command `FILE.conf.cmd` Input ~~~~~ The `FILE.conf` are standard Proxmox VE VM configuration files, so you can just copy over a config file from `/etc/pve/qemu-server` to add a configuration you want to have tested. Output ~~~~~~ For the expected output `FILE.conf.cmd` we check the KVM/QEMU command produced. As a single long line would be pretty hard to check for (problematic) changes by humans, we use a pretty format, i.e., where each key value pair is on it's own line. With this approach we can just diff expected and actual command and one can pin point pretty fast in which component (e.g., net, drives, CPU, ...) the issue is, if any. Such an output would look like: ---- /usr/bin/kvm \ -id 101 \ -name vm101 \ ... ---- TIP: If the expected output file does not exist we have nothing to check, but for convenience we will write it out. This should happen from clean code, and the result should not get applied blindly, but only after a (quick) sanity check. Environment ~~~~~~~~~~~ It makes sense to have a stable and controlled environment for tests, thus you one can use the 'description' in VM configurations to control this. The description consists of all lines beginning with a '#' as first non-whitespace character. Any environment variable follows the following format: ---- # NAME: VALUE ... rest of config... ---- There are the following variables you can control: * *TEST*: a one line description for your test, gets outputted one testing and should described in a short way any specialty about this specific test, i.e., what does this test wants to ensure. * *QEMU_VERSION*: the version we fake for this test, if not set we use the actual one installed on the host. * *HOST_ARCH*: the architecture we should fake for the test (aarch64 or x86_64), defaults to `x86_64` to allow making this optional and still guarantee stable tests The storage environment is currently hardcoded in the test code, you can extend it there if it's needed. // vim: noai:tw=78 ================================================ FILE: src/test/cfg2cmd/aarch64/simple-arm-host.conf ================================================ # TEST: Simple test for a basic configuration with aarch64 set as the host architecture # HOST_ARCH: aarch64 bootdisk: scsi0 bios: ovmf cores: 3 ide2: none,media=cdrom memory: 768 name: simple net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 scsi0: local:8006/vm-8006-disk-0.qcow2,discard=on,size=104858K scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 efidisk0: local:8006/vm-8006-disk-1.qcow2,efitype=4m ================================================ FILE: src/test/cfg2cmd/aarch64/simple-arm-host.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//AAVMF_CODE.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-1.qcow2","node-name":"e4abf08bba72d3a54b87a58d2f50906","read-only":false},"node-name":"f4abf08bba72d3a54b87a58d2f50906","read-only":false},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu cortex-a57 \ -m 768 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pcie.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pcie.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'usb-ehci,id=ehci,bus=pcie.0,addr=0x1' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'usb-kbd,id=keyboard,bus=ehci.0,port=2' \ -device 'virtio-gpu,id=vga,bus=pcie.0,addr=0x2' \ -device 'virtio-serial,id=spice,bus=pcie.0,addr=0x9' \ -chardev 'spicevmc,id=vdagent,name=vdagent' \ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \ -device 'virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pcie.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"ecd04be4259153b8293415fefa2a84c","read-only":false},"node-name":"fcd04be4259153b8293415fefa2a84c","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pcie.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,type=virt+pve0' ================================================ FILE: src/test/cfg2cmd/aarch64/simple-arm.conf ================================================ # TEST: Simple test for a basic configuration with aarch64 set as architecture arch: aarch64 bootdisk: scsi0 bios: ovmf cores: 3 ide2: none,media=cdrom memory: 768 name: simple net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 scsi0: local:8006/vm-8006-disk-0.qcow2,discard=on,size=104858K scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 efidisk0: local:8006/vm-8006-disk-1.qcow2,efitype=4m ================================================ FILE: src/test/cfg2cmd/aarch64/simple-arm.conf.cmd ================================================ /usr/bin/qemu-system-aarch64 \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//AAVMF_CODE.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-1.qcow2","node-name":"e4abf08bba72d3a54b87a58d2f50906","read-only":false},"node-name":"f4abf08bba72d3a54b87a58d2f50906","read-only":false},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu cortex-a57 \ -m 768 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pcie.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pcie.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'usb-ehci,id=ehci,bus=pcie.0,addr=0x1' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'usb-kbd,id=keyboard,bus=ehci.0,port=2' \ -device 'virtio-gpu,id=vga,bus=pcie.0,addr=0x2' \ -device 'virtio-serial,id=spice,bus=pcie.0,addr=0x9' \ -chardev 'spicevmc,id=vdagent,name=vdagent' \ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \ -device 'virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pcie.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"ecd04be4259153b8293415fefa2a84c","read-only":false},"node-name":"fcd04be4259153b8293415fefa2a84c","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pcie.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,accel=tcg,type=virt+pve0' ================================================ FILE: src/test/cfg2cmd/aarch64/simple-x86-on-arm-host.conf ================================================ # TEST: Simple test for a basic config with aarch64 set as the host arch and x86_64 as the VM arch # HOST_ARCH: aarch64 arch: x86_64 bootdisk: scsi0 bios: ovmf cores: 3 ide2: none,media=cdrom memory: 768 name: simple net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 scsi0: local:8006/vm-8006-disk-0.qcow2,discard=on,size=104858K scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 efidisk0: local:8006/vm-8006-disk-1.qcow2,efitype=4m ================================================ FILE: src/test/cfg2cmd/aarch64/simple-x86-on-arm-host.conf.cmd ================================================ /usr/bin/qemu-system-x86_64 -id 8006 -name 'simple,debug-threads=on' -no-shutdown -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' -mon 'chardev=qmp,mode=control' -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' -mon 'chardev=qmp-event,mode=control' -pidfile /var/run/qemu-server/8006.pid -daemonize -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE_4M.fd"},"node-name":"pflash0","read-only":true}' -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-1.qcow2","node-name":"e4abf08bba72d3a54b87a58d2f50906","read-only":false},"node-name":"f4abf08bba72d3a54b87a58d2f50906","read-only":false},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' -smp '3,sockets=1,cores=3,maxcpus=3' -nodefaults -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' -cpu qemu64 -m 768 -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' -global 'PIIX4_PM.disable_s3=1' -global 'PIIX4_PM.disable_s4=1' -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' -device 'VGA,id=vga,bus=pci.0,addr=0x2' -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"ecd04be4259153b8293415fefa2a84c","read-only":false},"node-name":"fcd04be4259153b8293415fefa2a84c","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown' -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' -machine 'pflash0=pflash0,pflash1=drive-efidisk0,hpet=off,accel=tcg,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/aio.conf ================================================ scsi0: local:8006/vm-8006-disk-0.raw,discard=on,size=104858K,aio=threads scsi1: local:8006/vm-8006-disk-1.raw,discard=on,size=104858K,aio=native scsi2: local:8006/vm-8006-disk-2.raw,discard=on,size=104858K,aio=io_uring scsi3: local:8006/vm-8006-disk-3.raw,discard=on,size=104858K scsi4: cifs-store:8006/vm-8006-disk-4.raw,discard=on,size=104858K scsi5: cifs-store:8006/vm-8006-disk-5.raw,discard=on,size=104858K,aio=io_uring scsi6: krbd-store:vm-8006-disk-6,discard=on,size=104858K scsi7: krbd-store:vm-8006-disk-7,discard=on,size=104858K,aio=io_uring scsi8: krbd-store:vm-8006-disk-8,discard=on,size=104858K,cache=writeback scsi9: krbd-store:vm-8006-disk-9,discard=on,size=104858K,cache=writeback,aio=io_uring scsi10: rbd-store:vm-8006-disk-8,discard=on,size=104858K scsi11: rbd-store:vm-8006-disk-8,discard=on,size=104858K,aio=io_uring scsi12: lvm-store:vm-8006-disk-9,discard=on,size=104858K scsi13: lvm-store:vm-8006-disk-9,discard=on,size=104858K,aio=io_uring ================================================ FILE: src/test/cfg2cmd/aio.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi2","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi3","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi4","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi5","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi6","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi7","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi8","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi9","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi10","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi11","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi12","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi13","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'lsi,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.raw","node-name":"e3b2553803d55d43b9986a0aac3e9a7","read-only":false},"node-name":"f3b2553803d55d43b9986a0aac3e9a7","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-1.raw","node-name":"e08707d013893852b3d4d42301a4298","read-only":false},"node-name":"f08707d013893852b3d4d42301a4298","read-only":false},"node-name":"drive-scsi1","read-only":false,"throttle-group":"throttle-drive-scsi1"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=1,drive=drive-scsi1,id=scsi1,device_id=drive-scsi1,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-2.raw","node-name":"edb0854bba55e8b2544ad937c9f5afc","read-only":false},"node-name":"fdb0854bba55e8b2544ad937c9f5afc","read-only":false},"node-name":"drive-scsi2","read-only":false,"throttle-group":"throttle-drive-scsi2"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=2,drive=drive-scsi2,id=scsi2,device_id=drive-scsi2,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-3.raw","node-name":"e9c170cb9491763cad3f31718205efc","read-only":false},"node-name":"f9c170cb9491763cad3f31718205efc","read-only":false},"node-name":"drive-scsi3","read-only":false,"throttle-group":"throttle-drive-scsi3"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=3,drive=drive-scsi3,id=scsi3,device_id=drive-scsi3,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/mnt/pve/cifs-store/images/8006/vm-8006-disk-4.raw","node-name":"ea34ecc24c40da0d53420ef344ced37","read-only":false},"node-name":"fa34ecc24c40da0d53420ef344ced37","read-only":false},"node-name":"drive-scsi4","read-only":false,"throttle-group":"throttle-drive-scsi4"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=4,drive=drive-scsi4,id=scsi4,device_id=drive-scsi4,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/mnt/pve/cifs-store/images/8006/vm-8006-disk-5.raw","node-name":"e39cacf47a4f4877072601505d90949","read-only":false},"node-name":"f39cacf47a4f4877072601505d90949","read-only":false},"node-name":"drive-scsi5","read-only":false,"throttle-group":"throttle-drive-scsi5"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=5,drive=drive-scsi5,id=scsi5,device_id=drive-scsi5,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/rbd-pve/fc4181a6-56eb-4f68-b452-8ba1f381ca2a/cpool/vm-8006-disk-6","node-name":"e7db1ee70981087e4a2861bc7da417b","read-only":false},"node-name":"f7db1ee70981087e4a2861bc7da417b","read-only":false},"node-name":"drive-scsi6","read-only":false,"throttle-group":"throttle-drive-scsi6"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=6,drive=drive-scsi6,id=scsi6,device_id=drive-scsi6,write-cache=on' \ -device 'lsi,id=scsihw1,bus=pci.0,addr=0x6' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/rbd-pve/fc4181a6-56eb-4f68-b452-8ba1f381ca2a/cpool/vm-8006-disk-7","node-name":"e2d2deac808301140a96c862fe3ea85","read-only":false},"node-name":"f2d2deac808301140a96c862fe3ea85","read-only":false},"node-name":"drive-scsi7","read-only":false,"throttle-group":"throttle-drive-scsi7"}' \ -device 'scsi-hd,bus=scsihw1.0,scsi-id=0,drive=drive-scsi7,id=scsi7,device_id=drive-scsi7,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/rbd-pve/fc4181a6-56eb-4f68-b452-8ba1f381ca2a/cpool/vm-8006-disk-8","node-name":"e9796b73db57b8943746ede7d0d3060","read-only":false},"node-name":"f9796b73db57b8943746ede7d0d3060","read-only":false},"node-name":"drive-scsi8","read-only":false,"throttle-group":"throttle-drive-scsi8"}' \ -device 'scsi-hd,bus=scsihw1.0,scsi-id=1,drive=drive-scsi8,id=scsi8,device_id=drive-scsi8,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/rbd-pve/fc4181a6-56eb-4f68-b452-8ba1f381ca2a/cpool/vm-8006-disk-9","node-name":"efa538892acc012edbdc5810035bf7d","read-only":false},"node-name":"ffa538892acc012edbdc5810035bf7d","read-only":false},"node-name":"drive-scsi9","read-only":false,"throttle-group":"throttle-drive-scsi9"}' \ -device 'scsi-hd,bus=scsihw1.0,scsi-id=2,drive=drive-scsi9,id=scsi9,device_id=drive-scsi9,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"auth-client-required":["none"],"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"rbd","image":"vm-8006-disk-8","node-name":"e6f4cbffa741d16bba69304eb2800ef","pool":"cpool","read-only":false,"server":[{"host":"127.0.0.42","port":"3300"},{"host":"127.0.0.21","port":"3300"},{"host":"::1","port":"3300"}]},"node-name":"f6f4cbffa741d16bba69304eb2800ef","read-only":false},"node-name":"drive-scsi10","read-only":false,"throttle-group":"throttle-drive-scsi10"}' \ -device 'scsi-hd,bus=scsihw1.0,scsi-id=3,drive=drive-scsi10,id=scsi10,device_id=drive-scsi10,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"auth-client-required":["none"],"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"rbd","image":"vm-8006-disk-8","node-name":"e42375c54de70f5f4be966d98c90255","pool":"cpool","read-only":false,"server":[{"host":"127.0.0.42","port":"3300"},{"host":"127.0.0.21","port":"3300"},{"host":"::1","port":"3300"}]},"node-name":"f42375c54de70f5f4be966d98c90255","read-only":false},"node-name":"drive-scsi11","read-only":false,"throttle-group":"throttle-drive-scsi11"}' \ -device 'scsi-hd,bus=scsihw1.0,scsi-id=4,drive=drive-scsi11,id=scsi11,device_id=drive-scsi11,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-9","node-name":"ed7b2c9e0133619fcf6cb8ce5903502","read-only":false},"node-name":"fd7b2c9e0133619fcf6cb8ce5903502","read-only":false},"node-name":"drive-scsi12","read-only":false,"throttle-group":"throttle-drive-scsi12"}' \ -device 'scsi-hd,bus=scsihw1.0,scsi-id=5,drive=drive-scsi12,id=scsi12,device_id=drive-scsi12,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-9","node-name":"ed85420a880203ca1401d00a8edf132","read-only":false},"node-name":"fd85420a880203ca1401d00a8edf132","read-only":false},"node-name":"drive-scsi13","read-only":false,"throttle-group":"throttle-drive-scsi13"}' \ -device 'scsi-hd,bus=scsihw1.0,scsi-id=6,drive=drive-scsi13,id=scsi13,device_id=drive-scsi13,write-cache=on' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/bootorder-empty.conf ================================================ # TEST: Test for an empty boot parameter producing no bootindices either cores: 3 boot: ide2: none,media=cdrom memory: 768 name: simple net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 scsi4: local:8006/vm-8006-disk-0.qcow2,discard=on,size=104858K smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 virtio0: local:8006/vm-8006-disk-0.qcow2,discard=on,iothread=1,size=104858K virtio1: local:8006/vm-8006-disk-0.qcow2,discard=on,iothread=1,size=104858K vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 ================================================ FILE: src/test/cfg2cmd/bootorder-empty.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -object '{"id":"throttle-drive-scsi4","limits":{},"qom-type":"throttle-group"}' \ -object 'iothread,id=iothread-virtio0' \ -object '{"id":"throttle-drive-virtio0","limits":{},"qom-type":"throttle-group"}' \ -object 'iothread,id=iothread-virtio1' \ -object '{"id":"throttle-drive-virtio1","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'ide-cd,bus=ide.1,unit=0,id=ide2' \ -device 'lsi,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e6bf62e20f6c14a2c19bd6f1f5ac36c","read-only":false},"node-name":"f6bf62e20f6c14a2c19bd6f1f5ac36c","read-only":false},"node-name":"drive-scsi4","read-only":false,"throttle-group":"throttle-drive-scsi4"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=4,drive=drive-scsi4,id=scsi4,device_id=drive-scsi4,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"edd19f6c1b3a6d5a6248c3376a91a16","read-only":false},"node-name":"fdd19f6c1b3a6d5a6248c3376a91a16","read-only":false},"node-name":"drive-virtio0","read-only":false,"throttle-group":"throttle-drive-virtio0"}' \ -device 'virtio-blk-pci,drive=drive-virtio0,id=virtio0,bus=pci.0,addr=0xa,iothread=iothread-virtio0,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"eeb683fb9c516c1a8707c917f0d7a38","read-only":false},"node-name":"feb683fb9c516c1a8707c917f0d7a38","read-only":false},"node-name":"drive-virtio1","read-only":false,"throttle-group":"throttle-drive-virtio1"}' \ -device 'virtio-blk-pci,drive=drive-virtio1,id=virtio1,bus=pci.0,addr=0xb,iothread=iothread-virtio1,write-cache=on' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,host_mtu=1500' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/bootorder-legacy.conf ================================================ # TEST: Test for a specific bootorder given by legacy 'boot' value cores: 3 boot: ndca bootdisk: virtio1 ide2: none,media=cdrom memory: 768 name: simple net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 scsi4: local:8006/vm-8006-disk-0.qcow2,discard=on,size=104858K smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 virtio0: local:8006/vm-8006-disk-0.qcow2,discard=on,iothread=1,size=104858K virtio1: local:8006/vm-8006-disk-0.qcow2,discard=on,iothread=1,size=104858K vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 ================================================ FILE: src/test/cfg2cmd/bootorder-legacy.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -object '{"id":"throttle-drive-scsi4","limits":{},"qom-type":"throttle-group"}' \ -object 'iothread,id=iothread-virtio0' \ -object '{"id":"throttle-drive-virtio0","limits":{},"qom-type":"throttle-group"}' \ -object 'iothread,id=iothread-virtio1' \ -object '{"id":"throttle-drive-virtio1","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \ -device 'lsi,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e6bf62e20f6c14a2c19bd6f1f5ac36c","read-only":false},"node-name":"f6bf62e20f6c14a2c19bd6f1f5ac36c","read-only":false},"node-name":"drive-scsi4","read-only":false,"throttle-group":"throttle-drive-scsi4"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=4,drive=drive-scsi4,id=scsi4,device_id=drive-scsi4,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"edd19f6c1b3a6d5a6248c3376a91a16","read-only":false},"node-name":"fdd19f6c1b3a6d5a6248c3376a91a16","read-only":false},"node-name":"drive-virtio0","read-only":false,"throttle-group":"throttle-drive-virtio0"}' \ -device 'virtio-blk-pci,drive=drive-virtio0,id=virtio0,bus=pci.0,addr=0xa,iothread=iothread-virtio0,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"eeb683fb9c516c1a8707c917f0d7a38","read-only":false},"node-name":"feb683fb9c516c1a8707c917f0d7a38","read-only":false},"node-name":"drive-virtio1","read-only":false,"throttle-group":"throttle-drive-virtio1"}' \ -device 'virtio-blk-pci,drive=drive-virtio1,id=virtio1,bus=pci.0,addr=0xb,iothread=iothread-virtio1,bootindex=302,write-cache=on' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=100,host_mtu=1500' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/bootorder.conf ================================================ # TEST: Test for a specific bootorder given by 'boot: order=' property cores: 3 boot: order=virtio1;net0;scsi4;ide2 ide2: none,media=cdrom memory: 768 name: simple net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 scsi4: local:8006/vm-8006-disk-0.qcow2,discard=on,size=104858K smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 virtio0: local:8006/vm-8006-disk-0.qcow2,discard=on,iothread=1,size=104858K virtio1: local:8006/vm-8006-disk-0.qcow2,discard=on,iothread=1,size=104858K vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 ================================================ FILE: src/test/cfg2cmd/bootorder.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -object '{"id":"throttle-drive-scsi4","limits":{},"qom-type":"throttle-group"}' \ -object 'iothread,id=iothread-virtio0' \ -object '{"id":"throttle-drive-virtio0","limits":{},"qom-type":"throttle-group"}' \ -object 'iothread,id=iothread-virtio1' \ -object '{"id":"throttle-drive-virtio1","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=103' \ -device 'lsi,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e6bf62e20f6c14a2c19bd6f1f5ac36c","read-only":false},"node-name":"f6bf62e20f6c14a2c19bd6f1f5ac36c","read-only":false},"node-name":"drive-scsi4","read-only":false,"throttle-group":"throttle-drive-scsi4"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=4,drive=drive-scsi4,id=scsi4,device_id=drive-scsi4,bootindex=102,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"edd19f6c1b3a6d5a6248c3376a91a16","read-only":false},"node-name":"fdd19f6c1b3a6d5a6248c3376a91a16","read-only":false},"node-name":"drive-virtio0","read-only":false,"throttle-group":"throttle-drive-virtio0"}' \ -device 'virtio-blk-pci,drive=drive-virtio0,id=virtio0,bus=pci.0,addr=0xa,iothread=iothread-virtio0,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"eeb683fb9c516c1a8707c917f0d7a38","read-only":false},"node-name":"feb683fb9c516c1a8707c917f0d7a38","read-only":false},"node-name":"drive-virtio1","read-only":false,"throttle-group":"throttle-drive-virtio1"}' \ -device 'virtio-blk-pci,drive=drive-virtio1,id=virtio1,bus=pci.0,addr=0xb,iothread=iothread-virtio1,bootindex=100,write-cache=on' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=101,host_mtu=1500' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/cputype-icelake-client-deprecation.conf ================================================ # TEST: test CPU type deprecation for Icelake-Client (never existed in the wild) bootdisk: scsi0 cores: 2 cpu: Icelake-Client ide2: none,media=cdrom memory: 768 name: simple ostype: l26 scsi0: local:8006/base-8006-disk-0.qcow2,discard=on,size=104858K scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 ================================================ FILE: src/test/cfg2cmd/cputype-icelake-client-deprecation.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu 'Icelake-Server,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,vendor=GenuineIntel' \ -m 768 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/base-8006-disk-0.qcow2","node-name":"e417d5947e69c5890b1e3ddf8a68167","read-only":false},"node-name":"f417d5947e69c5890b1e3ddf8a68167","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/custom-cpu-model-defaults.conf ================================================ # TEST: Check if custom CPU models are resolving defaults correctly cores: 3 cpu: custom-alldefault name: customcpu-defaults numa: 0 ostype: l26 smbios1: uuid=2ea3f676-dfa5-11e9-ae82-c721e12f3fce sockets: 1 ================================================ FILE: src/test/cfg2cmd/custom-cpu-model-defaults.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'customcpu-defaults,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=2ea3f676-dfa5-11e9-ae82-c721e12f3fce' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/custom-cpu-model-host-phys-bits.conf ================================================ # TEST: Check if custom CPU models are resolved correctly # EXPECT_WARN: warning: CPU flag/setting '-kvm_pv_unhalt' (set by custom CPU model) overwrites '+kvm_pv_unhalt' (set by PVE; to improve Linux guest spinlock performance) cores: 3 cpu: custom-qemu64,phys-bits=host name: customcpu numa: 0 ostype: win10 smbios1: uuid=2ea3f676-dfa5-11e9-ae82-c721e12f3fcf sockets: 1 ================================================ FILE: src/test/cfg2cmd/custom-cpu-model-host-phys-bits.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'customcpu,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=2ea3f676-dfa5-11e9-ae82-c721e12f3fcf' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -global 'kvm-pit.lost_tick_policy=discard' \ -cpu 'athlon,+aes,+avx,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vendor_id=testvend,hv_vpindex,+kvm_pv_eoi,-kvm_pv_unhalt,vendor=AuthenticAMD,host-phys-bits=true' \ -m 512 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2,edid=off' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -rtc 'driftfix=slew,base=localtime' \ -machine 'hpet=off,type=pc-i440fx-5.1+pve0' ================================================ FILE: src/test/cfg2cmd/custom-cpu-model.conf ================================================ # TEST: Check if custom CPU models are resolved correctly # EXPECT_WARN: warning: CPU flag/setting '-kvm_pv_unhalt' (set by custom CPU model) overwrites '+kvm_pv_unhalt' (set by PVE; to improve Linux guest spinlock performance) cores: 3 cpu: custom-qemu64,flags=+virt-ssbd name: customcpu numa: 0 ostype: win10 smbios1: uuid=2ea3f676-dfa5-11e9-ae82-c721e12f3fcf sockets: 1 ================================================ FILE: src/test/cfg2cmd/custom-cpu-model.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'customcpu,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=2ea3f676-dfa5-11e9-ae82-c721e12f3fcf' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -global 'kvm-pit.lost_tick_policy=discard' \ -cpu 'athlon,+aes,+avx,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vendor_id=testvend,hv_vpindex,+kvm_pv_eoi,-kvm_pv_unhalt,vendor=AuthenticAMD,+virt-ssbd,phys-bits=40' \ -m 512 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2,edid=off' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -rtc 'driftfix=slew,base=localtime' \ -machine 'hpet=off,type=pc-i440fx-5.1+pve0' ================================================ FILE: src/test/cfg2cmd/efi-ovmf-without-efidisk.conf ================================================ # TEST: Test VM with OVMF/EFI but no persistent efidisk for backward compat. smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 bios: ovmf ================================================ FILE: src/test/cfg2cmd/efi-ovmf-without-efidisk.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/tmp/8006-ovmf.fd","node-name":"e5b5f7a29888341a35f0f1428e70ba5","read-only":false},"node-name":"f5b5f7a29888341a35f0f1428e70ba5","read-only":false,"size":131072},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/efi-raw-old.conf ================================================ # TEST: Test raw efidisk size parameter on old version smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 bios: ovmf machine: pc-i440fx-4.1+pve0 efidisk0: local:100/vm-100-disk-0.raw ================================================ FILE: src/test/cfg2cmd/efi-raw-old.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-100-disk-0.raw' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=pc-i440fx-4.1+pve0' ================================================ FILE: src/test/cfg2cmd/efi-raw-template.conf ================================================ # TEST: Test raw efidisk size parameter smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 bios: ovmf efidisk0: local:100/base-100-disk-0.raw template: 1 ================================================ FILE: src/test/cfg2cmd/efi-raw-template.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/base-100-disk-0.raw","node-name":"e2ab65c8ec567acbeb645244f6c4982","read-only":true},"node-name":"f2ab65c8ec567acbeb645244f6c4982","read-only":true,"size":131072},"node-name":"drive-efidisk0","read-only":true,"throttle-group":"throttle-drive-efidisk0"}' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vga none \ -nographic \ -cpu qemu64 \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,accel=tcg,type=pc+pve0' \ -snapshot ================================================ FILE: src/test/cfg2cmd/efi-raw.conf ================================================ # TEST: Test raw efidisk size parameter smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 bios: ovmf efidisk0: local:100/vm-100-disk-0.raw ================================================ FILE: src/test/cfg2cmd/efi-raw.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-0.raw","node-name":"e1175f2a490414e7c53337589fde17a","read-only":false},"node-name":"f1175f2a490414e7c53337589fde17a","read-only":false,"size":131072},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/efi-secboot-and-tpm-q35.conf ================================================ # TEST: Test newer 4MB efidisk with secureboot, smm enforce and a TPM device on Q35 smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 bios: ovmf machine: q35 efidisk0: local:100/vm-100-disk-0.raw,efitype=4m,pre-enrolled-keys=1,size=528K tpmstate0: local:108/vm-100-disk-1.raw,size=4M,version=v2.0 ================================================ FILE: src/test/cfg2cmd/efi-secboot-and-tpm-q35.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE_4M.secboot.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-0.raw","node-name":"e1175f2a490414e7c53337589fde17a","read-only":false},"node-name":"f1175f2a490414e7c53337589fde17a","read-only":false,"size":540672},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'ICH9-LPC.disable_s3=1' \ -global 'ICH9-LPC.disable_s4=1' \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -chardev 'socket,id=tpmchar,path=/var/run/qemu-server/8006.swtpm' \ -tpmdev 'emulator,id=tpmdev,chardev=tpmchar' \ -device 'tpm-tis,tpmdev=tpmdev' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,type=q35+pve0' ================================================ FILE: src/test/cfg2cmd/efi-secboot-and-tpm.conf ================================================ # TEST: Test newer 4MB efidisk with secureboot and a TPM device smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 bios: ovmf efidisk0: local:100/vm-100-disk-0.raw,efitype=4m,pre-enrolled-keys=1,size=528K tpmstate0: local:108/vm-100-disk-1.raw,size=4M,version=v2.0 ================================================ FILE: src/test/cfg2cmd/efi-secboot-and-tpm.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE_4M.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-0.raw","node-name":"e1175f2a490414e7c53337589fde17a","read-only":false},"node-name":"f1175f2a490414e7c53337589fde17a","read-only":false,"size":540672},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -chardev 'socket,id=tpmchar,path=/var/run/qemu-server/8006.swtpm' \ -tpmdev 'emulator,id=tpmdev,chardev=tpmchar' \ -device 'tpm-tis,tpmdev=tpmdev' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/efidisk-on-rbd.conf ================================================ # TEST: Config with efi disk on RBD is very slow without cache - #3329 bios: ovmf bootdisk: scsi0 cores: 1 efidisk0: rbd-store:vm-100-disk-1,size=128K memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 numa: 1 ostype: l26 smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e688 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13a ================================================ FILE: src/test/cfg2cmd/efidisk-on-rbd.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e688' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"raw","file":{"auth-client-required":["none"],"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"rbd","image":"vm-100-disk-1","node-name":"eeb8f022b5551ad1d795611f112c767","pool":"cpool","read-only":false,"server":[{"host":"127.0.0.42","port":"3300"},{"host":"127.0.0.21","port":"3300"},{"host":"::1","port":"3300"}]},"node-name":"feb8f022b5551ad1d795611f112c767","read-only":false,"size":131072},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object 'memory-backend-ram,id=ram-node0,size=512M' \ -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13a' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/i440fx-viommu-intel.conf ================================================ # EXPECT_ERROR: to use Intel vIOMMU please set the machine type to q35 machine: pc,viommu=intel ================================================ FILE: src/test/cfg2cmd/i440fx-viommu-virtio.conf ================================================ machine: pc,viommu=virtio ================================================ FILE: src/test/cfg2cmd/i440fx-viommu-virtio.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device virtio-iommu-pci \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/i440fx-win10-hostpci.conf ================================================ # TEST: Config with i440fx, NUMA, hostpci passthrough, EFI & Windows bios: ovmf bootdisk: scsi0 cores: 1 efidisk0: local:100/vm-100-disk-1.qcow2,size=128K hostpci0: 0f:f2.0 machine: pc memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 numa: 1 ostype: win10 scsihw: virtio-scsi-pci smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 sockets: 2 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/i440fx-win10-hostpci.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ -smp '2,sockets=2,cores=1,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -global 'kvm-pit.lost_tick_policy=discard' \ -cpu 'kvm64,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vpindex,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep' \ -m 512 \ -object 'memory-backend-ram,id=ram-node0,size=256M' \ -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \ -object 'memory-backend-ram,id=ram-node1,size=256M' \ -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'vfio-pci,host=0000:0f:f2.0,id=hostpci0,bus=pci.0,addr=0x10' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -rtc 'driftfix=slew,base=localtime' \ -machine 'hpet=off,type=pc-i440fx-5.1+pve0' ================================================ FILE: src/test/cfg2cmd/ide-no-media-error.conf ================================================ # TEST: Config that doesn't specify an explicit `media` for an ISO drive # EXPECT_ERROR: ide3: explicit media parameter is required for iso images bootdisk: scsi0 cores: 2 ide0: cifs-store:iso/zero.iso,media=cdrom,size=112M ide1: cifs-store:iso/one.iso,media=cdrom,size=112M ide2: cifs-store:iso/two.iso,media=disk,size=112M ide3: cifs-store:iso/three.iso,size=112M memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 ostype: l26 scsi0: local:100/vm-100-disk-2.qcow2,size=10G scsihw: virtio-scsi-pci smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/ide.conf ================================================ # TEST: Config with default machine type, Linux & four IDE CD-ROMs bootdisk: scsi0 cores: 2 ide0: local:iso/zero.iso,media=cdrom,size=112M ide1: cifs-store:iso/one.iso,media=cdrom,size=112M ide2: cifs-store:iso/two.iso,media=cdrom,size=112M ide3: cifs-store:iso/three.iso,media=cdrom,size=112M memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 ostype: l26 scsi0: local:100/vm-100-disk-2.qcow2,size=10G scsihw: virtio-scsi-pci smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/ide.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object '{"id":"throttle-drive-ide0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-ide1","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-ide2","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-ide3","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/var/lib/vz/template/iso/zero.iso","node-name":"e19e15bf93b8cf09e2a5d1669648165","read-only":true},"node-name":"f19e15bf93b8cf09e2a5d1669648165","read-only":true},"node-name":"drive-ide0","read-only":true,"throttle-group":"throttle-drive-ide0"}' \ -device 'ide-cd,bus=ide.0,unit=0,drive=drive-ide0,id=ide0,bootindex=200' \ -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/one.iso","node-name":"e247a6535f864815c8e9dbb5a118336","read-only":true},"node-name":"f247a6535f864815c8e9dbb5a118336","read-only":true},"node-name":"drive-ide1","read-only":true,"throttle-group":"throttle-drive-ide1"}' \ -device 'ide-cd,bus=ide.0,unit=1,drive=drive-ide1,id=ide1,bootindex=201' \ -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/two.iso","node-name":"ec78a64f692c2fa9f873a111580aebd","read-only":true},"node-name":"fc78a64f692c2fa9f873a111580aebd","read-only":true},"node-name":"drive-ide2","read-only":true,"throttle-group":"throttle-drive-ide2"}' \ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=202' \ -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/three.iso","node-name":"e35557bae4bcbf9edc9f7ff7f132f30","read-only":true},"node-name":"f35557bae4bcbf9edc9f7ff7f132f30","read-only":true},"node-name":"drive-ide3","read-only":true,"throttle-group":"throttle-drive-ide3"}' \ -device 'ide-cd,bus=ide.1,unit=1,drive=drive-ide3,id=ide3,bootindex=203' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-2.qcow2","node-name":"ec11e0572184321efc5835152b95d5d","read-only":false},"node-name":"fc11e0572184321efc5835152b95d5d","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/memory-hotplug-hugepages.conf ================================================ # TEST: memory hotplug with 1GB hugepage cores: 2 memory: 18432 name: simple numa: 1 ostype: l26 smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 2 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 hotplug: memory hugepages: 1024 ================================================ FILE: src/test/cfg2cmd/memory-hotplug-hugepages.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '4,sockets=2,cores=2,maxcpus=4' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 'size=2048,slots=255,maxmem=524288M' \ -object 'memory-backend-file,id=ram-node0,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -numa 'node,nodeid=0,cpus=0-1,memdev=ram-node0' \ -object 'memory-backend-file,id=ram-node1,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -numa 'node,nodeid=1,cpus=2-3,memdev=ram-node1' \ -object 'memory-backend-file,id=mem-dimm0,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm0,memdev=mem-dimm0,node=0' \ -object 'memory-backend-file,id=mem-dimm1,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm1,memdev=mem-dimm1,node=1' \ -object 'memory-backend-file,id=mem-dimm2,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm2,memdev=mem-dimm2,node=0' \ -object 'memory-backend-file,id=mem-dimm3,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm3,memdev=mem-dimm3,node=1' \ -object 'memory-backend-file,id=mem-dimm4,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm4,memdev=mem-dimm4,node=0' \ -object 'memory-backend-file,id=mem-dimm5,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm5,memdev=mem-dimm5,node=1' \ -object 'memory-backend-file,id=mem-dimm6,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm6,memdev=mem-dimm6,node=0' \ -object 'memory-backend-file,id=mem-dimm7,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm7,memdev=mem-dimm7,node=1' \ -object 'memory-backend-file,id=mem-dimm8,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm8,memdev=mem-dimm8,node=0' \ -object 'memory-backend-file,id=mem-dimm9,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm9,memdev=mem-dimm9,node=1' \ -object 'memory-backend-file,id=mem-dimm10,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm10,memdev=mem-dimm10,node=0' \ -object 'memory-backend-file,id=mem-dimm11,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm11,memdev=mem-dimm11,node=1' \ -object 'memory-backend-file,id=mem-dimm12,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm12,memdev=mem-dimm12,node=0' \ -object 'memory-backend-file,id=mem-dimm13,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm13,memdev=mem-dimm13,node=1' \ -object 'memory-backend-file,id=mem-dimm14,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm14,memdev=mem-dimm14,node=0' \ -object 'memory-backend-file,id=mem-dimm15,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -device 'pc-dimm,id=dimm15,memdev=mem-dimm15,node=1' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/memory-hotplug.conf ================================================ # TEST: basic memory hotplug cores: 2 memory: 66560 name: simple numa: 1 ostype: l26 smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 2 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 hotplug: memory ================================================ FILE: src/test/cfg2cmd/memory-hotplug.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '4,sockets=2,cores=2,maxcpus=4' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 'size=1024,slots=255,maxmem=524288M' \ -object 'memory-backend-ram,id=ram-node0,size=512M' \ -numa 'node,nodeid=0,cpus=0-1,memdev=ram-node0' \ -object 'memory-backend-ram,id=ram-node1,size=512M' \ -numa 'node,nodeid=1,cpus=2-3,memdev=ram-node1' \ -object 'memory-backend-ram,id=mem-dimm0,size=512M' \ -device 'pc-dimm,id=dimm0,memdev=mem-dimm0,node=0' \ -object 'memory-backend-ram,id=mem-dimm1,size=512M' \ -device 'pc-dimm,id=dimm1,memdev=mem-dimm1,node=1' \ -object 'memory-backend-ram,id=mem-dimm2,size=512M' \ -device 'pc-dimm,id=dimm2,memdev=mem-dimm2,node=0' \ -object 'memory-backend-ram,id=mem-dimm3,size=512M' \ -device 'pc-dimm,id=dimm3,memdev=mem-dimm3,node=1' \ -object 'memory-backend-ram,id=mem-dimm4,size=512M' \ -device 'pc-dimm,id=dimm4,memdev=mem-dimm4,node=0' \ -object 'memory-backend-ram,id=mem-dimm5,size=512M' \ -device 'pc-dimm,id=dimm5,memdev=mem-dimm5,node=1' \ -object 'memory-backend-ram,id=mem-dimm6,size=512M' \ -device 'pc-dimm,id=dimm6,memdev=mem-dimm6,node=0' \ -object 'memory-backend-ram,id=mem-dimm7,size=512M' \ -device 'pc-dimm,id=dimm7,memdev=mem-dimm7,node=1' \ -object 'memory-backend-ram,id=mem-dimm8,size=512M' \ -device 'pc-dimm,id=dimm8,memdev=mem-dimm8,node=0' \ -object 'memory-backend-ram,id=mem-dimm9,size=512M' \ -device 'pc-dimm,id=dimm9,memdev=mem-dimm9,node=1' \ -object 'memory-backend-ram,id=mem-dimm10,size=512M' \ -device 'pc-dimm,id=dimm10,memdev=mem-dimm10,node=0' \ -object 'memory-backend-ram,id=mem-dimm11,size=512M' \ -device 'pc-dimm,id=dimm11,memdev=mem-dimm11,node=1' \ -object 'memory-backend-ram,id=mem-dimm12,size=512M' \ -device 'pc-dimm,id=dimm12,memdev=mem-dimm12,node=0' \ -object 'memory-backend-ram,id=mem-dimm13,size=512M' \ -device 'pc-dimm,id=dimm13,memdev=mem-dimm13,node=1' \ -object 'memory-backend-ram,id=mem-dimm14,size=512M' \ -device 'pc-dimm,id=dimm14,memdev=mem-dimm14,node=0' \ -object 'memory-backend-ram,id=mem-dimm15,size=512M' \ -device 'pc-dimm,id=dimm15,memdev=mem-dimm15,node=1' \ -object 'memory-backend-ram,id=mem-dimm16,size=512M' \ -device 'pc-dimm,id=dimm16,memdev=mem-dimm16,node=0' \ -object 'memory-backend-ram,id=mem-dimm17,size=512M' \ -device 'pc-dimm,id=dimm17,memdev=mem-dimm17,node=1' \ -object 'memory-backend-ram,id=mem-dimm18,size=512M' \ -device 'pc-dimm,id=dimm18,memdev=mem-dimm18,node=0' \ -object 'memory-backend-ram,id=mem-dimm19,size=512M' \ -device 'pc-dimm,id=dimm19,memdev=mem-dimm19,node=1' \ -object 'memory-backend-ram,id=mem-dimm20,size=512M' \ -device 'pc-dimm,id=dimm20,memdev=mem-dimm20,node=0' \ -object 'memory-backend-ram,id=mem-dimm21,size=512M' \ -device 'pc-dimm,id=dimm21,memdev=mem-dimm21,node=1' \ -object 'memory-backend-ram,id=mem-dimm22,size=512M' \ -device 'pc-dimm,id=dimm22,memdev=mem-dimm22,node=0' \ -object 'memory-backend-ram,id=mem-dimm23,size=512M' \ -device 'pc-dimm,id=dimm23,memdev=mem-dimm23,node=1' \ -object 'memory-backend-ram,id=mem-dimm24,size=512M' \ -device 'pc-dimm,id=dimm24,memdev=mem-dimm24,node=0' \ -object 'memory-backend-ram,id=mem-dimm25,size=512M' \ -device 'pc-dimm,id=dimm25,memdev=mem-dimm25,node=1' \ -object 'memory-backend-ram,id=mem-dimm26,size=512M' \ -device 'pc-dimm,id=dimm26,memdev=mem-dimm26,node=0' \ -object 'memory-backend-ram,id=mem-dimm27,size=512M' \ -device 'pc-dimm,id=dimm27,memdev=mem-dimm27,node=1' \ -object 'memory-backend-ram,id=mem-dimm28,size=512M' \ -device 'pc-dimm,id=dimm28,memdev=mem-dimm28,node=0' \ -object 'memory-backend-ram,id=mem-dimm29,size=512M' \ -device 'pc-dimm,id=dimm29,memdev=mem-dimm29,node=1' \ -object 'memory-backend-ram,id=mem-dimm30,size=512M' \ -device 'pc-dimm,id=dimm30,memdev=mem-dimm30,node=0' \ -object 'memory-backend-ram,id=mem-dimm31,size=512M' \ -device 'pc-dimm,id=dimm31,memdev=mem-dimm31,node=1' \ -object 'memory-backend-ram,id=mem-dimm32,size=1024M' \ -device 'pc-dimm,id=dimm32,memdev=mem-dimm32,node=0' \ -object 'memory-backend-ram,id=mem-dimm33,size=1024M' \ -device 'pc-dimm,id=dimm33,memdev=mem-dimm33,node=1' \ -object 'memory-backend-ram,id=mem-dimm34,size=1024M' \ -device 'pc-dimm,id=dimm34,memdev=mem-dimm34,node=0' \ -object 'memory-backend-ram,id=mem-dimm35,size=1024M' \ -device 'pc-dimm,id=dimm35,memdev=mem-dimm35,node=1' \ -object 'memory-backend-ram,id=mem-dimm36,size=1024M' \ -device 'pc-dimm,id=dimm36,memdev=mem-dimm36,node=0' \ -object 'memory-backend-ram,id=mem-dimm37,size=1024M' \ -device 'pc-dimm,id=dimm37,memdev=mem-dimm37,node=1' \ -object 'memory-backend-ram,id=mem-dimm38,size=1024M' \ -device 'pc-dimm,id=dimm38,memdev=mem-dimm38,node=0' \ -object 'memory-backend-ram,id=mem-dimm39,size=1024M' \ -device 'pc-dimm,id=dimm39,memdev=mem-dimm39,node=1' \ -object 'memory-backend-ram,id=mem-dimm40,size=1024M' \ -device 'pc-dimm,id=dimm40,memdev=mem-dimm40,node=0' \ -object 'memory-backend-ram,id=mem-dimm41,size=1024M' \ -device 'pc-dimm,id=dimm41,memdev=mem-dimm41,node=1' \ -object 'memory-backend-ram,id=mem-dimm42,size=1024M' \ -device 'pc-dimm,id=dimm42,memdev=mem-dimm42,node=0' \ -object 'memory-backend-ram,id=mem-dimm43,size=1024M' \ -device 'pc-dimm,id=dimm43,memdev=mem-dimm43,node=1' \ -object 'memory-backend-ram,id=mem-dimm44,size=1024M' \ -device 'pc-dimm,id=dimm44,memdev=mem-dimm44,node=0' \ -object 'memory-backend-ram,id=mem-dimm45,size=1024M' \ -device 'pc-dimm,id=dimm45,memdev=mem-dimm45,node=1' \ -object 'memory-backend-ram,id=mem-dimm46,size=1024M' \ -device 'pc-dimm,id=dimm46,memdev=mem-dimm46,node=0' \ -object 'memory-backend-ram,id=mem-dimm47,size=1024M' \ -device 'pc-dimm,id=dimm47,memdev=mem-dimm47,node=1' \ -object 'memory-backend-ram,id=mem-dimm48,size=1024M' \ -device 'pc-dimm,id=dimm48,memdev=mem-dimm48,node=0' \ -object 'memory-backend-ram,id=mem-dimm49,size=1024M' \ -device 'pc-dimm,id=dimm49,memdev=mem-dimm49,node=1' \ -object 'memory-backend-ram,id=mem-dimm50,size=1024M' \ -device 'pc-dimm,id=dimm50,memdev=mem-dimm50,node=0' \ -object 'memory-backend-ram,id=mem-dimm51,size=1024M' \ -device 'pc-dimm,id=dimm51,memdev=mem-dimm51,node=1' \ -object 'memory-backend-ram,id=mem-dimm52,size=1024M' \ -device 'pc-dimm,id=dimm52,memdev=mem-dimm52,node=0' \ -object 'memory-backend-ram,id=mem-dimm53,size=1024M' \ -device 'pc-dimm,id=dimm53,memdev=mem-dimm53,node=1' \ -object 'memory-backend-ram,id=mem-dimm54,size=1024M' \ -device 'pc-dimm,id=dimm54,memdev=mem-dimm54,node=0' \ -object 'memory-backend-ram,id=mem-dimm55,size=1024M' \ -device 'pc-dimm,id=dimm55,memdev=mem-dimm55,node=1' \ -object 'memory-backend-ram,id=mem-dimm56,size=1024M' \ -device 'pc-dimm,id=dimm56,memdev=mem-dimm56,node=0' \ -object 'memory-backend-ram,id=mem-dimm57,size=1024M' \ -device 'pc-dimm,id=dimm57,memdev=mem-dimm57,node=1' \ -object 'memory-backend-ram,id=mem-dimm58,size=1024M' \ -device 'pc-dimm,id=dimm58,memdev=mem-dimm58,node=0' \ -object 'memory-backend-ram,id=mem-dimm59,size=1024M' \ -device 'pc-dimm,id=dimm59,memdev=mem-dimm59,node=1' \ -object 'memory-backend-ram,id=mem-dimm60,size=1024M' \ -device 'pc-dimm,id=dimm60,memdev=mem-dimm60,node=0' \ -object 'memory-backend-ram,id=mem-dimm61,size=1024M' \ -device 'pc-dimm,id=dimm61,memdev=mem-dimm61,node=1' \ -object 'memory-backend-ram,id=mem-dimm62,size=1024M' \ -device 'pc-dimm,id=dimm62,memdev=mem-dimm62,node=0' \ -object 'memory-backend-ram,id=mem-dimm63,size=1024M' \ -device 'pc-dimm,id=dimm63,memdev=mem-dimm63,node=1' \ -object 'memory-backend-ram,id=mem-dimm64,size=2048M' \ -device 'pc-dimm,id=dimm64,memdev=mem-dimm64,node=0' \ -object 'memory-backend-ram,id=mem-dimm65,size=2048M' \ -device 'pc-dimm,id=dimm65,memdev=mem-dimm65,node=1' \ -object 'memory-backend-ram,id=mem-dimm66,size=2048M' \ -device 'pc-dimm,id=dimm66,memdev=mem-dimm66,node=0' \ -object 'memory-backend-ram,id=mem-dimm67,size=2048M' \ -device 'pc-dimm,id=dimm67,memdev=mem-dimm67,node=1' \ -object 'memory-backend-ram,id=mem-dimm68,size=2048M' \ -device 'pc-dimm,id=dimm68,memdev=mem-dimm68,node=0' \ -object 'memory-backend-ram,id=mem-dimm69,size=2048M' \ -device 'pc-dimm,id=dimm69,memdev=mem-dimm69,node=1' \ -object 'memory-backend-ram,id=mem-dimm70,size=2048M' \ -device 'pc-dimm,id=dimm70,memdev=mem-dimm70,node=0' \ -object 'memory-backend-ram,id=mem-dimm71,size=2048M' \ -device 'pc-dimm,id=dimm71,memdev=mem-dimm71,node=1' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/memory-hugepages-1g.conf ================================================ # TEST: memory with 1gb hugepages cores: 2 memory: 8192 name: simple numa: 1 ostype: l26 smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 2 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 hugepages: 1024 ================================================ FILE: src/test/cfg2cmd/memory-hugepages-1g.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '4,sockets=2,cores=2,maxcpus=4' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 8192 \ -object 'memory-backend-file,id=ram-node0,size=4096M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -numa 'node,nodeid=0,cpus=0-1,memdev=ram-node0' \ -object 'memory-backend-file,id=ram-node1,size=4096M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ -numa 'node,nodeid=1,cpus=2-3,memdev=ram-node1' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/memory-hugepages-2m.conf ================================================ # TEST: memory with 2mb hugepages cores: 2 memory: 8192 name: simple numa: 1 ostype: l26 smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 2 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 hugepages: 2 ================================================ FILE: src/test/cfg2cmd/memory-hugepages-2m.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '4,sockets=2,cores=2,maxcpus=4' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 8192 \ -object 'memory-backend-file,id=ram-node0,size=4096M,mem-path=/run/hugepages/kvm/2048kB,share=on,prealloc=yes' \ -numa 'node,nodeid=0,cpus=0-1,memdev=ram-node0' \ -object 'memory-backend-file,id=ram-node1,size=4096M,mem-path=/run/hugepages/kvm/2048kB,share=on,prealloc=yes' \ -numa 'node,nodeid=1,cpus=2-3,memdev=ram-node1' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/minimal-defaults-to-new-machine.conf ================================================ # TEST: newer machine version than QEMU version installed on node # QEMU_VERSION: 9.0.0 # EXPECT_ERROR: Installed QEMU version '9.0.0' is too old to run machine type 'pc-q35-42.9+pve0', please upgrade node 'localhost' smbios1: uuid=6cf17dc3-8341-4ecc-aebd-7503f2583fb3 machine: pc-q35-42.9 ================================================ FILE: src/test/cfg2cmd/minimal-defaults-unsupported-pve-version.conf ================================================ # TEST: newer machine version than QEMU version installed on node # QEMU_VERSION: 9.0.0 # EXPECT_ERROR: Installed qemu-server (max feature level for 9.0 is pve0) is too old to run machine type 'pc-q35-9.0+pve77', please upgrade node 'localhost' smbios1: uuid=6cf17dc3-8341-4ecc-aebd-7503f2583fb3 machine: pc-q35-9.0+pve77 ================================================ FILE: src/test/cfg2cmd/minimal-defaults.conf ================================================ # TEST: minimal configuration to detect default changes smbios1: uuid=6cf17dc3-8341-4ecc-aebd-7503f2583fb3 ================================================ FILE: src/test/cfg2cmd/minimal-defaults.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=6cf17dc3-8341-4ecc-aebd-7503f2583fb3' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/netdev-7.0-multiqueues.conf ================================================ # TEST: Simple test for netdev multi queue on 7.0 machine version bootdisk: scsi0 cores: 3 machine: pc-i440fx-7.0 memory: 768 name: netdev-multiq net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0,mtu=900,queues=2 ostype: l26 smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 ================================================ FILE: src/test/cfg2cmd/netdev-7.0-multiqueues.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'netdev-multiq,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on,queues=2' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,vectors=6,mq=on,bootindex=300,host_mtu=900' \ -machine 'type=pc-i440fx-7.0+pve0' ================================================ FILE: src/test/cfg2cmd/netdev-7.1-multiqueues.conf ================================================ # TEST: Simple test for netdev related stuff bootdisk: scsi0 cores: 3 memory: 768 name: netdev net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0,mtu=900,queues=2 ostype: l26 ================================================ FILE: src/test/cfg2cmd/netdev-7.1-multiqueues.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'netdev,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on,queues=2' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,vectors=6,mq=on,packed=on,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=900' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/netdev-7.1.conf ================================================ # TEST: Simple test for netdev related stuff bootdisk: scsi0 cores: 3 memory: 768 name: netdev net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0,mtu=900 ostype: l26 ================================================ FILE: src/test/cfg2cmd/netdev-7.1.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'netdev,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=900' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/netdev.conf ================================================ # TEST: Simple test for netdev related stuff bootdisk: scsi0 cores: 3 machine: pc-i440fx-5.0 memory: 768 name: netdev net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0,mtu=900 ostype: l26 ================================================ FILE: src/test/cfg2cmd/netdev.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'netdev,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300,host_mtu=900' \ -machine 'type=pc-i440fx-5.0+pve0' ================================================ FILE: src/test/cfg2cmd/netdev_vxlan.conf ================================================ # TEST: Test inheriting the MTU from a bridge with MTU != 1500 bootdisk: scsi0 cores: 3 memory: 768 name: netdev net0: virtio=A2:C0:43:77:08:A0,bridge=vxlan_bridge ostype: l26 ================================================ FILE: src/test/cfg2cmd/netdev_vxlan.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'netdev,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1450' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/old-qemu.conf ================================================ # TEST: Test QEMU version detection and expect fail on old version # QEMU_VERSION: 5.0.1 # EXPECT_ERROR: Detected old QEMU binary ('5.0.1', at least 6.0 is required) smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 ================================================ FILE: src/test/cfg2cmd/os-l24.conf ================================================ # TEST: Simple test for Linux OS type l24 ostype: l24 vga: qxl ================================================ FILE: src/test/cfg2cmd/os-l24.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'qxl-vga,id=vga,max_outputs=4,bus=pci.0,addr=0x2' \ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ -chardev 'spicevmc,id=vdagent,name=vdagent' \ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/os-other.conf ================================================ # TEST: Simple test for OS type other ostype: other ================================================ FILE: src/test/cfg2cmd/os-other.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/os-solaris.conf ================================================ # TEST: Simple test for OS type Solaris ostype: solaris ================================================ FILE: src/test/cfg2cmd/os-solaris.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep,-x2apic \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/ostype-usb13-error.conf ================================================ # TEST: Test error for old ostype type with newer usb config # EXPECT_ERROR: using usb13 is only possible with machine type >= 7.1 and ostype l26 or windows > 7 cores: 2 memory: 768 name: q35-usb3-error net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 ostype: w2k scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 vga: qxl usb13: spice ================================================ FILE: src/test/cfg2cmd/pinned-version-pxe-pve.conf ================================================ # TEST: for a basic configuration with a .pxe machine and +pve pinned bootdisk: scsi0 cores: 3 ide2: none,media=cdrom machine: pc-q35-4.1+pve2.pxe memory: 1024 name: pinned net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 numa: 0 ostype: l26 scsi0: local:8006/vm-8006-disk-0.raw,discard=on,size=104858K scsihw: virtio-scsi-pci smbios1: uuid=c7fdd046-fefc-11e9-832e-770e1d5636a0 sockets: 1 vmgenid: bdd46b98-fefc-11e9-97b4-d72c378e0f96 # add rng0 to stress +pve2 version requirement rng0: source=/dev/urandom ================================================ FILE: src/test/cfg2cmd/pinned-version-pxe-pve.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'pinned,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=c7fdd046-fefc-11e9-832e-770e1d5636a0' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 1024 \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=bdd46b98-fefc-11e9-97b4-d72c378e0f96' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -object 'rng-random,filename=/dev/urandom,id=rng0' \ -device 'virtio-rng-pci,rng=rng0,max-bytes=1024,period=1000,bus=pci.1,addr=0x1d' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.raw,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300,romfile=pxe-virtio.rom' \ -machine 'type=pc-q35-4.1+pve2' ================================================ FILE: src/test/cfg2cmd/pinned-version-pxe.conf ================================================ # TEST: for a basic configuration with a .pxe machine bootdisk: scsi0 cores: 3 ide2: none,media=cdrom machine: pc-q35-5.1.pxe memory: 1024 name: pinned net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 numa: 0 ostype: l26 scsi0: local:8006/vm-8006-disk-0.raw,discard=on,size=104858K scsihw: virtio-scsi-pci smbios1: uuid=c7fdd046-fefc-11e9-832e-770e1d5636a0 sockets: 1 vmgenid: bdd46b98-fefc-11e9-97b4-d72c378e0f96 ================================================ FILE: src/test/cfg2cmd/pinned-version-pxe.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'pinned,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=c7fdd046-fefc-11e9-832e-770e1d5636a0' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 1024 \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=bdd46b98-fefc-11e9-97b4-d72c378e0f96' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.raw,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300,romfile=pxe-virtio.rom' \ -machine 'type=pc-q35-5.1+pve0' ================================================ FILE: src/test/cfg2cmd/pinned-version.conf ================================================ # TEST: Simple test for a basic configuration with pinned QEMU machine version bootdisk: scsi0 cores: 3 ide2: none,media=cdrom machine: pc-q35-3.1 memory: 1024 name: pinned net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 numa: 0 ostype: l26 scsi0: local:8006/vm-8006-disk-0.raw,discard=on,size=104858K scsihw: virtio-scsi-pci smbios1: uuid=c7fdd046-fefc-11e9-832e-770e1d5636a0 sockets: 1 vmgenid: bdd46b98-fefc-11e9-97b4-d72c378e0f96 ================================================ FILE: src/test/cfg2cmd/pinned-version.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'pinned,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=c7fdd046-fefc-11e9-832e-770e1d5636a0' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 1024 \ -readconfig /usr/share/qemu-server/pve-q35.cfg \ -device 'vmgenid,guid=bdd46b98-fefc-11e9-97b4-d72c378e0f96' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.raw,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -machine 'type=pc-q35-3.1+pve0' ================================================ FILE: src/test/cfg2cmd/q35-ide.conf ================================================ # TEST: Config with q35, Linux & four IDE CD-ROMs bootdisk: scsi0 cores: 2 ide0: cifs-store:iso/zero.iso,media=cdrom,size=112M ide1: cifs-store:iso/one.iso,media=cdrom,size=112M ide2: cifs-store:iso/two.iso,media=cdrom,size=112M ide3: cifs-store:iso/three.iso,media=cdrom,size=112M machine: q35 memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 ostype: l26 scsi0: local:100/vm-100-disk-2.qcow2,size=10G scsihw: virtio-scsi-pci smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/q35-ide.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object '{"id":"throttle-drive-ide0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-ide1","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-ide2","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-ide3","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -global 'ICH9-LPC.disable_s3=1' \ -global 'ICH9-LPC.disable_s4=1' \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/zero.iso","node-name":"e1677eafc00b7016099210662868e38","read-only":true},"node-name":"f1677eafc00b7016099210662868e38","read-only":true},"node-name":"drive-ide0","read-only":true,"throttle-group":"throttle-drive-ide0"}' \ -device 'ide-cd,bus=ide.0,unit=0,drive=drive-ide0,id=ide0,bootindex=200' \ -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/one.iso","node-name":"e247a6535f864815c8e9dbb5a118336","read-only":true},"node-name":"f247a6535f864815c8e9dbb5a118336","read-only":true},"node-name":"drive-ide1","read-only":true,"throttle-group":"throttle-drive-ide1"}' \ -device 'ide-cd,bus=ide.2,unit=0,drive=drive-ide1,id=ide1,bootindex=201' \ -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/two.iso","node-name":"ec78a64f692c2fa9f873a111580aebd","read-only":true},"node-name":"fc78a64f692c2fa9f873a111580aebd","read-only":true},"node-name":"drive-ide2","read-only":true,"throttle-group":"throttle-drive-ide2"}' \ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=202' \ -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/three.iso","node-name":"e35557bae4bcbf9edc9f7ff7f132f30","read-only":true},"node-name":"f35557bae4bcbf9edc9f7ff7f132f30","read-only":true},"node-name":"drive-ide3","read-only":true,"throttle-group":"throttle-drive-ide3"}' \ -device 'ide-cd,bus=ide.3,unit=0,drive=drive-ide3,id=ide3,bootindex=203' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-2.qcow2","node-name":"ec11e0572184321efc5835152b95d5d","read-only":false},"node-name":"fc11e0572184321efc5835152b95d5d","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'hpet=off,type=q35+pve0' ================================================ FILE: src/test/cfg2cmd/q35-linux-hostpci-driver-keep.conf ================================================ # TEST: Config with q35, NUMA, hostpci passthrough, EFI & Linux & driver option bios: ovmf bootdisk: scsi0 cores: 1 efidisk0: local:100/vm-100-disk-1.qcow2,size=128K hostpci0: 00:ff.1,driver=keep hostpci1: d0:13.0,pcie=1,driver=vfio hostpci2: 00:f4.0 hostpci3: d0:15.1,pcie=1 hostpci4: d0:17.0,pcie=1,rombar=0 hostpci7: d0:15.2,pcie=1 machine: q35 memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 numa: 1 ostype: l26 scsihw: virtio-scsi-pci smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 sockets: 2 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/q35-linux-hostpci-driver-keep.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e70e3017c5a79fdee5a04aa92ac1e9c","read-only":false},"node-name":"f70e3017c5a79fdee5a04aa92ac1e9c","read-only":false},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \ -smp '2,sockets=2,cores=1,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object 'memory-backend-ram,id=ram-node0,size=256M' \ -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \ -object 'memory-backend-ram,id=ram-node1,size=256M' \ -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \ -global 'ICH9-LPC.disable_s3=1' \ -global 'ICH9-LPC.disable_s4=1' \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'vfio-pci,host=0000:00:ff.1,id=hostpci0,bus=pci.0,addr=0x10' \ -device 'vfio-pci,host=0000:d0:13.0,id=hostpci1,bus=ich9-pcie-port-2,addr=0x0' \ -device 'vfio-pci,host=0000:00:f4.0,id=hostpci2,bus=pci.0,addr=0x1b' \ -device 'vfio-pci,host=0000:d0:15.1,id=hostpci3,bus=ich9-pcie-port-4,addr=0x0' \ -device 'pcie-root-port,id=ich9-pcie-port-5,addr=10.0,x-speed=16,x-width=32,multifunction=on,bus=pcie.0,port=5,chassis=5' \ -device 'vfio-pci,host=0000:d0:17.0,id=hostpci4,bus=ich9-pcie-port-5,addr=0x0,rombar=0' \ -device 'pcie-root-port,id=ich9-pcie-port-8,addr=10.3,x-speed=16,x-width=32,multifunction=on,bus=pcie.0,port=8,chassis=8' \ -device 'vfio-pci,host=0000:d0:15.2,id=hostpci7,bus=ich9-pcie-port-8,addr=0x0' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,hpet=off,type=q35+pve0' ================================================ FILE: src/test/cfg2cmd/q35-linux-hostpci-mapping.conf ================================================ # TEST: Config with q35, NUMA, hostpci mapping passthrough, EFI & Linux bios: ovmf bootdisk: scsi0 cores: 1 efidisk0: local:100/vm-100-disk-1.qcow2,size=128K hostpci0: mapping=someNic hostpci1: mapping=someGpu,mdev=some-model hostpci2: mapping=someNic machine: q35 memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 numa: 1 ostype: l26 scsihw: virtio-scsi-pci smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 sockets: 2 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/q35-linux-hostpci-mapping.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e70e3017c5a79fdee5a04aa92ac1e9c","read-only":false},"node-name":"f70e3017c5a79fdee5a04aa92ac1e9c","read-only":false},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \ -smp '2,sockets=2,cores=1,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object 'memory-backend-ram,id=ram-node0,size=256M' \ -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \ -object 'memory-backend-ram,id=ram-node1,size=256M' \ -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \ -global 'ICH9-LPC.disable_s3=1' \ -global 'ICH9-LPC.disable_s4=1' \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'vfio-pci,host=0000:07:10.0,id=hostpci0,bus=pci.0,addr=0x10' \ -device 'vfio-pci,sysfsdev=/sys/bus/mdev/devices/00000001-0000-0000-0000-000000008006,id=hostpci1,bus=pci.0,addr=0x11' \ -device 'vfio-pci,host=0000:07:10.1,id=hostpci2,bus=pci.0,addr=0x1b' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,hpet=off,type=q35+pve0' ================================================ FILE: src/test/cfg2cmd/q35-linux-hostpci-multifunction.conf ================================================ # TEST: Config with q35, NUMA, hostpci passthrough, EFI & Linux bios: ovmf bootdisk: scsi0 cores: 1 efidisk0: local:100/vm-100-disk-1.qcow2,size=128K hostpci0: f0:43 hostpci1: 1234:f0:43 machine: q35 memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 numa: 1 ostype: l26 scsihw: virtio-scsi-pci smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 sockets: 2 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/q35-linux-hostpci-multifunction.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e70e3017c5a79fdee5a04aa92ac1e9c","read-only":false},"node-name":"f70e3017c5a79fdee5a04aa92ac1e9c","read-only":false},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \ -smp '2,sockets=2,cores=1,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object 'memory-backend-ram,id=ram-node0,size=256M' \ -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \ -object 'memory-backend-ram,id=ram-node1,size=256M' \ -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \ -global 'ICH9-LPC.disable_s3=1' \ -global 'ICH9-LPC.disable_s4=1' \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'vfio-pci,host=0000:f0:43.0,id=hostpci0.0,bus=pci.0,addr=0x10.0,multifunction=on' \ -device 'vfio-pci,host=0000:f0:43.1,id=hostpci0.1,bus=pci.0,addr=0x10.1' \ -device 'vfio-pci,host=1234:f0:43.1,id=hostpci1,bus=pci.0,addr=0x11' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,hpet=off,type=q35+pve0' ================================================ FILE: src/test/cfg2cmd/q35-linux-hostpci-template.conf ================================================ # TEST: Config with q35, NUMA, hostpci passthrough, EFI, TPM & Linux as a template bios: ovmf bootdisk: scsi0 cores: 1 efidisk0: local:100/base-100-disk-1.qcow2,size=128K hostpci0: 00:ff.1 hostpci1: d0:13.0,pcie=1 hostpci2: 00:f4.0 hostpci3: d0:15.1,pcie=1 hostpci4: d0:17.0,pcie=1,rombar=0 hostpci7: d0:15.2,pcie=1 machine: q35 memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 numa: 1 ostype: l26 scsihw: virtio-scsi-pci scsi0: local:100/base-100-disk-2.raw,size=10G smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 sockets: 2 template: 1 tpmstate0: local:100/base-100-disk-1.raw,size=4M,version=v2.0 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/q35-linux-hostpci-template.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/base-100-disk-1.qcow2","node-name":"eb6bec0e3c391fabb7fb7dd73ced9bf","read-only":true},"node-name":"fb6bec0e3c391fabb7fb7dd73ced9bf","read-only":true},"node-name":"drive-efidisk0","read-only":true,"throttle-group":"throttle-drive-efidisk0"}' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vga none \ -nographic \ -cpu qemu64 \ -m 512 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/base-100-disk-2.raw","node-name":"e24dfe239201bb9924fc4cfb899ca70","read-only":true},"node-name":"f24dfe239201bb9924fc4cfb899ca70","read-only":true},"node-name":"drive-scsi0","read-only":true,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,write-cache=on' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,accel=tcg,type=pc+pve0' \ -snapshot ================================================ FILE: src/test/cfg2cmd/q35-linux-hostpci-x-pci-overrides.conf ================================================ # TEST: Overriding PCI vendor/device IDs reported to guest bios: ovmf bootdisk: scsi0 cores: 1 efidisk0: local:100/vm-100-disk-1.qcow2,size=128K hostpci0: 00:ff.1,vendor-id=0x1234,device-id=0x5678,sub-vendor-id=0x2233,sub-device-id=0x0000 hostpci1: d0:13.0,pcie=1,vendor-id=0x1234,device-id=0x5678 machine: q35 memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 numa: 1 ostype: l26 scsihw: virtio-scsi-pci smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 sockets: 2 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/q35-linux-hostpci-x-pci-overrides.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e70e3017c5a79fdee5a04aa92ac1e9c","read-only":false},"node-name":"f70e3017c5a79fdee5a04aa92ac1e9c","read-only":false},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \ -smp '2,sockets=2,cores=1,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object 'memory-backend-ram,id=ram-node0,size=256M' \ -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \ -object 'memory-backend-ram,id=ram-node1,size=256M' \ -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \ -global 'ICH9-LPC.disable_s3=1' \ -global 'ICH9-LPC.disable_s4=1' \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'vfio-pci,host=0000:00:ff.1,id=hostpci0,bus=pci.0,addr=0x10,x-pci-vendor-id=0x1234,x-pci-device-id=0x5678,x-pci-sub-vendor-id=0x2233,x-pci-sub-device-id=0x0000' \ -device 'vfio-pci,host=0000:d0:13.0,id=hostpci1,bus=ich9-pcie-port-2,addr=0x0,x-pci-vendor-id=0x1234,x-pci-device-id=0x5678' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,hpet=off,type=q35+pve0' ================================================ FILE: src/test/cfg2cmd/q35-linux-hostpci.conf ================================================ # TEST: Config with q35, NUMA, hostpci passthrough, EFI & Linux bios: ovmf bootdisk: scsi0 cores: 1 efidisk0: local:100/vm-100-disk-1.qcow2,size=128K hostpci0: 00:ff.1 hostpci1: d0:13.0,pcie=1 hostpci2: 00:f4.0 hostpci3: d0:15.1,pcie=1 hostpci4: d0:17.0,pcie=1,rombar=0 hostpci7: d0:15.2,pcie=1 machine: q35 memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 numa: 1 ostype: l26 scsihw: virtio-scsi-pci smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 sockets: 2 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/q35-linux-hostpci.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e70e3017c5a79fdee5a04aa92ac1e9c","read-only":false},"node-name":"f70e3017c5a79fdee5a04aa92ac1e9c","read-only":false},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \ -smp '2,sockets=2,cores=1,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object 'memory-backend-ram,id=ram-node0,size=256M' \ -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \ -object 'memory-backend-ram,id=ram-node1,size=256M' \ -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \ -global 'ICH9-LPC.disable_s3=1' \ -global 'ICH9-LPC.disable_s4=1' \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'vfio-pci,host=0000:00:ff.1,id=hostpci0,bus=pci.0,addr=0x10' \ -device 'vfio-pci,host=0000:d0:13.0,id=hostpci1,bus=ich9-pcie-port-2,addr=0x0' \ -device 'vfio-pci,host=0000:00:f4.0,id=hostpci2,bus=pci.0,addr=0x1b' \ -device 'vfio-pci,host=0000:d0:15.1,id=hostpci3,bus=ich9-pcie-port-4,addr=0x0' \ -device 'pcie-root-port,id=ich9-pcie-port-5,addr=10.0,x-speed=16,x-width=32,multifunction=on,bus=pcie.0,port=5,chassis=5' \ -device 'vfio-pci,host=0000:d0:17.0,id=hostpci4,bus=ich9-pcie-port-5,addr=0x0,rombar=0' \ -device 'pcie-root-port,id=ich9-pcie-port-8,addr=10.3,x-speed=16,x-width=32,multifunction=on,bus=pcie.0,port=8,chassis=8' \ -device 'vfio-pci,host=0000:d0:15.2,id=hostpci7,bus=ich9-pcie-port-8,addr=0x0' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,hpet=off,type=q35+pve0' ================================================ FILE: src/test/cfg2cmd/q35-no-viommu-with-aw-bits.conf ================================================ # TEST: Check that aw-bits cannot be set without viommu # EXPECT_ERROR: cannot set aw-bits if no vIOMMU is configured machine: q35,aw-bits=39 ================================================ FILE: src/test/cfg2cmd/q35-simple-6.0.conf ================================================ # TEST: Config with q35, Linux & nothing much else but on 6.0 bios: ovmf bootdisk: scsi0 cores: 2 efidisk0: local:100/vm-100-disk-1.qcow2,size=128K machine: pc-q35-6.0 memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 ostype: l26 scsihw: virtio-scsi-pci smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/q35-simple-6.0.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -machine 'type=pc-q35-6.0+pve0' ================================================ FILE: src/test/cfg2cmd/q35-simple-7.0.conf ================================================ # TEST: Config with q35, Linux & nothing much else but on 7.0 bios: ovmf bootdisk: scsi0 cores: 2 efidisk0: local:100/vm-100-disk-1.qcow2,size=128K machine: pc-q35-7.0 meta: creation-qemu=6.1 memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 ostype: l26 scsihw: virtio-scsi-pci smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/q35-simple-7.0.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -machine 'type=pc-q35-7.0+pve0' ================================================ FILE: src/test/cfg2cmd/q35-simple-pinned-6.1.conf ================================================ # TEST: Config with q35, Linux & nothing much else # bios: ovmf bootdisk: scsi0 cores: 2 efidisk0: local:100/vm-100-disk-1.qcow2,size=128K machine: pc-q35-6.1 memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 ostype: l26 scsihw: virtio-scsi-pci smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/q35-simple-pinned-6.1.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -machine 'type=pc-q35-6.1+pve0' ================================================ FILE: src/test/cfg2cmd/q35-simple.conf ================================================ # TEST: Config with q35, Linux & nothing much else # bios: ovmf bootdisk: scsi0 cores: 2 efidisk0: local:100/vm-100-disk-1.qcow2,size=128K machine: q35 memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 ostype: l26 scsihw: virtio-scsi-pci smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/q35-simple.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e70e3017c5a79fdee5a04aa92ac1e9c","read-only":false},"node-name":"f70e3017c5a79fdee5a04aa92ac1e9c","read-only":false},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'ICH9-LPC.disable_s3=1' \ -global 'ICH9-LPC.disable_s4=1' \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,hpet=off,type=q35+pve0' ================================================ FILE: src/test/cfg2cmd/q35-usb13-error.conf ================================================ # TEST: Test usb error for q35 and older machine type # EXPECT_ERROR: using usb13 is only possible with machine type >= 7.1 and ostype l26 or windows > 7 cores: 2 memory: 768 name: q35-usb3-error net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 ostype: l26 machine: pc-q35-4.0 scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 vga: qxl usb13: spice ================================================ FILE: src/test/cfg2cmd/q35-usb2.conf ================================================ # TEST: Test Q35 USB2 passthrough combination cores: 2 memory: 768 name: q35-usb2 net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 ostype: l26 machine: pc-q35-4.0 scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 vga: qxl usb1: spice ================================================ FILE: src/test/cfg2cmd/q35-usb2.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'q35-usb2,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -chardev 'spicevmc,id=usbredirchardev1,name=usbredir' \ -device 'usb-redir,chardev=usbredirchardev1,id=usbredirdev1,bus=ehci.0' \ -device 'qxl-vga,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ -chardev 'spicevmc,id=vdagent,name=vdagent' \ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -machine 'type=pc-q35-4.0+pve0' ================================================ FILE: src/test/cfg2cmd/q35-usb3.conf ================================================ # TEST: Test Q35 USB3 passthrough combination cores: 2 memory: 768 name: q35-usb3 net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 ostype: l26 machine: pc-q35-4.0 scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 vga: qxl usb1: spice,usb3=1 ================================================ FILE: src/test/cfg2cmd/q35-usb3.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'q35-usb3,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'nec-usb-xhci,id=xhci,bus=pci.1,addr=0x1b' \ -chardev 'spicevmc,id=usbredirchardev1,name=usbredir' \ -device 'usb-redir,chardev=usbredirchardev1,id=usbredirdev1,bus=xhci.0' \ -device 'qxl-vga,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ -chardev 'spicevmc,id=vdagent,name=vdagent' \ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -machine 'type=pc-q35-4.0+pve0' ================================================ FILE: src/test/cfg2cmd/q35-viommu-intel-aw-bits.conf ================================================ # TEST: Check if aw-bits are propagated correctly to intel-iommu device machine: q35,viommu=intel,aw-bits=39 ================================================ FILE: src/test/cfg2cmd/q35-viommu-intel-aw-bits.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'ICH9-LPC.disable_s3=1' \ -global 'ICH9-LPC.disable_s4=1' \ -device 'intel-iommu,intremap=on,caching-mode=on,aw-bits=39' \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=q35+pve0,kernel-irqchip=split' ================================================ FILE: src/test/cfg2cmd/q35-viommu-intel-guest-phys-bits.conf ================================================ machine: q35,viommu=intel cpu: host,guest-phys-bits=39 ================================================ FILE: src/test/cfg2cmd/q35-viommu-intel-guest-phys-bits.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu 'host,+kvm_pv_eoi,+kvm_pv_unhalt,guest-phys-bits=39' \ -m 512 \ -global 'ICH9-LPC.disable_s3=1' \ -global 'ICH9-LPC.disable_s4=1' \ -device 'intel-iommu,intremap=on,caching-mode=on' \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=q35+pve0,kernel-irqchip=split' ================================================ FILE: src/test/cfg2cmd/q35-viommu-intel.conf ================================================ machine: q35,viommu=intel ================================================ FILE: src/test/cfg2cmd/q35-viommu-intel.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'ICH9-LPC.disable_s3=1' \ -global 'ICH9-LPC.disable_s4=1' \ -device 'intel-iommu,intremap=on,caching-mode=on' \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=q35+pve0,kernel-irqchip=split' ================================================ FILE: src/test/cfg2cmd/q35-viommu-virtio-aw-bits.conf ================================================ # TEST: Check if aw-bits are propagated correctly to virtio-iommu-pci device machine: q35,viommu=virtio,aw-bits=39 ================================================ FILE: src/test/cfg2cmd/q35-viommu-virtio-aw-bits.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'ICH9-LPC.disable_s3=1' \ -global 'ICH9-LPC.disable_s4=1' \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'virtio-iommu-pci,aw-bits=39' \ -machine 'type=q35+pve0' ================================================ FILE: src/test/cfg2cmd/q35-viommu-virtio.conf ================================================ machine: type=q35,viommu=virtio ================================================ FILE: src/test/cfg2cmd/q35-viommu-virtio.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'ICH9-LPC.disable_s3=1' \ -global 'ICH9-LPC.disable_s4=1' \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device virtio-iommu-pci \ -machine 'type=q35+pve0' ================================================ FILE: src/test/cfg2cmd/q35-win10-hostpci.conf ================================================ # TEST: Config with q35, NUMA, hostpci passthrough, EFI & Windows bios: ovmf bootdisk: scsi0 cores: 1 efidisk0: local:100/vm-100-disk-1.qcow2,size=128K hostpci0: f0:42.0 hostpci1: f0:43.0,pcie=1 hostpci4: 00:43.1,pcie=1 machine: q35 memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 numa: 1 ostype: win10 scsihw: virtio-scsi-pci smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 sockets: 2 vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d ================================================ FILE: src/test/cfg2cmd/q35-win10-hostpci.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ -smp '2,sockets=2,cores=1,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -global 'kvm-pit.lost_tick_policy=discard' \ -cpu 'kvm64,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vpindex,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep' \ -m 512 \ -object 'memory-backend-ram,id=ram-node0,size=256M' \ -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \ -object 'memory-backend-ram,id=ram-node1,size=256M' \ -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'vfio-pci,host=0000:f0:42.0,id=hostpci0,bus=pci.0,addr=0x10' \ -device 'vfio-pci,host=0000:f0:43.0,id=hostpci1,bus=ich9-pcie-port-2,addr=0x0' \ -device 'pcie-root-port,id=ich9-pcie-port-5,addr=10.0,x-speed=16,x-width=32,multifunction=on,bus=pcie.0,port=5,chassis=5' \ -device 'vfio-pci,host=0000:00:43.1,id=hostpci4,bus=ich9-pcie-port-5,addr=0x0' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -rtc 'driftfix=slew,base=localtime' \ -machine 'hpet=off,type=pc-q35-5.1+pve0' ================================================ FILE: src/test/cfg2cmd/q35-windows-pinning.conf ================================================ # TEST: Config with q35, win, meta to test version pinning # machine: q35 meta: creation-qemu=9.2.0,ctime=1741179133 ostype: win11 ================================================ FILE: src/test/cfg2cmd/q35-windows-pinning.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -global 'kvm-pit.lost_tick_policy=discard' \ -cpu 'kvm64,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vpindex,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep' \ -m 512 \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1,edid=off' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -rtc 'driftfix=slew,base=localtime' \ -machine 'hpet=off,type=pc-q35-9.2+pve0' ================================================ FILE: src/test/cfg2cmd/qemu-xhci-7.1.conf ================================================ # TEST: Test for new xhci controller with new machine version cores: 2 machine: pc-i440fx-7.1 memory: 768 name: spiceusb3 net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 ostype: l26 scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 vga: qxl usb1: spice usb5: spice usb13: host=1-14 ================================================ FILE: src/test/cfg2cmd/qemu-xhci-7.1.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'spiceusb3,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'qemu-xhci,p2=15,p3=15,id=xhci,bus=pci.1,addr=0x1b' \ -chardev 'spicevmc,id=usbredirchardev1,name=usbredir' \ -device 'usb-redir,chardev=usbredirchardev1,id=usbredirdev1,bus=xhci.0,port=2' \ -chardev 'spicevmc,id=usbredirchardev5,name=usbredir' \ -device 'usb-redir,chardev=usbredirchardev5,id=usbredirdev5,bus=xhci.0,port=6' \ -device 'usb-host,bus=xhci.0,port=14,hostbus=1,hostport=14,id=usb13' \ -device 'qxl-vga,id=vga,max_outputs=4,bus=pci.0,addr=0x2' \ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ -chardev 'spicevmc,id=vdagent,name=vdagent' \ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \ -machine 'type=pc-i440fx-7.1+pve0' ================================================ FILE: src/test/cfg2cmd/qemu-xhci-q35-7.1.conf ================================================ # TEST: Test Q35 USB passthrough combination with qemu-xhci cores: 2 memory: 768 name: q35-qemu-xhci net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 ostype: l26 machine: pc-q35-7.1 scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 usb1: spice ================================================ FILE: src/test/cfg2cmd/qemu-xhci-q35-7.1.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'q35-qemu-xhci,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'qemu-xhci,p2=15,p3=15,id=xhci,bus=pci.1,addr=0x1b' \ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ -chardev 'spicevmc,id=usbredirchardev1,name=usbredir' \ -device 'usb-redir,chardev=usbredirchardev1,id=usbredirdev1,bus=xhci.0,port=2' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \ -machine 'type=pc-q35-7.1+pve0' ================================================ FILE: src/test/cfg2cmd/qga-fs-freeze-backup-legacy.conf ================================================ # TEST: Ensure the deprecated freeze-fs-on-backup agent key is still parsed. agent: enabled=1,freeze-fs-on-backup=0 ================================================ FILE: src/test/cfg2cmd/qga-fs-freeze-backup-legacy.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -chardev 'socket,path=/var/run/qemu-server/8006.qga,server=on,wait=off,id=qga0' \ -device 'virtio-serial,id=qga0,bus=pci.0,addr=0x8' \ -device 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/qga-fs-freeze.conf ================================================ # TEST: Ensure agent sub-properties do not affect the QEMU command line. agent: 1,freeze-fs=1 ================================================ FILE: src/test/cfg2cmd/qga-fs-freeze.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -chardev 'socket,path=/var/run/qemu-server/8006.qga,server=on,wait=off,id=qga0' \ -device 'virtio-serial,id=qga0,bus=pci.0,addr=0x8' \ -device 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/qga-minimal.conf ================================================ # TEST: Ensure agent sub-properties do not affect the QEMU command line. agent: 1 ================================================ FILE: src/test/cfg2cmd/qga-minimal.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -chardev 'socket,path=/var/run/qemu-server/8006.qga,server=on,wait=off,id=qga0' \ -device 'virtio-serial,id=qga0,bus=pci.0,addr=0x8' \ -device 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/scsiblk.conf ================================================ boot: order=scsi0 meta: creation-qemu=10.0.2,ctime=1755189639 scsi0: /dev/sdg,scsiblock=1,size=32G smbios1: uuid=d47e2d5c-7068-46d4-9733-ff94083e28f6 vmgenid: 0a212e2f-0d9a-4006-971e-17be9fcf3efe ================================================ FILE: src/test/cfg2cmd/scsiblk.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=d47e2d5c-7068-46d4-9733-ff94083e28f6' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=0a212e2f-0d9a-4006-971e-17be9fcf3efe' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'lsi,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"host_device","filename":"/dev/sdg","node-name":"e8c91179e47ae3eb292a03618be5cf4","read-only":false},"node-name":"drive-scsi0","read-only":false}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/scsihw-lsi.conf ================================================ # TEST: Simple test for LSI SCSI controller bootdisk: scsi0 name: simple scsi0: lvm-store:vm-8006-disk-0,discard=on,size=104858K scsi1: lvm-store:vm-8006-disk-1,cache=writeback,discard=on,size=104858K scsi5: lvm-store:vm-8006-disk-2,cache=writethrough,discard=on,size=104858K scsi9: lvm-store:vm-8006-disk-3,cache=directsync,discard=on,size=104858K scsi16: lvm-store:vm-8006-disk-4,cache=directsync,discard=on,size=104858K scsihw: lsi ================================================ FILE: src/test/cfg2cmd/scsihw-lsi.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi5","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi9","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi16","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'pci-bridge,id=pci.4,chassis_nr=4,bus=pci.1,addr=0x1c' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'lsi,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-0","node-name":"e0378a375d635b0f473569544c7c207","read-only":false},"node-name":"f0378a375d635b0f473569544c7c207","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-1","node-name":"e68a63e6c55e4a6fe8c847a8f60af7e","read-only":false},"node-name":"f68a63e6c55e4a6fe8c847a8f60af7e","read-only":false},"node-name":"drive-scsi1","read-only":false,"throttle-group":"throttle-drive-scsi1"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=1,drive=drive-scsi1,id=scsi1,device_id=drive-scsi1,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-2","node-name":"e02ef0cb3873a2ecfddb04f546b6b26","read-only":false},"node-name":"f02ef0cb3873a2ecfddb04f546b6b26","read-only":false},"node-name":"drive-scsi5","read-only":false,"throttle-group":"throttle-drive-scsi5"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=5,drive=drive-scsi5,id=scsi5,device_id=drive-scsi5,write-cache=off' \ -device 'lsi,id=scsihw1,bus=pci.0,addr=0x6' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-3","node-name":"e77e7a9ba0adc041419f45894ffb1ed","read-only":false},"node-name":"f77e7a9ba0adc041419f45894ffb1ed","read-only":false},"node-name":"drive-scsi9","read-only":false,"throttle-group":"throttle-drive-scsi9"}' \ -device 'scsi-hd,bus=scsihw1.0,scsi-id=2,drive=drive-scsi9,id=scsi9,device_id=drive-scsi9,write-cache=off' \ -device 'lsi,id=scsihw2,bus=pci.4,addr=0x1' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-4","node-name":"e86ac0b43c3f3cb5be16052b349a9b7","read-only":false},"node-name":"f86ac0b43c3f3cb5be16052b349a9b7","read-only":false},"node-name":"drive-scsi16","read-only":false,"throttle-group":"throttle-drive-scsi16"}' \ -device 'scsi-hd,bus=scsihw2.0,scsi-id=2,drive=drive-scsi16,id=scsi16,device_id=drive-scsi16,write-cache=off' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/scsihw-lsi53c810.conf ================================================ # TEST: Simple test for LSI 53c810 SCSI controller bootdisk: scsi0 name: simple scsi0: lvm-store:vm-8006-disk-0,discard=on,size=104858K scsi1: lvm-store:vm-8006-disk-1,cache=writeback,discard=on,size=104858K scsi5: lvm-store:vm-8006-disk-2,cache=writethrough,discard=on,size=104858K scsi9: lvm-store:vm-8006-disk-3,cache=directsync,discard=on,size=104858K scsi16: lvm-store:vm-8006-disk-4,cache=directsync,discard=on,size=104858K scsihw: lsi53c810 ================================================ FILE: src/test/cfg2cmd/scsihw-lsi53c810.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi5","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi9","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi16","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'pci-bridge,id=pci.4,chassis_nr=4,bus=pci.1,addr=0x1c' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'lsi53c810,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-0","node-name":"e0378a375d635b0f473569544c7c207","read-only":false},"node-name":"f0378a375d635b0f473569544c7c207","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-1","node-name":"e68a63e6c55e4a6fe8c847a8f60af7e","read-only":false},"node-name":"f68a63e6c55e4a6fe8c847a8f60af7e","read-only":false},"node-name":"drive-scsi1","read-only":false,"throttle-group":"throttle-drive-scsi1"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=1,drive=drive-scsi1,id=scsi1,device_id=drive-scsi1,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-2","node-name":"e02ef0cb3873a2ecfddb04f546b6b26","read-only":false},"node-name":"f02ef0cb3873a2ecfddb04f546b6b26","read-only":false},"node-name":"drive-scsi5","read-only":false,"throttle-group":"throttle-drive-scsi5"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=5,drive=drive-scsi5,id=scsi5,device_id=drive-scsi5,write-cache=off' \ -device 'lsi53c810,id=scsihw1,bus=pci.0,addr=0x6' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-3","node-name":"e77e7a9ba0adc041419f45894ffb1ed","read-only":false},"node-name":"f77e7a9ba0adc041419f45894ffb1ed","read-only":false},"node-name":"drive-scsi9","read-only":false,"throttle-group":"throttle-drive-scsi9"}' \ -device 'scsi-hd,bus=scsihw1.0,scsi-id=2,drive=drive-scsi9,id=scsi9,device_id=drive-scsi9,write-cache=off' \ -device 'lsi53c810,id=scsihw2,bus=pci.4,addr=0x1' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-4","node-name":"e86ac0b43c3f3cb5be16052b349a9b7","read-only":false},"node-name":"f86ac0b43c3f3cb5be16052b349a9b7","read-only":false},"node-name":"drive-scsi16","read-only":false,"throttle-group":"throttle-drive-scsi16"}' \ -device 'scsi-hd,bus=scsihw2.0,scsi-id=2,drive=drive-scsi16,id=scsi16,device_id=drive-scsi16,write-cache=off' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/scsihw-megasas.conf ================================================ # TEST: Simple test for MegaRAID SAS SCSI controller bootdisk: scsi0 name: simple scsi0: lvm-store:vm-8006-disk-0,discard=on,size=104858K scsi1: lvm-store:vm-8006-disk-1,cache=writeback,discard=on,size=104858K scsi5: lvm-store:vm-8006-disk-2,cache=writethrough,discard=on,size=104858K scsi9: lvm-store:vm-8006-disk-3,cache=directsync,discard=on,size=104858K scsi16: lvm-store:vm-8006-disk-4,cache=directsync,discard=on,size=104858K scsihw: megasas ================================================ FILE: src/test/cfg2cmd/scsihw-megasas.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi5","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi9","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi16","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'megasas,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-0","node-name":"e0378a375d635b0f473569544c7c207","read-only":false},"node-name":"f0378a375d635b0f473569544c7c207","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-1","node-name":"e68a63e6c55e4a6fe8c847a8f60af7e","read-only":false},"node-name":"f68a63e6c55e4a6fe8c847a8f60af7e","read-only":false},"node-name":"drive-scsi1","read-only":false,"throttle-group":"throttle-drive-scsi1"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi1,id=scsi1,device_id=drive-scsi1,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-2","node-name":"e02ef0cb3873a2ecfddb04f546b6b26","read-only":false},"node-name":"f02ef0cb3873a2ecfddb04f546b6b26","read-only":false},"node-name":"drive-scsi5","read-only":false,"throttle-group":"throttle-drive-scsi5"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=5,drive=drive-scsi5,id=scsi5,device_id=drive-scsi5,write-cache=off' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-3","node-name":"e77e7a9ba0adc041419f45894ffb1ed","read-only":false},"node-name":"f77e7a9ba0adc041419f45894ffb1ed","read-only":false},"node-name":"drive-scsi9","read-only":false,"throttle-group":"throttle-drive-scsi9"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=9,drive=drive-scsi9,id=scsi9,device_id=drive-scsi9,write-cache=off' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-4","node-name":"e86ac0b43c3f3cb5be16052b349a9b7","read-only":false},"node-name":"f86ac0b43c3f3cb5be16052b349a9b7","read-only":false},"node-name":"drive-scsi16","read-only":false,"throttle-group":"throttle-drive-scsi16"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=16,drive=drive-scsi16,id=scsi16,device_id=drive-scsi16,write-cache=off' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/scsihw-pvscsi.conf ================================================ # TEST: Simple test for PVSCSI controller bootdisk: scsi0 name: simple scsi0: lvm-store:vm-8006-disk-0,discard=on,size=104858K scsi1: lvm-store:vm-8006-disk-1,cache=writeback,discard=on,size=104858K scsi5: lvm-store:vm-8006-disk-2,cache=writethrough,discard=on,size=104858K scsi9: lvm-store:vm-8006-disk-3,cache=directsync,discard=on,size=104858K scsi16: lvm-store:vm-8006-disk-4,cache=directsync,discard=on,size=104858K scsihw: pvscsi ================================================ FILE: src/test/cfg2cmd/scsihw-pvscsi.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi5","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi9","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi16","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'pvscsi,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-0","node-name":"e0378a375d635b0f473569544c7c207","read-only":false},"node-name":"f0378a375d635b0f473569544c7c207","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-1","node-name":"e68a63e6c55e4a6fe8c847a8f60af7e","read-only":false},"node-name":"f68a63e6c55e4a6fe8c847a8f60af7e","read-only":false},"node-name":"drive-scsi1","read-only":false,"throttle-group":"throttle-drive-scsi1"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=1,drive=drive-scsi1,id=scsi1,device_id=drive-scsi1,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-2","node-name":"e02ef0cb3873a2ecfddb04f546b6b26","read-only":false},"node-name":"f02ef0cb3873a2ecfddb04f546b6b26","read-only":false},"node-name":"drive-scsi5","read-only":false,"throttle-group":"throttle-drive-scsi5"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=5,drive=drive-scsi5,id=scsi5,device_id=drive-scsi5,write-cache=off' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-3","node-name":"e77e7a9ba0adc041419f45894ffb1ed","read-only":false},"node-name":"f77e7a9ba0adc041419f45894ffb1ed","read-only":false},"node-name":"drive-scsi9","read-only":false,"throttle-group":"throttle-drive-scsi9"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=9,drive=drive-scsi9,id=scsi9,device_id=drive-scsi9,write-cache=off' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-4","node-name":"e86ac0b43c3f3cb5be16052b349a9b7","read-only":false},"node-name":"f86ac0b43c3f3cb5be16052b349a9b7","read-only":false},"node-name":"drive-scsi16","read-only":false,"throttle-group":"throttle-drive-scsi16"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=16,drive=drive-scsi16,id=scsi16,device_id=drive-scsi16,write-cache=off' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/scsihw-virtio-scsi-single.conf ================================================ # TEST: Simple test for VirtIO SCSI single controller bootdisk: scsi0 name: simple scsi0: lvm-store:vm-8006-disk-0,discard=on,size=104858K scsi1: lvm-store:vm-8006-disk-1,cache=writeback,discard=on,size=104858K scsi5: lvm-store:vm-8006-disk-2,cache=writethrough,discard=on,size=104858K scsi9: lvm-store:vm-8006-disk-3,cache=directsync,discard=on,size=104858K scsi16: lvm-store:vm-8006-disk-4,cache=directsync,discard=on,size=104858K scsihw: virtio-scsi-single ================================================ FILE: src/test/cfg2cmd/scsihw-virtio-scsi-single.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi5","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi9","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi16","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'pci-bridge,id=pci.3,chassis_nr=3,bus=pci.0,addr=0x5' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'virtio-scsi-pci,id=virtioscsi0,bus=pci.3,addr=0x1' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-0","node-name":"e0378a375d635b0f473569544c7c207","read-only":false},"node-name":"f0378a375d635b0f473569544c7c207","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=virtioscsi0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -device 'virtio-scsi-pci,id=virtioscsi1,bus=pci.3,addr=0x2' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-1","node-name":"e68a63e6c55e4a6fe8c847a8f60af7e","read-only":false},"node-name":"f68a63e6c55e4a6fe8c847a8f60af7e","read-only":false},"node-name":"drive-scsi1","read-only":false,"throttle-group":"throttle-drive-scsi1"}' \ -device 'scsi-hd,bus=virtioscsi1.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi1,id=scsi1,device_id=drive-scsi1,write-cache=on' \ -device 'virtio-scsi-pci,id=virtioscsi5,bus=pci.3,addr=0x6' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-2","node-name":"e02ef0cb3873a2ecfddb04f546b6b26","read-only":false},"node-name":"f02ef0cb3873a2ecfddb04f546b6b26","read-only":false},"node-name":"drive-scsi5","read-only":false,"throttle-group":"throttle-drive-scsi5"}' \ -device 'scsi-hd,bus=virtioscsi5.0,channel=0,scsi-id=0,lun=5,drive=drive-scsi5,id=scsi5,device_id=drive-scsi5,write-cache=off' \ -device 'virtio-scsi-pci,id=virtioscsi9,bus=pci.3,addr=0xa' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-3","node-name":"e77e7a9ba0adc041419f45894ffb1ed","read-only":false},"node-name":"f77e7a9ba0adc041419f45894ffb1ed","read-only":false},"node-name":"drive-scsi9","read-only":false,"throttle-group":"throttle-drive-scsi9"}' \ -device 'scsi-hd,bus=virtioscsi9.0,channel=0,scsi-id=0,lun=9,drive=drive-scsi9,id=scsi9,device_id=drive-scsi9,write-cache=off' \ -device 'virtio-scsi-pci,id=virtioscsi16,bus=pci.3,addr=0x11' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-4","node-name":"e86ac0b43c3f3cb5be16052b349a9b7","read-only":false},"node-name":"f86ac0b43c3f3cb5be16052b349a9b7","read-only":false},"node-name":"drive-scsi16","read-only":false,"throttle-group":"throttle-drive-scsi16"}' \ -device 'scsi-hd,bus=virtioscsi16.0,channel=0,scsi-id=0,lun=16,drive=drive-scsi16,id=scsi16,device_id=drive-scsi16,write-cache=off' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/seabios_serial.conf ================================================ # TEST: Test for smm-related regression with SeaBIOS and serial display bootdisk: scsi0 cores: 3 ide2: none,media=cdrom memory: 768 name: seabiosserial net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 scsi0: local:8006/vm-8006-disk-0.qcow2,discard=on,size=104858K scsihw: virtio-scsi-pci serial0: socket smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 vga: serial0 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 ================================================ FILE: src/test/cfg2cmd/seabios_serial.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'seabiosserial,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -nographic \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -chardev 'socket,id=serial0,path=/var/run/qemu-server/8006.serial0,server=on,wait=off' \ -device 'isa-serial,chardev=serial0' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"ecd04be4259153b8293415fefa2a84c","read-only":false},"node-name":"fcd04be4259153b8293415fefa2a84c","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'hpet=off,smm=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/sev-es.conf ================================================ # TEST: Test raw efidisk size parameter # HW_CAPABILITIES: amd-turin-9005 smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 bios: ovmf efidisk0: local:100/vm-100-disk-0.raw,efitype=4m,pre-enrolled-keys=1,size=528K amd-sev: type=es ================================================ FILE: src/test/cfg2cmd/sev-es.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_SEV_CODE_4M.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-0.raw","node-name":"e1175f2a490414e7c53337589fde17a","read-only":false},"node-name":"f1175f2a490414e7c53337589fde17a","read-only":false,"size":540672},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -object 'sev-guest,id=sev0,cbitpos=51,reduced-phys-bits=6,policy=0xc' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,type=pc+pve0,confidential-guest-support=sev0' ================================================ FILE: src/test/cfg2cmd/sev-snp.conf ================================================ # TEST: Test raw efidisk size parameter # HW_CAPABILITIES: amd-turin-9005 smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 bios: ovmf amd-sev: type=snp ================================================ FILE: src/test/cfg2cmd/sev-snp.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -bios /usr/share/pve-edk2-firmware//OVMF_SEV_4M.fd \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -object 'sev-snp-guest,id=sev0,cbitpos=51,reduced-phys-bits=6,policy=0xb0000' \ -machine 'type=pc+pve0,confidential-guest-support=sev0' ================================================ FILE: src/test/cfg2cmd/sev-std.conf ================================================ # TEST: Test raw efidisk size parameter # HW_CAPABILITIES: amd-turin-9005 smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 bios: ovmf efidisk0: local:100/vm-100-disk-0.raw,efitype=4m,pre-enrolled-keys=1,size=528K amd-sev: type=std ================================================ FILE: src/test/cfg2cmd/sev-std.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \ -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_SEV_CODE_4M.fd"},"node-name":"pflash0","read-only":true}' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-0.raw","node-name":"e1175f2a490414e7c53337589fde17a","read-only":false},"node-name":"f1175f2a490414e7c53337589fde17a","read-only":false,"size":540672},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -object 'sev-guest,id=sev0,cbitpos=51,reduced-phys-bits=6,policy=0x8' \ -machine 'pflash0=pflash0,pflash1=drive-efidisk0,type=pc+pve0,confidential-guest-support=sev0' ================================================ FILE: src/test/cfg2cmd/simple-backingchain.conf ================================================ # TEST: Simple test for external snapshot backing chain name: simple parent: snap3 scsi0: localsnapext:8006/vm-8006-disk-0.qcow2,size=1G scsi1: lvm-store:vm-8006-disk-0.qcow2,size=1G [snap1] name: simple scsi0: localsnapext:8006/vm-8006-disk-0.qcow2,size=1G scsi1: lvm-store:vm-8006-disk-0.qcow2,size=1G snaptime: 1748933042 [snap2] parent: snap1 name: simple scsi0: localsnapext:8006/vm-8006-disk-0.qcow2,size=1G scsi1: lvm-store:vm-8006-disk-0.qcow2,size=1G snaptime: 1748933043 [snap3] parent: snap2 name: simple scsi0: localsnapext:8006/vm-8006-disk-0.qcow2,size=1G scsi1: lvm-store:vm-8006-disk-0.qcow2,size=1G snaptime: 1748933044 ================================================ FILE: src/test/cfg2cmd/simple-backingchain.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'lsi,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"backing":{"backing":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","discard-no-unref":true,"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vzsnapext/images/8006/snap1-vm-8006-disk-0.qcow2","node-name":"ea91a385a49a008a4735c0aec5c6749","read-only":false},"node-name":"fa91a385a49a008a4735c0aec5c6749","read-only":false},"cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","discard-no-unref":true,"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vzsnapext/images/8006/snap2-vm-8006-disk-0.qcow2","node-name":"ec0289317073959d450248d8cd7a480","read-only":false},"node-name":"fc0289317073959d450248d8cd7a480","read-only":false},"cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","discard-no-unref":true,"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vzsnapext/images/8006/vm-8006-disk-0.qcow2","node-name":"e74f4959037afb46eddc7313c43dfdd","read-only":false},"node-name":"f74f4959037afb46eddc7313c43dfdd","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,write-cache=on' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"backing":{"backing":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"host_device","filename":"/dev/veegee/snap1-vm-8006-disk-0.qcow2","node-name":"e25f58d3e6e11f2065ad41253988915","read-only":false},"node-name":"f25f58d3e6e11f2065ad41253988915","read-only":false},"cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"host_device","filename":"/dev/veegee/snap2-vm-8006-disk-0.qcow2","node-name":"e9415bb5e484c1e25d25063b01686fe","read-only":false},"node-name":"f9415bb5e484c1e25d25063b01686fe","read-only":false},"cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-0.qcow2","node-name":"e87358a470ca311f94d5cc61d1eb428","read-only":false},"node-name":"f87358a470ca311f94d5cc61d1eb428","read-only":false},"node-name":"drive-scsi1","read-only":false,"throttle-group":"throttle-drive-scsi1"}' \ -device 'scsi-hd,bus=scsihw0.0,scsi-id=1,drive=drive-scsi1,id=scsi1,device_id=drive-scsi1,write-cache=on' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/simple-balloon-free-page-reporting.conf ================================================ # TEST: Simple test for balloon free page reporting enabled by default on 6.2 bootdisk: scsi0 cores: 3 ide2: none,media=cdrom machine: pc-i440fx-6.2 memory: 768 name: simple net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 scsi0: local:8006/vm-8006-disk-0.qcow2,discard=on,size=104858K scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 vmgenid: c773c261-d800-4348-1010-1010add53cf8 ================================================ FILE: src/test/cfg2cmd/simple-balloon-free-page-reporting.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-1010-1010add53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-scsi0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -machine 'type=pc-i440fx-6.2+pve0' ================================================ FILE: src/test/cfg2cmd/simple-btrfs.conf ================================================ # TEST: Simple test for a BTRFS backed VM, which shouldn't use cache=none like other storages bootdisk: scsi0 cores: 3 ide2: none,media=cdrom memory: 768 name: simple net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 scsi0: btrfs-store:8006/vm-8006-disk-0.raw,discard=on,size=104858K scsi1: btrfs-store:8006/vm-8006-disk-0.raw,cache=writeback,discard=on,size=104858K scsi2: btrfs-store:8006/vm-8006-disk-0.raw,cache=writethrough,discard=on,size=104858K scsi3: btrfs-store:8006/vm-8006-disk-0.raw,cache=directsync,discard=on,size=104858K scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 vmgenid: c773c261-d800-4348-1010-1010add53cf8 ================================================ FILE: src/test/cfg2cmd/simple-btrfs.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi2","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi3","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-1010-1010add53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/butter/bread/images/8006/vm-8006-disk-0/disk.raw","node-name":"e99aff0ff797aa030a22e9f580076dd","read-only":false},"node-name":"f99aff0ff797aa030a22e9f580076dd","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/butter/bread/images/8006/vm-8006-disk-0/disk.raw","node-name":"e7b2fd2a8c5dbfc550d9781e5df8841","read-only":false},"node-name":"f7b2fd2a8c5dbfc550d9781e5df8841","read-only":false},"node-name":"drive-scsi1","read-only":false,"throttle-group":"throttle-drive-scsi1"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi1,id=scsi1,device_id=drive-scsi1,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/butter/bread/images/8006/vm-8006-disk-0/disk.raw","node-name":"ed78b07bb04c2cbd8aedc648e885569","read-only":false},"node-name":"fd78b07bb04c2cbd8aedc648e885569","read-only":false},"node-name":"drive-scsi2","read-only":false,"throttle-group":"throttle-drive-scsi2"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=2,drive=drive-scsi2,id=scsi2,device_id=drive-scsi2,write-cache=off' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/butter/bread/images/8006/vm-8006-disk-0/disk.raw","node-name":"e7487c01d831e2b51a5446980170ec9","read-only":false},"node-name":"f7487c01d831e2b51a5446980170ec9","read-only":false},"node-name":"drive-scsi3","read-only":false,"throttle-group":"throttle-drive-scsi3"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=3,drive=drive-scsi3,id=scsi3,device_id=drive-scsi3,write-cache=off' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/simple-cifs.conf ================================================ # TEST: Simple test for a CIFS storage ide2: none,media=cdrom name: simple ostype: l26 scsi0: cifs-store:8006/vm-8006-disk-0.raw,discard=on,size=104858K scsi1: cifs-store:8006/vm-8006-disk-0.raw,cache=writeback,discard=on,size=104858K scsi2: cifs-store:8006/vm-8006-disk-0.raw,cache=writethrough,discard=on,size=104858K scsi3: cifs-store:8006/vm-8006-disk-0.raw,cache=directsync,discard=on,size=104858K scsihw: virtio-scsi-pci ================================================ FILE: src/test/cfg2cmd/simple-cifs.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi2","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi3","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/mnt/pve/cifs-store/images/8006/vm-8006-disk-0.raw","node-name":"e2b3b8f2d6a23adc1aa3ecd195dbaf5","read-only":false},"node-name":"f2b3b8f2d6a23adc1aa3ecd195dbaf5","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/mnt/pve/cifs-store/images/8006/vm-8006-disk-0.raw","node-name":"ee4d9a961200a669c1a8182632aba3e","read-only":false},"node-name":"fe4d9a961200a669c1a8182632aba3e","read-only":false},"node-name":"drive-scsi1","read-only":false,"throttle-group":"throttle-drive-scsi1"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi1,id=scsi1,device_id=drive-scsi1,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/mnt/pve/cifs-store/images/8006/vm-8006-disk-0.raw","node-name":"e6a3bf7eee1e2636cbe31f62b537b6c","read-only":false},"node-name":"f6a3bf7eee1e2636cbe31f62b537b6c","read-only":false},"node-name":"drive-scsi2","read-only":false,"throttle-group":"throttle-drive-scsi2"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=2,drive=drive-scsi2,id=scsi2,device_id=drive-scsi2,write-cache=off' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/mnt/pve/cifs-store/images/8006/vm-8006-disk-0.raw","node-name":"e7042ee58e764b1296ad54014cb9a03","read-only":false},"node-name":"f7042ee58e764b1296ad54014cb9a03","read-only":false},"node-name":"drive-scsi3","read-only":false,"throttle-group":"throttle-drive-scsi3"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=3,drive=drive-scsi3,id=scsi3,device_id=drive-scsi3,write-cache=off' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/simple-disk-passthrough.conf ================================================ # TEST: Simple test for disk && cdrom passthrough bootdisk: scsi0 cores: 3 ide2: cdrom,media=cdrom memory: 768 name: simple net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 scsi0: /dev/sda scsi1: /mnt/file.raw scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 ================================================ FILE: src/test/cfg2cmd/simple-disk-passthrough.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -object '{"id":"throttle-drive-ide2","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"driver":"host_cdrom","filename":"/dev/cdrom","node-name":"ee50e59431a6228dc388fc821b35696","read-only":true},"node-name":"fe50e59431a6228dc388fc821b35696","read-only":true},"node-name":"drive-ide2","read-only":true,"throttle-group":"throttle-drive-ide2"}' \ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"host_device","filename":"/dev/sda","node-name":"eec235c1b362ebd19d5e98959b4c171","read-only":false},"node-name":"fec235c1b362ebd19d5e98959b4c171","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/mnt/file.raw","node-name":"e234a4e3b89ac3adac9bdbf0c3dd6b4","read-only":false},"node-name":"f234a4e3b89ac3adac9bdbf0c3dd6b4","read-only":false},"node-name":"drive-scsi1","read-only":false,"throttle-group":"throttle-drive-scsi1"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi1,id=scsi1,device_id=drive-scsi1,write-cache=on' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/simple-lvm.conf ================================================ # TEST: Simple test for LVM backed VM bootdisk: scsi0 name: simple scsi0: lvm-store:vm-8006-disk-0,discard=on,size=104858K scsi1: lvm-store:vm-8006-disk-0,cache=writeback,discard=on,size=104858K scsi2: lvm-store:vm-8006-disk-0,cache=writethrough,discard=on,size=104858K scsi3: lvm-store:vm-8006-disk-0,cache=directsync,discard=on,size=104858K scsihw: virtio-scsi-pci ================================================ FILE: src/test/cfg2cmd/simple-lvm.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi2","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi3","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-0","node-name":"e0378a375d635b0f473569544c7c207","read-only":false},"node-name":"f0378a375d635b0f473569544c7c207","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-0","node-name":"e2fbae024c8a771f708f4a5391211b0","read-only":false},"node-name":"f2fbae024c8a771f708f4a5391211b0","read-only":false},"node-name":"drive-scsi1","read-only":false,"throttle-group":"throttle-drive-scsi1"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi1,id=scsi1,device_id=drive-scsi1,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-0","node-name":"e4328c26b141e3efe1564cb60bf1155","read-only":false},"node-name":"f4328c26b141e3efe1564cb60bf1155","read-only":false},"node-name":"drive-scsi2","read-only":false,"throttle-group":"throttle-drive-scsi2"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=2,drive=drive-scsi2,id=scsi2,device_id=drive-scsi2,write-cache=off' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"native","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/veegee/vm-8006-disk-0","node-name":"e68e10f8128f05fe5f7e85cc1f9922b","read-only":false},"node-name":"f68e10f8128f05fe5f7e85cc1f9922b","read-only":false},"node-name":"drive-scsi3","read-only":false,"throttle-group":"throttle-drive-scsi3"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=3,drive=drive-scsi3,id=scsi3,device_id=drive-scsi3,write-cache=off' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/simple-lvmthin.conf ================================================ # TEST: Simple test for LVMthin backed VM bootdisk: scsi0 name: simple scsi0: local-lvm:vm-8006-disk-0,discard=on,size=104858K scsi1: local-lvm:vm-8006-disk-0,cache=writeback,discard=on,size=104858K scsi2: local-lvm:vm-8006-disk-0,cache=writethrough,discard=on,size=104858K scsi3: local-lvm:vm-8006-disk-0,cache=directsync,discard=on,size=104858K scsihw: virtio-scsi-pci ================================================ FILE: src/test/cfg2cmd/simple-lvmthin.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi2","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi3","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/pve/vm-8006-disk-0","node-name":"e6d87b01b7bb888b8426534a542ff1c","read-only":false},"node-name":"f6d87b01b7bb888b8426534a542ff1c","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/pve/vm-8006-disk-0","node-name":"e96d9ece81aa4271aa2d8485184f66b","read-only":false},"node-name":"f96d9ece81aa4271aa2d8485184f66b","read-only":false},"node-name":"drive-scsi1","read-only":false,"throttle-group":"throttle-drive-scsi1"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi1,id=scsi1,device_id=drive-scsi1,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/pve/vm-8006-disk-0","node-name":"e0b89788ef97beda10a850ab45897d9","read-only":false},"node-name":"f0b89788ef97beda10a850ab45897d9","read-only":false},"node-name":"drive-scsi2","read-only":false,"throttle-group":"throttle-drive-scsi2"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=2,drive=drive-scsi2,id=scsi2,device_id=drive-scsi2,write-cache=off' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/pve/vm-8006-disk-0","node-name":"ea7b6871af66ca3e13e95bd74570aa2","read-only":false},"node-name":"fa7b6871af66ca3e13e95bd74570aa2","read-only":false},"node-name":"drive-scsi3","read-only":false,"throttle-group":"throttle-drive-scsi3"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=3,drive=drive-scsi3,id=scsi3,device_id=drive-scsi3,write-cache=off' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/simple-rbd.conf ================================================ # TEST: Simple test for RBD && KRBD backend vm bootdisk: scsi0 cores: 3 ide2: none,media=cdrom memory: 768 name: simple net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 scsi0: rbd-store:vm-8006-disk-0,discard=on,size=104858K scsi1: rbd-store:vm-8006-disk-0,discard=on,cache=writeback,size=104858K scsi2: rbd-store:vm-8006-disk-0,discard=on,cache=writethrough,size=104858K scsi3: rbd-store:vm-8006-disk-0,discard=on,cache=directsync,size=104858K scsi4: krbd-store:vm-8006-disk-0,discard=on,size=104858K scsi5: krbd-store:vm-8006-disk-0,cache=writeback,discard=on,size=104858K scsi6: krbd-store:vm-8006-disk-0,cache=writethrough,discard=on,size=104858K scsi7: krbd-store:vm-8006-disk-0,cache=directsync,discard=on,size=104858K scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 vmgenid: c773c261-d800-4348-1010-1010add53cf8 ================================================ FILE: src/test/cfg2cmd/simple-rbd.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi2","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi3","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi4","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi5","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi6","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi7","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-1010-1010add53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"auth-client-required":["none"],"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"rbd","image":"vm-8006-disk-0","node-name":"e8e1af6f55c6a2466f178045aa79710","pool":"cpool","read-only":false,"server":[{"host":"127.0.0.42","port":"3300"},{"host":"127.0.0.21","port":"3300"},{"host":"::1","port":"3300"}]},"node-name":"f8e1af6f55c6a2466f178045aa79710","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"auth-client-required":["none"],"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"rbd","image":"vm-8006-disk-0","node-name":"e3990bba2ed1f48c5bb23e9f37b4cec","pool":"cpool","read-only":false,"server":[{"host":"127.0.0.42","port":"3300"},{"host":"127.0.0.21","port":"3300"},{"host":"::1","port":"3300"}]},"node-name":"f3990bba2ed1f48c5bb23e9f37b4cec","read-only":false},"node-name":"drive-scsi1","read-only":false,"throttle-group":"throttle-drive-scsi1"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi1,id=scsi1,device_id=drive-scsi1,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"auth-client-required":["none"],"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"rbd","image":"vm-8006-disk-0","node-name":"e3beccc2a8f2eacb8b5df8055a7d093","pool":"cpool","read-only":false,"server":[{"host":"127.0.0.42","port":"3300"},{"host":"127.0.0.21","port":"3300"},{"host":"::1","port":"3300"}]},"node-name":"f3beccc2a8f2eacb8b5df8055a7d093","read-only":false},"node-name":"drive-scsi2","read-only":false,"throttle-group":"throttle-drive-scsi2"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=2,drive=drive-scsi2,id=scsi2,device_id=drive-scsi2,write-cache=off' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"auth-client-required":["none"],"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"rbd","image":"vm-8006-disk-0","node-name":"eef923d5dfcee93fbc712b03f9f21af","pool":"cpool","read-only":false,"server":[{"host":"127.0.0.42","port":"3300"},{"host":"127.0.0.21","port":"3300"},{"host":"::1","port":"3300"}]},"node-name":"fef923d5dfcee93fbc712b03f9f21af","read-only":false},"node-name":"drive-scsi3","read-only":false,"throttle-group":"throttle-drive-scsi3"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=3,drive=drive-scsi3,id=scsi3,device_id=drive-scsi3,write-cache=off' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/rbd-pve/fc4181a6-56eb-4f68-b452-8ba1f381ca2a/cpool/vm-8006-disk-0","node-name":"eb2c7a292f03b9f6d015cf83ae79730","read-only":false},"node-name":"fb2c7a292f03b9f6d015cf83ae79730","read-only":false},"node-name":"drive-scsi4","read-only":false,"throttle-group":"throttle-drive-scsi4"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=4,drive=drive-scsi4,id=scsi4,device_id=drive-scsi4,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/rbd-pve/fc4181a6-56eb-4f68-b452-8ba1f381ca2a/cpool/vm-8006-disk-0","node-name":"e5258ec75558b1f102af1e20e677fd0","read-only":false},"node-name":"f5258ec75558b1f102af1e20e677fd0","read-only":false},"node-name":"drive-scsi5","read-only":false,"throttle-group":"throttle-drive-scsi5"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=5,drive=drive-scsi5,id=scsi5,device_id=drive-scsi5,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/rbd-pve/fc4181a6-56eb-4f68-b452-8ba1f381ca2a/cpool/vm-8006-disk-0","node-name":"edb33cdcea8ec3e2225509c4945227e","read-only":false},"node-name":"fdb33cdcea8ec3e2225509c4945227e","read-only":false},"node-name":"drive-scsi6","read-only":false,"throttle-group":"throttle-drive-scsi6"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=6,drive=drive-scsi6,id=scsi6,device_id=drive-scsi6,write-cache=off' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/rbd-pve/fc4181a6-56eb-4f68-b452-8ba1f381ca2a/cpool/vm-8006-disk-0","node-name":"eb0b017124a47505c97a5da052e0141","read-only":false},"node-name":"fb0b017124a47505c97a5da052e0141","read-only":false},"node-name":"drive-scsi7","read-only":false,"throttle-group":"throttle-drive-scsi7"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=7,drive=drive-scsi7,id=scsi7,device_id=drive-scsi7,write-cache=off' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/simple-virtio-blk.conf ================================================ # TEST: Test for a basic configuration with a VirtIO Block IOThread disk bootdisk: virtio0 cores: 3 ide2: none,media=cdrom memory: 768 name: simple net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 virtio0: local:8006/vm-8006-disk-0.qcow2,discard=on,iothread=1,size=104858K vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 ================================================ FILE: src/test/cfg2cmd/simple-virtio-blk.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -object 'iothread,id=iothread-virtio0' \ -object '{"id":"throttle-drive-virtio0","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"edd19f6c1b3a6d5a6248c3376a91a16","read-only":false},"node-name":"fdd19f6c1b3a6d5a6248c3376a91a16","read-only":false},"node-name":"drive-virtio0","read-only":false,"throttle-group":"throttle-drive-virtio0"}' \ -device 'virtio-blk-pci,drive=drive-virtio0,id=virtio0,bus=pci.0,addr=0xa,iothread=iothread-virtio0,bootindex=100,write-cache=on' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/simple-zfs-over-iscsi.conf ================================================ # TEST: Simple test for zfs-over-scsi backed VM. bootdisk: scsi0 cores: 3 ide2: none,media=cdrom memory: 768 name: simple net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 scsi0: zfs-over-iscsi-store:vm-8006-disk-0,discard=on,size=104858K scsi1: zfs-over-iscsi-store:vm-8006-disk-0,cache=writeback,discard=on,size=104858K scsi2: zfs-over-iscsi-store:vm-8006-disk-0,cache=writethrough,discard=on,size=104858K scsi3: zfs-over-iscsi-store:vm-8006-disk-0,cache=directsync,discard=on,size=104858K scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 vmgenid: c773c261-d800-4348-1010-1010add53cf8 ================================================ FILE: src/test/cfg2cmd/simple-zfs-over-iscsi.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi2","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-scsi3","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-1010-1010add53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"iscsi","lun":0,"node-name":"e7106ac43d4f125a1911487dd9e3e42","portal":"127.0.0.1","read-only":false,"target":"iqn.2019-10.org.test:foobar","transport":"tcp"},"node-name":"f7106ac43d4f125a1911487dd9e3e42","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"iscsi","lun":0,"node-name":"efdb73e0d0acc5a60e3ff438cb20113","portal":"127.0.0.1","read-only":false,"target":"iqn.2019-10.org.test:foobar","transport":"tcp"},"node-name":"ffdb73e0d0acc5a60e3ff438cb20113","read-only":false},"node-name":"drive-scsi1","read-only":false,"throttle-group":"throttle-drive-scsi1"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi1,id=scsi1,device_id=drive-scsi1,write-cache=on' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"iscsi","lun":0,"node-name":"eab527a81b458aa9603dca5e2505f6e","portal":"127.0.0.1","read-only":false,"target":"iqn.2019-10.org.test:foobar","transport":"tcp"},"node-name":"fab527a81b458aa9603dca5e2505f6e","read-only":false},"node-name":"drive-scsi2","read-only":false,"throttle-group":"throttle-drive-scsi2"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=2,drive=drive-scsi2,id=scsi2,device_id=drive-scsi2,write-cache=off' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"iscsi","lun":0,"node-name":"e915a332310039f7a3feed6901eb5da","portal":"127.0.0.1","read-only":false,"target":"iqn.2019-10.org.test:foobar","transport":"tcp"},"node-name":"f915a332310039f7a3feed6901eb5da","read-only":false},"node-name":"drive-scsi3","read-only":false,"throttle-group":"throttle-drive-scsi3"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=3,drive=drive-scsi3,id=scsi3,device_id=drive-scsi3,write-cache=off' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/simple1-template.conf ================================================ # TEST: Simple test for a basic template configuration bootdisk: scsi0 cores: 3 ide2: none,media=cdrom memory: 768 name: simple net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 sata0: local:8006/base-8006-disk-0.qcow2,discard=on,size=104858K scsi0: local:8006/base-8006-disk-1.qcow2,discard=on,size=104858K scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 template: 1 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 ================================================ FILE: src/test/cfg2cmd/simple1-template.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vga none \ -nographic \ -cpu qemu64 \ -m 512 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -object '{"id":"throttle-drive-sata0","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/base-8006-disk-1.qcow2","node-name":"e1085774206ae4a6b6bf8426ff08f16","read-only":true},"node-name":"f1085774206ae4a6b6bf8426ff08f16","read-only":true},"node-name":"drive-scsi0","read-only":true,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,write-cache=on' \ -device 'ahci,id=ahci0,multifunction=on,bus=pci.0,addr=0x7' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/base-8006-disk-0.qcow2","node-name":"eab334c2e07734480f33dd80d89871b","read-only":true},"node-name":"fab334c2e07734480f33dd80d89871b","read-only":true},"node-name":"drive-sata0","read-only":true,"throttle-group":"throttle-drive-sata0"}' \ -device 'ide-cd,bus=ahci0.0,drive=drive-sata0,id=sata0,write-cache=on' \ -machine 'accel=tcg,smm=off,type=pc+pve0' \ -snapshot ================================================ FILE: src/test/cfg2cmd/simple1.conf ================================================ # TEST: Simple test for a basic configuration with no special things bootdisk: scsi0 cores: 3 ide2: none,media=cdrom memory: 768 name: simple net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 numa: 0 ostype: l26 scsi0: local:8006/vm-8006-disk-0.qcow2,discard=on,size=104858K scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 sockets: 1 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 ================================================ FILE: src/test/cfg2cmd/simple1.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'simple,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ -blockdev '{"detect-zeroes":"unmap","discard":"unmap","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"ecd04be4259153b8293415fefa2a84c","read-only":false},"node-name":"fcd04be4259153b8293415fefa2a84c","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/spice-enhancments.conf ================================================ # TEST: Test for SPICE enhancements machine: pc-i440fx-4.0 smbios1: uuid=363a6126-5f48-43e1-811f-013294a946a0 spice_enhancements: foldersharing=1,videostreaming=all vga: qxl vmgenid: 719b9591-1b0d-4e43-bca2-22b9ffbd3568 ================================================ FILE: src/test/cfg2cmd/spice-enhancments.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=363a6126-5f48-43e1-811f-013294a946a0' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=719b9591-1b0d-4e43-bca2-22b9ffbd3568' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'qxl-vga,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ -chardev 'spicevmc,id=vdagent,name=vdagent' \ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ -chardev 'spiceport,id=foldershare,name=org.spice-space.webdav.0' \ -device 'virtserialport,chardev=foldershare,name=org.spice-space.webdav.0' \ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on,streaming-video=all' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=pc-i440fx-4.0+pve0' ================================================ FILE: src/test/cfg2cmd/spice-linux-4.1.conf ================================================ # TEST: Test for SPICE with SPICE with max_outputs cores: 2 machine: pc-i440fx-4.1 memory: 768 name: spicelinux net0: virtio=A2:C0:43:67:08:A1,bridge=vmbr0 ostype: l26 scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 vga: qxl ================================================ FILE: src/test/cfg2cmd/spice-linux-4.1.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'spicelinux,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'qxl-vga,id=vga,max_outputs=4,bus=pci.0,addr=0x2' \ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ -chardev 'spicevmc,id=vdagent,name=vdagent' \ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:67:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -machine 'type=pc-i440fx-4.1+pve0' ================================================ FILE: src/test/cfg2cmd/spice-usb3.conf ================================================ # TEST: Test for SPICE with a USB3 based SPICE port added cores: 2 machine: pc-i440fx-4.0 memory: 768 name: spiceusb3 net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 ostype: l26 scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 vga: qxl usb1: spice,usb3=1 ================================================ FILE: src/test/cfg2cmd/spice-usb3.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'spiceusb3,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'nec-usb-xhci,id=xhci,bus=pci.1,addr=0x1b' \ -chardev 'spicevmc,id=usbredirchardev1,name=usbredir' \ -device 'usb-redir,chardev=usbredirchardev1,id=usbredirdev1,bus=xhci.0' \ -device 'qxl-vga,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ -chardev 'spicevmc,id=vdagent,name=vdagent' \ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -machine 'type=pc-i440fx-4.0+pve0' ================================================ FILE: src/test/cfg2cmd/spice-win.conf ================================================ # TEST: Test for SPICE under Win10 with a USB3 based SPICE port added cores: 2 machine: pc-i440fx-4.0 memory: 768 name: spiceusb3 net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 ostype: win10 scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec461 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf7 vga: qxl2 usb1: spice,usb3=1 ================================================ FILE: src/test/cfg2cmd/spice-win.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'spiceusb3,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec461' \ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -global 'kvm-pit.lost_tick_policy=discard' \ -cpu 'kvm64,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vpindex,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep' \ -m 768 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf7' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'nec-usb-xhci,id=xhci,bus=pci.1,addr=0x1b' \ -chardev 'spicevmc,id=usbredirchardev1,name=usbredir' \ -device 'usb-redir,chardev=usbredirchardev1,id=usbredirdev1,bus=xhci.0' \ -device 'qxl-vga,id=vga,bus=pci.0,addr=0x2' \ -device 'qxl,id=vga1,ram_size=67108864,vram_size=33554432,bus=pci.0,addr=0x18' \ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ -chardev 'spicevmc,id=vdagent,name=vdagent' \ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -rtc 'driftfix=slew,base=localtime' \ -machine 'hpet=off,type=pc-i440fx-4.0+pve0' ================================================ FILE: src/test/cfg2cmd/startdate-l26.conf ================================================ # TEST: Simple test for startdate parameter with Linux ostype: l26 startdate: 2006-06-17T16:01:21 ================================================ FILE: src/test/cfg2cmd/startdate-l26.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -rtc 'base=2006-06-17T16:01:21' \ -machine 'hpet=off,type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/startdate-win11.conf ================================================ # TEST: Simple test for startdate parameter with Windows and explicitly disabled time drift fix ostype: win11 tdf: 0 startdate: 2020-01-11 ================================================ FILE: src/test/cfg2cmd/startdate-win11.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -global 'kvm-pit.lost_tick_policy=discard' \ -cpu 'kvm64,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vpindex,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep' \ -m 512 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2,edid=off' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -rtc 'base=2020-01-11' \ -machine 'hpet=off,type=pc-i440fx-5.1+pve0' ================================================ FILE: src/test/cfg2cmd/unsupported-storage-content-type.conf ================================================ # TEST: Unsupported storage content type in a volume disk # EXPECT_ERROR: storage 'noimages' does not support content-type 'images' scsi0: noimages:8006/vm-8006-disk-0.raw,iothread=1,size=32G ================================================ FILE: src/test/cfg2cmd/usb13-error.conf ================================================ # TEST: Test error for old machine type with newer usb config # EXPECT_ERROR: using usb13 is only possible with machine type >= 7.1 and ostype l26 or windows > 7 cores: 2 machine: pc-i440fx-4.0 memory: 768 name: q35-usb3-error net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 ostype: l26 scsihw: virtio-scsi-pci smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 vga: qxl usb13: spice ================================================ FILE: src/test/cfg2cmd/vnc-clipboard-spice.conf ================================================ # TEST: Test for a VNC clipboard with a SPICE QXL display vga: qxl,clipboard=vnc ================================================ FILE: src/test/cfg2cmd/vnc-clipboard-spice.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'qxl-vga,id=vga,max_outputs=4,bus=pci.0,addr=0x2' \ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ -chardev 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on' \ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/cfg2cmd/vnc-clipboard-std.conf ================================================ # TEST: Test for a VNC clipboard with a std display vga: std,clipboard=vnc ================================================ FILE: src/test/cfg2cmd/vnc-clipboard-std.conf.cmd ================================================ /usr/bin/kvm \ -id 8006 \ -name 'vm8006,debug-threads=on' \ -no-shutdown \ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -global 'PIIX4_PM.disable_s3=1' \ -global 'PIIX4_PM.disable_s4=1' \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ -chardev 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on' \ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=pc+pve0' ================================================ FILE: src/test/parse-config-expected/cloudinit-snapshot.conf ================================================ boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 parent: cloudinit scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 sockets: 1 unused0: rbd:vm-120-disk-0 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 [special:cloudinit] ipconfig0: ip=dhcp,ip6=dhcp name: deb122 [cloudinit] boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 ostype: l26 scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 snaptime: 1737549549 sockets: 1 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 ================================================ FILE: src/test/parse-config-expected/cloudinit-snapshot.conf.strict.error ================================================ vm 8006 - unable to parse value of 'numa' - type check ('boolean') failed - got 'verify meee~ :)' ================================================ FILE: src/test/parse-config-expected/duplicate-sections.conf ================================================ boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip=dhcp,ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb122 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 parent: foo scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 sockets: 1 unused0: rbd:vm-120-disk-0 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 [PENDING] vga: qxl [special:cloudinit] name: deb123 [foo] boot: order=scsi0 cores: 4 cpu: host ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip=dhcp,ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 snaptime: 1737548747 sockets: 1 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 ================================================ FILE: src/test/parse-config-expected/duplicate-sections.conf.strict.error ================================================ vm 8006 - duplicate section: pending ================================================ FILE: src/test/parse-config-expected/unknown-sections.conf ================================================ boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 parent: foo scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 sockets: 1 unused0: rbd:vm-120-disk-0 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 [PENDING] bios: ovmf [special:cloudinit] ipconfig0: ip=dhcp,ip6=dhcp name: deb122 [foo] boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip=dhcp,ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 snaptime: 1737548747 sockets: 1 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 ================================================ FILE: src/test/parse-config-expected/unknown-sections.conf.strict.error ================================================ vm 8006 - skipping unknown section: 'special:unknown123' ================================================ FILE: src/test/parse-config-expected/verify-snapshot.conf ================================================ boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 parent: snap scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 sockets: 1 unused0: rbd:vm-120-disk-0 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 [snap] boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 ostype: l26 scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 snaptime: 1737549549 sockets: 1 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 ================================================ FILE: src/test/parse-config-expected/verify-snapshot.conf.strict.error ================================================ vm 8006 - unable to parse value of 'numa' - type check ('boolean') failed - got 'verify meee~ :)' ================================================ FILE: src/test/parse-config-input/cloudinit-snapshot.conf ================================================ boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 parent: cloudinit scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 sockets: 1 unused0: rbd:vm-120-disk-0 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 [special:cloudinit] ipconfig0: ip=dhcp,ip6=dhcp name: deb122 [cloudinit] boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: verify meee~ :) ostype: l26 scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 snaptime: 1737549549 sockets: 1 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 ================================================ FILE: src/test/parse-config-input/duplicate-sections.conf ================================================ boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip=dhcp,ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb122 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 parent: foo scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 sockets: 1 unused0: rbd:vm-120-disk-0 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 [PENDING] bios: ovmf [PENDING] vga: qxl [special:cloudinit] name: deb12 [special:cloudinit] name: deb123 [foo] boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip=dhcp,ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 snaptime: 1737548747 sockets: 1 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 [foo] boot: order=scsi0 cores: 4 cpu: host ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip=dhcp,ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 snaptime: 1737548747 sockets: 1 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 ================================================ FILE: src/test/parse-config-input/fleecing-section.conf ================================================ boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 sockets: 1 unused0: rbd:vm-120-disk-0 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 [special:fleecing] fleecing-images: zfs:vm-120-fleece-0 ================================================ FILE: src/test/parse-config-input/locked.conf ================================================ # locked bootdisk: scsi0 cores: 1 ide2: none,media=cdrom lock: backup memory: 512 name: apache net0: virtio=92:38:11:FD:ED:87,bridge=vmbr0,firewall=1 numa: 0 ostype: l26 scsi0: mydir:1422/vm-1422-disk-0.qcow2,size=4G scsihw: virtio-scsi-pci smbios1: uuid=ddf91b3f-a597-42be-9a7e-fb6421dcd5cd sockets: 1 unused7: mydir:1422/vm-1422-disk-8.qcow2 vmgenid: 0 ================================================ FILE: src/test/parse-config-input/plain.conf ================================================ # plain VM bootdisk: scsi0 cores: 1 ide2: none,media=cdrom memory: 512 name: apache net0: virtio=92:38:11:FD:ED:87,bridge=vmbr0,firewall=1 numa: 0 ostype: l26 scsi0: mydir:142/vm-142-disk-0.qcow2,size=4G scsihw: virtio-scsi-pci smbios1: uuid=ddf91b3f-a597-42be-9a7e-fb6421dcd5cd sockets: 1 tags: foo bar vmgenid: 0 ================================================ FILE: src/test/parse-config-input/regular-vm-efi.conf ================================================ # regular VM with an EFI disk bios: ovmf boot: order=scsi0;ide2;net0 cores: 1 efidisk0: mydir:139/vm-139-disk-0.qcow2,size=128K ide2: local:iso/debian-10.6.0-amd64-netinst.iso,media=cdrom memory: 2048 name: eficloneclone net0: virtio=7A:6C:A5:8B:11:93,bridge=vmbr0,firewall=1 numa: 0 ostype: l26 scsi0: rbdkvm:vm-139-disk-1,size=4G scsihw: virtio-scsi-pci smbios1: uuid=21a7e7bc-3cd2-4232-a009-a41f4ee992ae sockets: 1 vmgenid: 0 ================================================ FILE: src/test/parse-config-input/sections.conf ================================================ boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 parent: foo scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 sockets: 1 unused0: rbd:vm-120-disk-0 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 [PENDING] bios: ovmf [special:cloudinit] ipconfig0: ip=dhcp,ip6=dhcp name: deb122 [foo] boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip=dhcp,ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 snaptime: 1737548747 sockets: 1 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 ================================================ FILE: src/test/parse-config-input/snapshots.conf ================================================ boot: order=scsi1;ide2;net0;ide1 cores: 4 cpu: x86-64-v2-AES ide0: dir:111/vm-111-disk-2.qcow2,size=1G ide1: sani:iso/virtio-win-0.1.266.iso,media=cdrom,size=707456K ide2: sani:iso/Win2019-evaluation.iso,media=cdrom,size=4985424K machine: pc-i440fx-9.1 memory: 4096 meta: creation-qemu=9.1.2,ctime=1736349024 name: win-machine-ver net0: virtio=BC:24:11:A3:DA:B1,bridge=vnet0,firewall=1 net1: e1000=BC:24:11:79:D5:65,bridge=vnet0,firewall=1 numa: 0 ostype: win10 parent: win19_5_2_plus_stuff scsi0: dir:111/vm-111-disk-1.qcow2,iothread=1,size=1G scsi1: lvmthinbig:vm-111-disk-0,iothread=1,size=32G scsihw: virtio-scsi-single smbios1: uuid=2c4a2cda-712b-44ab-8728-51f5e734b658 sockets: 1 unused0: rbd:vm-111-disk-0 vga: qxl virtio0: dir:111/vm-111-disk-0.qcow2,iothread=1,size=1G vmgenid: 713da648-38a6-489e-b0b2-dd9cef419f33 [machine_version_5_1] boot: order=ide0;ide2;net0 cores: 4 cpu: x86-64-v2-AES ide0: lvmthinbig:vm-111-disk-0,size=32G ide2: sani:iso/Win2016-1616-evaluation.ISO,media=cdrom,size=5198078K memory: 4096 meta: creation-qemu=9.1.2,ctime=1736349024 name: win-machine-ver net0: e1000=BC:24:11:A3:DA:B1,bridge=vnet0,firewall=1 numa: 0 ostype: win10 scsihw: virtio-scsi-single smbios1: uuid=2c4a2cda-712b-44ab-8728-51f5e734b658 snaptime: 1736939109 sockets: 1 vmgenid: 1f314a76-50a3-4b92-9307-c8c6e313d3ca [machine_version_5_1_with_virtio] boot: order=ide0;ide2;net0;ide1 cores: 4 cpu: x86-64-v2-AES ide0: lvmthinbig:vm-111-disk-0,size=32G ide1: sani:iso/virtio-win-0.1.266.iso,media=cdrom,size=707456K ide2: sani:iso/Win2016-1616-evaluation.ISO,media=cdrom,size=5198078K memory: 4096 meta: creation-qemu=9.1.2,ctime=1736349024 name: win-machine-ver net0: virtio=BC:24:11:A3:DA:B1,bridge=vnet0,firewall=1 numa: 0 ostype: win10 parent: machine_version_5_1 scsi0: dir:111/vm-111-disk-1.qcow2,iothread=1,size=1G scsihw: virtio-scsi-single smbios1: uuid=2c4a2cda-712b-44ab-8728-51f5e734b658 snaptime: 1736940462 sockets: 1 virtio0: dir:111/vm-111-disk-0.qcow2,iothread=1,size=1G vmgenid: 4f602356-cb9c-45ad-a554-d76d95c7c0f8 [ovmf_machine_version_5_1] bios: ovmf boot: order=ide0;ide2;net0;ide1 cores: 4 cpu: x86-64-v2-AES efidisk0: rbd:vm-111-disk-0,efitype=4m,pre-enrolled-keys=1,size=1M ide0: lvmthinbig:vm-111-disk-0,size=32G ide1: sani:iso/virtio-win-0.1.266.iso,media=cdrom,size=707456K ide2: sani:iso/Win2016-1616-evaluation.ISO,media=cdrom,size=5198078K machine: pc-q35-5.1 memory: 4096 meta: creation-qemu=9.1.2,ctime=1736349024 name: win-machine-ver net0: e1000=BC:24:11:A3:DA:B1,bridge=vnet0,firewall=1 numa: 0 ostype: win10 parent: machine_version_5_1_with_virtio scsi0: dir:111/vm-111-disk-1.qcow2,iothread=1,size=1G scsihw: virtio-scsi-single smbios1: uuid=2c4a2cda-712b-44ab-8728-51f5e734b658 snaptime: 1736943308 sockets: 1 virtio0: dir:111/vm-111-disk-0.qcow2,iothread=1,size=1G vmgenid: 4f602356-cb9c-45ad-a554-d76d95c7c0f8 [ovmf_machine_version_5_1_virtio] bios: ovmf boot: order=ide0;ide2;net0;ide1 cores: 4 cpu: x86-64-v2-AES efidisk0: rbd:vm-111-disk-0,efitype=4m,pre-enrolled-keys=1,size=1M ide0: lvmthinbig:vm-111-disk-0,size=32G ide1: sani:iso/virtio-win-0.1.266.iso,media=cdrom,size=707456K ide2: sani:iso/Win2016-1616-evaluation.ISO,media=cdrom,size=5198078K machine: pc-q35-5.1 memory: 4096 meta: creation-qemu=9.1.2,ctime=1736349024 name: win-machine-ver net0: virtio=BC:24:11:A3:DA:B1,bridge=vnet0,firewall=1 numa: 0 ostype: win10 parent: ovmf_machine_version_5_1 scsi0: dir:111/vm-111-disk-1.qcow2,iothread=1,size=1G scsihw: virtio-scsi-single smbios1: uuid=2c4a2cda-712b-44ab-8728-51f5e734b658 snaptime: 1736944525 sockets: 1 virtio0: dir:111/vm-111-disk-0.qcow2,iothread=1,size=1G vmgenid: 00b95468-4f34-4faa-b0af-b214ff5bbcdf [static-network] bios: ovmf boot: order=ide0;ide2;net0;ide1 cores: 4 cpu: x86-64-v2-AES efidisk0: rbd:vm-111-disk-0,efitype=4m,pre-enrolled-keys=1,size=1M ide0: lvmthinbig:vm-111-disk-0,size=32G ide1: sani:iso/virtio-win-0.1.266.iso,media=cdrom,size=707456K ide2: sani:iso/Win2016-1616-evaluation.ISO,media=cdrom,size=5198078K machine: pc-q35-5.1 memory: 4096 meta: creation-qemu=9.1.2,ctime=1736349024 name: win-machine-ver net0: virtio=BC:24:11:A3:DA:B1,bridge=vnet0,firewall=1 numa: 0 ostype: win10 parent: ovmf_machine_version_5_1_virtio scsi0: dir:111/vm-111-disk-1.qcow2,iothread=1,size=1G scsihw: virtio-scsi-single smbios1: uuid=2c4a2cda-712b-44ab-8728-51f5e734b658 snaptime: 1736945713 sockets: 1 virtio0: dir:111/vm-111-disk-0.qcow2,iothread=1,size=1G vmgenid: 5d65fc62-2cb1-4945-9641-631b37c265a5 [win19_5_2] boot: order=scsi1;ide2;net0;ide1 cores: 4 cpu: x86-64-v2-AES ide1: sani:iso/virtio-win-0.1.266.iso,media=cdrom,size=707456K ide2: sani:iso/Win2019-evaluation.iso,media=cdrom,size=4985424K machine: pc-i440fx-5.2 memory: 4096 meta: creation-qemu=9.1.2,ctime=1736349024 name: win-machine-ver net0: virtio=BC:24:11:A3:DA:B1,bridge=vnet0,firewall=1 net1: e1000=BC:24:11:79:D5:65,bridge=vnet0,firewall=1 numa: 0 ostype: win10 parent: machine_version_5_1_with_virtio scsi0: dir:111/vm-111-disk-1.qcow2,iothread=1,size=1G scsi1: lvmthinbig:vm-111-disk-0,iothread=1,size=32G scsihw: virtio-scsi-single smbios1: uuid=2c4a2cda-712b-44ab-8728-51f5e734b658 snaptime: 1736950690 sockets: 1 virtio0: dir:111/vm-111-disk-0.qcow2,iothread=1,size=1G vmgenid: f259de06-fa08-4ff7-8ba9-b1233a726ac4 [win19_5_2_plus_stuff] boot: order=scsi1;ide2;net0;ide1 cores: 4 cpu: x86-64-v2-AES ide0: dir:111/vm-111-disk-2.qcow2,size=1G ide1: sani:iso/virtio-win-0.1.266.iso,media=cdrom,size=707456K ide2: sani:iso/Win2019-evaluation.iso,media=cdrom,size=4985424K machine: pc-i440fx-5.2 memory: 4096 meta: creation-qemu=9.1.2,ctime=1736349024 name: win-machine-ver net0: virtio=BC:24:11:A3:DA:B1,bridge=vnet0,firewall=1 net1: e1000=BC:24:11:79:D5:65,bridge=vnet0,firewall=1 numa: 0 ostype: win10 parent: win19_5_2 scsi0: dir:111/vm-111-disk-1.qcow2,iothread=1,size=1G scsi1: lvmthinbig:vm-111-disk-0,iothread=1,size=32G scsihw: virtio-scsi-single smbios1: uuid=2c4a2cda-712b-44ab-8728-51f5e734b658 snaptime: 1736951300 sockets: 1 vga: qxl virtio0: dir:111/vm-111-disk-0.qcow2,iothread=1,size=1G vmgenid: 713da648-38a6-489e-b0b2-dd9cef419f33 ================================================ FILE: src/test/parse-config-input/unknown-sections.conf ================================================ boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 parent: foo scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 sockets: 1 unused0: rbd:vm-120-disk-0 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 [special:unknown123] name: foo [PENDING] bios: ovmf [special:unknown124] bios: seabios [special:cloudinit] ipconfig0: ip=dhcp,ip6=dhcp name: deb122 [special:unknown125] name: bar [foo] boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip=dhcp,ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 snaptime: 1737548747 sockets: 1 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 [:3] name: baz cat: nya~ ================================================ FILE: src/test/parse-config-input/verify-snapshot.conf ================================================ boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: 0 ostype: l26 parent: snap scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 sockets: 1 unused0: rbd:vm-120-disk-0 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 [snap] boot: order=scsi0 cores: 2 cpu: x86-64-v2-AES ide2: lvm:vm-120-cloudinit,media=cdrom ipconfig0: ip6=dhcp memory: 4096 meta: creation-qemu=9.0.2,ctime=1725975013 name: deb1223 net0: vmxnet3=BC:24:11:2C:69:EC,bridge=vnet0,firewall=1 numa: verify meee~ :) ostype: l26 scsi0: nfs:120/vm-120-disk-0.qcow2,iothread=1,size=4G scsihw: virtio-scsi-single smbios1: uuid=b3247ab1-1fe6-428e-965b-08a1b64a8746 snaptime: 1737549549 sockets: 1 vmgenid: 7079e97c-50e3-4079-afe7-23e67566b946 ================================================ FILE: src/test/restore-config-expected/139.conf ================================================ # regular VM with an EFI disk bios: ovmf boot: order=scsi0;ide2;net0 cores: 1 efidisk0: target:139/vm-139-disk-0.qcow2,size=128K ide2: local:iso/debian-10.6.0-amd64-netinst.iso,media=cdrom memory: 2048 name: eficloneclone net0: virtio=7A:6C:A5:8B:11:93,bridge=vmbr0,firewall=1 numa: 0 ostype: l26 scsi0: target:139/vm-139-disk-1.raw,size=4G scsihw: virtio-scsi-pci smbios1: uuid=21a7e7bc-3cd2-4232-a009-a41f4ee992ae sockets: 1 vmgenid: 0 ================================================ FILE: src/test/restore-config-expected/142.conf ================================================ # plain VM bootdisk: scsi0 cores: 1 ide2: none,media=cdrom memory: 512 name: apache net0: virtio=92:38:11:FD:ED:87,bridge=vmbr0,firewall=1 numa: 0 ostype: l26 scsi0: target:142/vm-142-disk-0.qcow2,size=4G scsihw: virtio-scsi-pci smbios1: uuid=ddf91b3f-a597-42be-9a7e-fb6421dcd5cd sockets: 1 vmgenid: 0 tags: foo bar ================================================ FILE: src/test/restore-config-expected/1422.conf ================================================ # some properties should be filtered bootdisk: scsi0 cores: 1 ide2: none,media=cdrom memory: 512 name: apache net0: virtio=92:38:11:FD:ED:87,bridge=vmbr0,firewall=1 numa: 0 ostype: l26 scsi0: target:1422/vm-1422-disk-0.qcow2,size=4G scsihw: virtio-scsi-pci smbios1: uuid=ddf91b3f-a597-42be-9a7e-fb6421dcd5cd sockets: 1 vmgenid: 0 ================================================ FILE: src/test/restore-config-expected/179.conf ================================================ # many disks boot: order=scsi0;ide2;net0 cores: 1 ide2: none,media=cdrom memory: 2048 net0: virtio=26:15:5B:73:3F:7C,bridge=vmbr0,firewall=1 numa: 0 ostype: l26 scsi0: target:179/vm-179-disk-0.qcow2,cache=none,discard=on,size=32G,ssd=1 scsi1: target:179/vm-179-disk-1.qcow2,cache=writethrough,size=32G scsi2: target:179/vm-179-disk-2.qcow2,mbps_rd=7,mbps_wr=7,replicate=0,size=32G scsi3: target:179/vm-179-disk-3.vmdk,size=32G #scsi4: myfs:179/vm-179-disk-1.qcow2,backup=0,size=32G scsihw: virtio-scsi-pci smbios1: uuid=1819ead7-a55d-4544-8d38-29ca94869a9c sockets: 1 vmgenid: 0 ================================================ FILE: src/test/restore-config-input/139.conf ================================================ # regular VM with an EFI disk bios: ovmf boot: order=scsi0;ide2;net0 cores: 1 efidisk0: mydir:139/vm-139-disk-0.qcow2,size=128K ide2: local:iso/debian-10.6.0-amd64-netinst.iso,media=cdrom memory: 2048 name: eficloneclone net0: virtio=7A:6C:A5:8B:11:93,bridge=vmbr0,firewall=1 numa: 0 ostype: l26 scsi0: rbdkvm:vm-139-disk-1,size=4G scsihw: virtio-scsi-pci smbios1: uuid=21a7e7bc-3cd2-4232-a009-a41f4ee992ae sockets: 1 vmgenid: 0 #qmdump#map:efidisk0:drive-efidisk0:mydir:qcow2: #qmdump#map:scsi0:drive-scsi0:rbdkvm:: ================================================ FILE: src/test/restore-config-input/142.conf ================================================ # plain VM bootdisk: scsi0 cores: 1 ide2: none,media=cdrom memory: 512 name: apache net0: virtio=92:38:11:FD:ED:87,bridge=vmbr0,firewall=1 numa: 0 ostype: l26 scsi0: mydir:142/vm-142-disk-0.qcow2,size=4G scsihw: virtio-scsi-pci smbios1: uuid=ddf91b3f-a597-42be-9a7e-fb6421dcd5cd sockets: 1 vmgenid: 0 tags: foo bar #qmdump#map:scsi0:drive-scsi0:mydir:qcow2: ================================================ FILE: src/test/restore-config-input/1422.conf ================================================ # some properties should be filtered bootdisk: scsi0 cores: 1 ide2: none,media=cdrom memory: 512 name: apache net0: virtio=92:38:11:FD:ED:87,bridge=vmbr0,firewall=1 numa: 0 ostype: l26 scsi0: mydir:1422/vm-1422-disk-0.qcow2,size=4G unused7: mydir:1422/vm-1422-disk-8.qcow2 parent: snap lock: backup scsihw: virtio-scsi-pci smbios1: uuid=ddf91b3f-a597-42be-9a7e-fb6421dcd5cd sockets: 1 vmgenid: 0 #qmdump#map:scsi0:drive-scsi0:mydir:qcow2: ================================================ FILE: src/test/restore-config-input/179.conf ================================================ # many disks boot: order=scsi0;ide2;net0 cores: 1 ide2: none,media=cdrom memory: 2048 net0: virtio=26:15:5B:73:3F:7C,bridge=vmbr0,firewall=1 numa: 0 ostype: l26 scsi0: myfs:179/vm-179-disk-4.qcow2,cache=none,discard=on,size=32G,ssd=1 scsi1: myfs:179/vm-179-disk-0.qcow2,cache=writethrough,size=32G scsi2: myfs:179/vm-179-disk-2.qcow2,mbps_rd=7,mbps_wr=7,replicate=0,size=32G scsi3: myfs:179/vm-179-disk-3.vmdk,size=32G scsi4: myfs:179/vm-179-disk-1.qcow2,backup=0,size=32G scsihw: virtio-scsi-pci smbios1: uuid=1819ead7-a55d-4544-8d38-29ca94869a9c sockets: 1 vmgenid: 0 #qmdump#map:scsi0:drive-scsi0:myfs:qcow2: #qmdump#map:scsi1:drive-scsi1:myfs:qcow2: #qmdump#map:scsi2:drive-scsi2:myfs:qcow2: #qmdump#map:scsi3:drive-scsi3:myfs:vmdk: ================================================ FILE: src/test/run_config2command_tests.pl ================================================ #!/usr/bin/perl use v5.36; use lib qw(..); use JSON qw(decode_json); use Test::More; use Test::MockModule; use Socket qw(AF_INET AF_INET6); use PVE::File qw(file_get_contents file_set_contents); use PVE::Tools qw(run_command); use PVE::INotify; use PVE::SysFSTools; use PVE::QemuConfig; use PVE::QemuServer; use PVE::QemuServer::Drive; use PVE::QemuServer::Helpers; use PVE::QemuServer::Monitor; use PVE::QemuServer::OVMF; use PVE::QemuServer::QMPHelpers; use PVE::QemuServer::CPUConfig; use PVE::Storage; my $base_env = { storage_config => { ids => { local => { content => { images => 1, iso => 1, }, path => '/var/lib/vz', type => 'dir', shared => 0, }, localsnapext => { content => { images => 1, }, path => '/var/lib/vzsnapext', type => 'dir', shared => 0, 'snapshot-as-volume-chain' => 1, }, noimages => { content => { iso => 1, }, path => '/var/lib/vz', type => 'dir', }, 'btrfs-store' => { content => { images => 1, }, path => '/butter/bread', type => 'btrfs', }, 'cifs-store' => { shared => 1, path => '/mnt/pve/cifs-store', username => 'guest', server => '127.0.0.42', type => 'cifs', share => 'CIFShare', content => { images => 1, iso => 1, }, }, 'rbd-store' => { monhost => '127.0.0.42,127.0.0.21,::1', fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a', content => { images => 1, }, type => 'rbd', pool => 'cpool', username => 'admin', shared => 1, }, 'krbd-store' => { monhost => '127.0.0.42,127.0.0.21,::1', fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a', content => { images => 1, }, type => 'rbd', pool => 'cpool', username => 'admin', shared => 1, krbd => 1, }, 'zfs-over-iscsi-store' => { type => 'zfs', iscsiprovider => "comstar", lio_tpg => "tpg1", portal => "127.0.0.1", target => "iqn.2019-10.org.test:foobar", pool => "tank", content => { images => 1, }, }, 'lvm-store' => { vgname => 'veegee', type => 'lvm', content => { images => 1, }, }, 'local-lvm' => { vgname => 'pve', bwlimit => 'restore=1024', type => 'lvmthin', thinpool => 'data', content => { images => 1, }, }, }, }, vmid => 8006, real_qemu_version => PVE::QemuServer::Helpers::kvm_user_version(), # not yet mocked }; my $pci_devs = [ "0000:00:43.1", "0000:00:f4.0", "0000:00:ff.1", "0000:0f:f2.0", "0000:d0:13.0", "0000:d0:15.1", "0000:d0:15.2", "0000:d0:17.0", "0000:f0:42.0", "0000:f0:43.0", "0000:f0:43.1", "1234:f0:43.1", "0000:01:00.4", "0000:01:00.5", "0000:01:00.6", "0000:07:10.0", "0000:07:10.1", "0000:07:10.4", ]; my $pci_map_config = { ids => { someGpu => { type => 'pci', mdev => 1, map => [ 'node=localhost,path=0000:01:00.4,id=10de:2231,iommugroup=1', 'node=localhost,path=0000:01:00.5,id=10de:2231,iommugroup=1', 'node=localhost,path=0000:01:00.6,id=10de:2231,iommugroup=1', ], }, someNic => { type => 'pci', map => [ 'node=localhost,path=0000:07:10.0,id=8086:1520,iommugroup=2', 'node=localhost,path=0000:07:10.1,id=8086:1520,iommugroup=2', 'node=localhost,path=0000:07:10.4,id=8086:1520,iommugroup=2', ], }, }, }; my $usb_map_config = {}, my $cpu_hw_capabilities = { # Gathered from an AMD EPYC 9475F running kernel 6.11.11-2-pve 'amd-turin-9005' => '{ "amd-sev": { "cbitpos": 51, "reduced-phys-bits": 6, "sev-support": true,' . ' "sev-support-es": true, "sev-support-snp": true } }', # TODO: others? }; my $current_test; # = { # description => 'Test description', # if available # qemu_version => '2.12', # host_arch => 'HOST_ARCH', # expected_error => 'error message', # expected_warning => 'warning message', # config => { config hash }, # expected => [ expected outcome cmd line array ], # }; # use the config description to allow changing environment, fields are: # TEST: A single line describing the test, gets outputted # QEMU_VERSION: \d+\.\d+(\.\d+)? (defaults to current version) # HOST_ARCH: x86_64 | aarch64 (default to x86_64, to make tests stable) # EXPECT_ERROR: For negative tests # EXPECT_WARN(ING): that is expected # HW_CAPABILITIES: to defined the HW caps the test should expose # all fields are optional sub parse_test($config_fn) { $current_test = {}; # reset my $fake_config_fn = "$config_fn/qemu-server/8006.conf"; my $config_raw = file_get_contents($config_fn); my $config = PVE::QemuServer::parse_vm_config($fake_config_fn, $config_raw); $current_test->{config} = $config; my $description = $config->{description} // ''; while ($description =~ /^\h*(.*?)\h*$/gm) { my $line = $1; next if !$line || $line =~ /^#/; $line =~ s/^\s+//; $line =~ s/\s+$//; if ($line =~ /^TEST:\s*(.*)\s*$/) { $current_test->{description} = "$1"; } elsif ($line =~ /^QEMU_VERSION:\s*(.*)\s*$/) { $current_test->{qemu_version} = "$1"; } elsif ($line =~ /^HOST_ARCH:\s*(.*)\s*$/) { $current_test->{host_arch} = "$1"; } elsif ($line =~ /^EXPECT_ERROR:\s*(.*)\s*$/) { $current_test->{expect_error} = "$1"; } elsif ($line =~ /^EXPECT_WARN(?:ING)?:\s*(.*)\s*$/) { $current_test->{expect_warning} = "$1"; } elsif ($line =~ /^HW_CAPABILITIES:\s*(.*)\s*$/) { $current_test->{hw_capabilities} = "$1"; # either a CPU from above hash or raw JSON } } $config_fn =~ /([^\/]+)$/; my $testname = "$1"; if (my $desc = $current_test->{description}) { $testname = "'$testname' - $desc"; } $current_test->{testname} = $testname; PVE::QemuServer::CPUConfig::initialize_cpu_models(); } sub get_test_qemu_version { $current_test->{qemu_version} // $base_env->{real_qemu_version} // '2.12'; } my $qemu_server_module; $qemu_server_module = Test::MockModule->new('PVE::QemuServer'); $qemu_server_module->mock( kvm_user_version => sub { return get_test_qemu_version(); }, kvm_version => sub { return get_test_qemu_version(); }, kernel_has_vhost_net => sub { return 1; # TODO: make this per-test configurable? }, get_iscsi_initiator_name => sub { return 'iqn.1993-08.org.debian:01:aabbccddeeff'; }, cleanup_pci_devices => { # do nothing }, ); my $qemu_server_ovmf_module = Test::MockModule->new("PVE::QemuServer::OVMF"); $qemu_server_ovmf_module->mock( file_exists => sub { my ($path) = @_; return 1; }, file_get_size => sub { my ($path) = @_; if ($path =~ m/OVMF(32)?_(SEV_)?VARS_4M/) { return 528 * 1024; } elsif ($path =~ m/OVMF_VARS/) { return 128 * 1024; } elsif ($path =~ m/AAVMF_VARS/) { return 64 * 1024 * 1024; } elsif ($path =~ m/RISCV_VIRT_VARS/) { return 32 * 1024 * 1024; } else { die "unknown ovmf vars image '$path' - implement me"; } }, ); my $storage_module = Test::MockModule->new("PVE::Storage"); $storage_module->mock( activate_volumes => sub { return; }, deactivate_volumes => sub { return; }, volume_snapshot_info => sub { my ($cfg, $volid) = @_; my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); my $snapshots = {}; if ($storeid eq 'localsnapext') { $snapshots = { current => { file => 'var/lib/vzsnapext/images/8006/vm-8006-disk-0.qcow2', parent => 'snap2', }, snap2 => { file => '/var/lib/vzsnapext/images/8006/snap2-vm-8006-disk-0.qcow2', parent => 'snap1', }, snap1 => { file => '/var/lib/vzsnapext/images/8006/snap1-vm-8006-disk-0.qcow2', }, }; } elsif ($storeid eq 'lvm-store') { $snapshots = { current => { file => '/dev/veegee/vm-8006-disk-0.qcow2', parent => 'snap2', }, snap2 => { file => '/dev/veegee/snap2-vm-8006-disk-0.qcow2', parent => 'snap1', }, snap1 => { file => '/dev/veegee/snap1-vm-8006-disk-0.qcow2', }, }; } return $snapshots; }, ); my $file_stat_module = Test::MockModule->new("File::stat"); $file_stat_module->mock( stat => sub { my ($path) = @_; my $st = $file_stat_module->original('stat')->('./run_config2command_tests.pl'); $st->[2] = 25008 if $path =~ m!/dev/!; # block device return $st; }, ); my $zfsplugin_module = Test::MockModule->new("PVE::Storage::ZFSPlugin"); $zfsplugin_module->mock( zfs_get_lu_name => sub { return "foobar"; }, zfs_get_lun_number => sub { return "0"; }, ); my $rbdplugin_module = Test::MockModule->new("PVE::Storage::RBDPlugin"); $rbdplugin_module->mock( rbd_volume_config_set => sub { return; }, ); my $qemu_server_config; $qemu_server_config = Test::MockModule->new('PVE::QemuConfig'); $qemu_server_config->mock( load_config => sub { my ($class, $vmid, $node) = @_; return $current_test->{config}; }, ); my $qemu_server_helpers; $qemu_server_helpers = Test::MockModule->new('PVE::QemuServer::Helpers'); $qemu_server_helpers->mock( get_host_phys_address_bits => sub { return 46; }, ); my $qemu_server_memory; $qemu_server_memory = Test::MockModule->new('PVE::QemuServer::Memory'); $qemu_server_memory->mock( hugepages_chunk_size_supported => sub { return 1; }, host_numanode_exists => sub { my ($id) = @_; return 1; }, ); my $pve_common_tools; $pve_common_tools = Test::MockModule->new('PVE::Tools'); $pve_common_tools->mock( next_vnc_port => sub { my ($family, $address) = @_; return '5900'; }, next_spice_port => sub { my ($family, $address) = @_; return '61000'; }, getaddrinfo_all => sub { my ($hostname, @opts) = @_; die "need stable hostname" if $hostname ne 'localhost'; return ( { addr => Socket::pack_sockaddr_in(0, Socket::INADDR_LOOPBACK), family => AF_INET, # IPv4 protocol => 6, socktype => 1, }, ); }, get_host_arch => sub { return $current_test->{host_arch} // 'x86_64'; }, ); my $pve_cpuconfig; $pve_cpuconfig = Test::MockModule->new('PVE::QemuServer::CPUConfig'); $pve_cpuconfig->mock( load_custom_model_conf => sub { # mock custom CPU model config return PVE::QemuServer::CPUConfig->parse_config( "cpu-models.conf", < sub { my $hw_capabilities_raw; if (!defined($current_test->{hw_capabilities})) { # default to barebone uncapable HW $hw_capabilities_raw = '{"amd-sev":{"cbitpos":0,"reduced-phys-bits":0,"sev-support":false,' . '"sev-support-es":false,"sev-support-snp":false}}'; } elsif ( my $cpu_hw_caps = $cpu_hw_capabilities->{ lc($current_test->{hw_capabilities}) } ) { $hw_capabilities_raw = $cpu_hw_caps; } else { $hw_capabilities_raw = $current_test->{hw_capabilities}; } my $hw_capabilities = decode_json($hw_capabilities_raw); return $hw_capabilities; }, ); my $pve_common_network; $pve_common_network = Test::MockModule->new('PVE::Network'); $pve_common_network->mock( read_bridge_mtu => sub { my ($bridge_name) = @_; if ($bridge_name eq 'vxlan_bridge') { return 1450; } return 1500; }, ); my $pve_common_inotify; $pve_common_inotify = Test::MockModule->new('PVE::INotify'); $pve_common_inotify->mock( nodename => sub { return 'localhost'; }, ); my $pve_common_sysfstools; $pve_common_sysfstools = Test::MockModule->new('PVE::SysFSTools'); $pve_common_sysfstools->mock( lspci => sub { my ($filter, $verbose) = @_; return [ map { { id => $_ } } grep { !defined($filter) || (!ref($filter) && $_ =~ m/^(0000:)?\Q$filter\E/) || (ref($filter) eq 'CODE' && $filter->({ id => $_ })) } sort @$pci_devs ]; }, pci_device_info => sub { my ($path, $noerr) = @_; if ($path =~ m/^0000:01:00/) { return { mdev => 1, iommugroup => 1, mdev => 1, vendor => "0x10de", device => "0x2231", }; } elsif ($path =~ m/^0000:07:10/) { return { iommugroup => 2, vendor => "0x8086", device => "0x1520", }; } else { return {}; } }, ); my $qemu_drive_module; $qemu_drive_module = Test::MockModule->new('PVE::QemuServer::Drive'); $qemu_drive_module->mock( get_cdrom_path => sub { return "/dev/cdrom"; }, ); my $qemu_monitor_module; $qemu_monitor_module = Test::MockModule->new('PVE::QemuServer::Monitor'); $qemu_monitor_module->mock( mon_cmd => sub { my ($vmid, $cmd) = @_; die "invalid vmid: $vmid (expected: $base_env->{vmid})" if $vmid != $base_env->{vmid}; if ($cmd eq 'query-version') { my $ver = get_test_qemu_version(); $ver =~ m/(\d+)\.(\d+)(?:\.(\d+))?/; return { qemu => { major => $1, minor => $2, micro => $3, }, }; } die "unexpected QMP command: '$cmd'"; }, ); my $mapping_usb_module = Test::MockModule->new("PVE::Mapping::USB"); $mapping_usb_module->mock( config => sub { return $usb_map_config; }, ); my $mapping_pci_module = Test::MockModule->new("PVE::Mapping::PCI"); $mapping_pci_module->mock( config => sub { return $pci_map_config; }, ); my $pci_module = Test::MockModule->new("PVE::QemuServer::PCI"); $pci_module->mock( reserve_pci_usage => sub { die "reserve_pci_usage should not be called for 'qm showcmd'\n"; }, create_nvidia_device => sub { die "create_nvidia_device should not be called for 'qm showcmd'\n"; }, ); sub diff($a, $b) { return if $a eq $b; my ($ra, $wa) = POSIX::pipe(); my ($rb, $wb) = POSIX::pipe(); my $ha = IO::Handle->new_from_fd($wa, 'w'); my $hb = IO::Handle->new_from_fd($wb, 'w'); open my $diffproc, '-|', 'diff', '-up', "/proc/self/fd/$ra", "/proc/self/fd/$rb" ## no critic or die "failed to run program 'diff': $!"; POSIX::close($ra); POSIX::close($rb); open my $f1, '<', \$a; open my $f2, '<', \$b; my ($line1, $line2); do { $ha->print($line1) if defined($line1 = <$f1>); $hb->print($line2) if defined($line2 = <$f2>); } while (defined($line1 // $line2)); close $f1; close $f2; close $ha; close $hb; local $/ = undef; my $diff = <$diffproc>; close $diffproc; die "files differ:\n$diff"; } $SIG{__WARN__} = sub { my $warning = shift; chomp $warning; if (my $warn_expect = $current_test->{expect_warning}) { if ($warn_expect ne $warning) { fail($current_test->{testname}); note("warning does not match expected error: '$warning' != '$warn_expect'"); } else { note("got expected warning '$warning'"); return; } } fail($current_test->{testname}); note("got unexpected warning '$warning'"); }; sub do_test($config_fn) { die "no such input test config: $config_fn\n" if !-f $config_fn; parse_test $config_fn; my $testname = $current_test->{testname}; my ($vmid, $storecfg) = $base_env->@{qw(vmid storage_config)}; my $cmdline = eval { PVE::QemuServer::vm_commandline($storecfg, $vmid) }; my $err = $@; if (my $err_expect = $current_test->{expect_error}) { if (!$err) { fail("$testname"); note("did NOT get any error, but expected error: $err_expect"); return; } chomp $err; if ($err ne $err_expect) { fail("$testname"); note("error does not match expected error: '$err' !~ '$err_expect'"); } else { pass("$testname"); } return; } elsif ($err) { fail("$testname"); note("got unexpected error: $err"); return; } # check if QEMU version set correctly and test version_cmp (my $qemu_major = get_test_qemu_version()) =~ s/\..*$//; die "runs_at_least_qemu_version returned false, maybe error in version_cmp?" if !PVE::QemuServer::QMPHelpers::runs_at_least_qemu_version($vmid, $qemu_major); $cmdline =~ s/ -/ \\\n -/g; # same as qm showcmd --pretty $cmdline .= "\n"; my $cmd_fn = "$config_fn.cmd"; if (-f $cmd_fn) { my $cmdline_expected = file_get_contents($cmd_fn); my $cmd_expected = [split /\s*\\?\n\s*/, $cmdline_expected]; my $cmd = [split /\s*\\?\n\s*/, $cmdline]; # uncomment for easier debugging #file_set_contents("$cmd_fn.tmp", $cmdline); my $exp = join("\n", @$cmd_expected); my $got = join("\n", @$cmd); eval { diff($exp, $got) }; if (my $err = $@) { fail("$testname"); note($err); } else { pass("$testname"); } } else { file_set_contents($cmd_fn, $cmdline); } } print "testing config to command stability\n"; # exec tests my $test_target = shift // 'cfg2cmd'; if (-f $test_target) { do_test($test_target); } elsif (-d $test_target) { PVE::File::dir_glob_foreach( $test_target, qr/.+\.conf/, sub { my ($file) = @_; do_test("${test_target}/${file}"); }, ); } else { die "test target '$test_target' is neither file nor directory, cannot proceed\n"; } done_testing(); ================================================ FILE: src/test/run_parse_config_tests.pl ================================================ #!/usr/bin/perl # Tests parsing and writing VM configuration files. # The parsing part is already covered by the config2command test too, but that only focuses on the # main section, not other section types and does not also test parsing in strict mode. # # If no expected file exists, the input is assumed to be equal to the expected output. # If $file.strict.error (respectively $file.non-strict.error) exists, it is assumed to be the # expected error when parsing the config in strict (respectively non-strict) mode. use strict; use warnings; use lib qw(..); use File::Path qw(make_path remove_tree); use Test::MockModule; use Test::More; use PVE::QemuServer; use PVE::Tools; my $INPUT_DIR = './parse-config-input'; my $OUTPUT_DIR = './parse-config-output'; my $EXPECTED_DIR = './parse-config-expected'; # NOTE update when you add/remove tests plan tests => 2 * 10; sub run_tests { my ($strict) = @_; PVE::Tools::dir_glob_foreach( './parse-config-input', '.*\.conf', sub { my ($file) = @_; my $strict_mode = $strict ? 'strict' : 'non-strict'; my $expected_err_file = "${EXPECTED_DIR}/${file}.${strict_mode}.error"; my $expected_err; $expected_err = PVE::Tools::file_get_contents($expected_err_file) if -f $expected_err_file; my $fake_config_fn = "$file/qemu-server/8006.conf"; my $input_file = "${INPUT_DIR}/${file}"; my $input = PVE::Tools::file_get_contents($input_file); my $conf = eval { PVE::QemuServer::parse_vm_config($fake_config_fn, $input, $strict); }; if (my $err = $@) { if ($expected_err) { is($err, $expected_err, $file); } else { note("got unexpected error '$err'"); fail($file); } return; } if ($expected_err) { note("expected error for strict mode did not occur: '$expected_err'"); fail($file); return; } my $output = eval { PVE::QemuServer::write_vm_config($fake_config_fn, $conf); }; if (my $err = $@) { note("got unexpected error '$err'"); fail($file); return; } my $output_file = "${OUTPUT_DIR}/${file}"; PVE::Tools::file_set_contents($output_file, $output); my $expected_file = "${EXPECTED_DIR}/${file}"; $expected_file = $input_file if !-f $expected_file; my $cmd = ['diff', '-u', $expected_file, $output_file]; if (system(@$cmd) == 0) { pass($file); } else { fail($file); } }, ); } make_path(${OUTPUT_DIR}); run_tests(0); run_tests(1); remove_tree(${OUTPUT_DIR}) or die "failed to remove output directory\n"; done_testing(); ================================================ FILE: src/test/run_pci_addr_checks.pl ================================================ #!/usr/bin/perl use strict; use warnings; use experimental 'smartmatch'; use lib qw(..); use Test::More; use PVE::Tools qw(file_get_contents); use PVE::QemuServer::PCI; my $qemu_cfg_base_path = "../usr"; # not our format but that what QEMU gets passed with '-readconfig' sub slurp_qemu_config { my ($fn) = @_; my $raw = file_get_contents($fn); my $lineno = 0; my $cfg = {}; my $group; my $skip_to_next_group; while ($raw =~ /^\h*(.*?)\h*$/gm) { my $line = $1; $lineno++; next if !$line || $line =~ /^#/; # tried to follow qemu's qemu_config_parse function if ($line =~ /\[(\S{1,63}) "([^"\]]{1,63})"\]/) { $group = $2; $skip_to_next_group = 0; if ($1 ne 'device') { $group = undef; $skip_to_next_group = 1; } } elsif ($line =~ /\[([^\]]{1,63})\]/) { $group = undef; $skip_to_next_group = 1; } elsif ($group) { if ($line =~ /(\S{1,63}) = "([^\"]{1,1023})"/) { my ($k, $v) = ($1, $2); $cfg->{$group}->{$k} = $v; } else { print "ignoring $fn:$lineno: $line\n"; } } else { warn "ignore $fn:$lineno, currently no group\n" if !$skip_to_next_group; } } return $cfg; } sub extract_qemu_config_addrs { my ($qemu_cfg) = @_; my $addr_map = {}; for my $k (keys %$qemu_cfg) { my $v = $qemu_cfg->{$k}; next if !$v || !defined($v->{bus}) || !defined($v->{addr}); my $bus = $v->{bus}; $bus =~ s/pci\.//; $addr_map->{$k} = { bus => $bus, addr => $v->{addr} }; } return $addr_map; } print "testing PCI(e) address conflicts\n"; # exec tests #FIXME: make cross PCI <-> PCIe check sense at all?? my $addr_map = {}; my ($fail, $ignored) = (0, 0); sub check_conflict { my ($id, $what, $ignore_if_same_key) = @_; my ($bus, $addr) = $what->@{qw(bus addr)}; my $full_addr = "$bus:$addr"; if (defined(my $conflict = $addr_map->{$full_addr})) { if (my @ignores = $what->{conflict_ok}) { if ($conflict ~~ @ignores) { note("OK: ignore conflict for '$full_addr' between '$id' and '$conflict'"); $ignored++; return; } } # this allows to read multiple pve-*.cfg qemu configs, and check them # normally their OK if they conflict is on the same key. Else TODO?? return if $ignore_if_same_key && $id eq $conflict; note("ERR: conflict for '$full_addr' between '$id' and '$conflict'"); $fail++; } else { $addr_map->{$full_addr} = $id; } } my $pci_map = PVE::QemuServer::PCI::get_pci_addr_map(); while (my ($id, $what) = each %$pci_map) { check_conflict($id, $what); } my $pcie_map = PVE::QemuServer::PCI::get_pcie_addr_map(); while (my ($id, $what) = each %$pcie_map) { check_conflict($id, $what); } my $pve_qm_cfg = slurp_qemu_config("$qemu_cfg_base_path/pve-q35.cfg"); my $pve_qm_cfg_map = extract_qemu_config_addrs($pve_qm_cfg); while (my ($id, $what) = each %$pve_qm_cfg_map) { check_conflict($id, $what); } # FIXME: restart with clean conflict $addr_map with only get_pci*_addr_map ones? my $pve_qm4_cfg = slurp_qemu_config("$qemu_cfg_base_path/pve-q35-4.0.cfg"); my $pve_qm4_cfg_map = extract_qemu_config_addrs($pve_qm4_cfg); while (my ($id, $what) = each %$pve_qm4_cfg_map) { check_conflict($id, $what, 1); } my $pve_qm_usb_cfg = slurp_qemu_config("$qemu_cfg_base_path/pve-usb.cfg"); my $pve_qm_usb_cfg_map = extract_qemu_config_addrs($pve_qm_usb_cfg); while (my ($id, $what) = each %$pve_qm_usb_cfg_map) { check_conflict($id, $what, 1); } if ($fail) { fail("PCI(e) address conflict check, ignored: $ignored, conflicts: $fail"); } else { pass("PCI(e) address conflict check, ignored: $ignored"); } done_testing(); ================================================ FILE: src/test/run_pci_reservation_tests.pl ================================================ #!/usr/bin/perl use strict; use warnings; use lib qw(..); my $vmid = 8006; use Test::MockModule; use Test::More; use PVE::Mapping::PCI; use PVE::QemuServer::PCI; my $pci_devs = [ "0000:00:43.1", "0000:00:f4.0", "0000:00:ff.1", "0000:0f:f2.0", "0000:d0:13.0", "0000:d0:15.1", "0000:d0:15.2", "0000:d0:17.0", "0000:f0:42.0", "0000:f0:43.0", "0000:f0:43.1", "1234:f0:43.1", "0000:01:00.4", "0000:01:00.5", "0000:01:00.6", "0000:07:10.0", "0000:07:10.1", "0000:07:10.4", ]; my $pci_map_config = { ids => { someGpu => { type => 'pci', mdev => 1, map => [ 'node=localhost,path=0000:01:00.4,id=10de:2231,iommugroup=1', 'node=localhost,path=0000:01:00.5,id=10de:2231,iommugroup=1', 'node=localhost,path=0000:01:00.6,id=10de:2231,iommugroup=1', ], }, someNic => { type => 'pci', map => [ 'node=localhost,path=0000:07:10.0,id=8086:1520,iommugroup=2', 'node=localhost,path=0000:07:10.1,id=8086:1520,iommugroup=2', 'node=localhost,path=0000:07:10.4,id=8086:1520,iommugroup=2', ], }, }, }; my $tests = [ { name => 'reservation-is-respected', conf => { hostpci0 => 'mapping=someNic', hostpci1 => 'mapping=someGpu,mdev=some-model', hostpci2 => 'mapping=someNic', }, expected => { hostpci0 => { ids => [{ id => '0000:07:10.0' }] }, hostpci1 => { ids => [ { id => '0000:01:00.4' }, { id => '0000:01:00.5' }, { id => '0000:01:00.6' }, ], mdev => 'some-model', }, hostpci2 => { ids => [{ id => '0000:07:10.4' }] }, }, }, ]; plan tests => scalar($tests->@*); my $pve_common_inotify; $pve_common_inotify = Test::MockModule->new('PVE::INotify'); $pve_common_inotify->mock( nodename => sub { return 'localhost'; }, ); my $pve_common_sysfstools; $pve_common_sysfstools = Test::MockModule->new('PVE::SysFSTools'); $pve_common_sysfstools->mock( lspci => sub { my ($filter, $verbose) = @_; return [ map { { id => $_ } } grep { !defined($filter) || (!ref($filter) && $_ =~ m/^(0000:)?\Q$filter\E/) || (ref($filter) eq 'CODE' && $filter->({ id => $_ })) } sort @$pci_devs ]; }, pci_device_info => sub { my ($path, $noerr) = @_; if ($path =~ m/^0000:01:00/) { return { mdev => 1, iommugroup => 1, mdev => 1, vendor => "0x10de", device => "0x2231", }; } elsif ($path =~ m/^0000:07:10/) { return { iommugroup => 2, vendor => "0x8086", device => "0x1520", }; } else { return {}; } }, ); my $mapping_pci_module = Test::MockModule->new("PVE::Mapping::PCI"); $mapping_pci_module->mock( config => sub { return $pci_map_config; }, ); my $pci_module = Test::MockModule->new("PVE::QemuServer::PCI"); $pci_module->mock( reserve_pci_usage => sub { my ($ids, $vmid, $timeout, $pid, $dryrun) = @_; $ids = [$ids] if !ref($ids); for my $id (@$ids) { if ($id eq "0000:07:10.1") { die "reserved"; } } return undef; }, create_nvidia_device => sub { return 1; }, ); for my $test ($tests->@*) { my ($name, $conf, $expected) = $test->@{qw(name conf expected)}; my $pci_devices; eval { my $devices = PVE::QemuServer::PCI::parse_hostpci_devices($conf); use JSON; $pci_devices = PVE::QemuServer::PCI::choose_hostpci_devices($devices, $vmid); }; if (my $err = $@) { is($err, $expected, $name); } elsif ($pci_devices) { is_deeply($pci_devices, $expected, $name); } else { fail($name); note("no result"); } } done_testing(); ================================================ FILE: src/test/run_qemu_img_convert_tests.pl ================================================ #!/usr/bin/perl use strict; use warnings; use lib qw(..); use Test::More; use Test::MockModule; use PVE::QemuServer::QemuImage; my $vmid = 8006; my $storage_config = { ids => { local => { content => { images => 1, }, path => "/var/lib/vz", type => "dir", shared => 0, }, localsnapext => { content => { images => 1, }, path => "/var/lib/vzsnapext", type => "dir", 'snapshot-as-volume-chain' => 1, shared => 0, }, btrfs => { content => { images => 1, }, path => "/var/lib/btrfs", type => "btrfs", shared => 0, }, "krbd-store" => { monhost => "127.0.0.42,127.0.0.21,::1", fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a', content => { images => 1, }, type => "rbd", krbd => 1, pool => "apool", username => "admin", shared => 1, }, "rbd-store" => { monhost => "127.0.0.42,127.0.0.21,::1", fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a', content => { images => 1, }, type => "rbd", pool => "cpool", username => "admin", shared => 1, }, "local-lvm" => { vgname => "pve", bwlimit => "restore=1024", type => "lvmthin", thinpool => "data", content => { images => 1, }, }, "lvm-store" => { vgname => "pve", type => "lvm", content => { images => 1, }, 'snapshot-as-volume-chain' => 1, }, "zfs-over-iscsi" => { type => "zfs", iscsiprovider => "LIO", lio_tpg => "tpg1", portal => "127.0.0.1", target => "iqn.2019-10.org.test:foobar", pool => "tank", }, }, }; my $tests = [ { name => 'qcow2raw', parameters => ["local:$vmid/vm-$vmid-disk-0.qcow2", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "qcow2", "-O", "raw", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.qcow2", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "raw2qcow2", parameters => ["local:$vmid/vm-$vmid-disk-0.raw", "local:$vmid/vm-$vmid-disk-0.qcow2", 1024 * 10], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "qcow2", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.qcow2", ], }, { name => "local2rbd", parameters => ["local:$vmid/vm-$vmid-disk-0.raw", "rbd-store:vm-$vmid-disk-0", 1024 * 10], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", "rbd:cpool/vm-$vmid-disk-0:mon_host=127.0.0.42;127.0.0.21;[\\:\\:1]:auth_supported=none", ], }, { name => "rbd2local", parameters => ["rbd-store:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw", "rbd:cpool/vm-$vmid-disk-0:mon_host=127.0.0.42;127.0.0.21;[\\:\\:1]:auth_supported=none", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "local2zos", parameters => ["local:$vmid/vm-$vmid-disk-0.raw", "zfs-over-iscsi:vm-$vmid-disk-0", 1024 * 10], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "--target-image-opts", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", "file.driver=iscsi,file.transport=tcp,file.initiator-name=foobar,file.portal=127.0.0.1,file.target=iqn.2019-10.org.test:foobar,file.lun=1,driver=raw", ], }, { name => "zos2local", parameters => ["zfs-over-iscsi:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "--image-opts", "-O", "raw", "file.driver=iscsi,file.transport=tcp,file.initiator-name=foobar,file.portal=127.0.0.1,file.target=iqn.2019-10.org.test:foobar,file.lun=1,driver=raw", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "zos2rbd", parameters => ["zfs-over-iscsi:vm-$vmid-disk-0", "rbd-store:vm-$vmid-disk-0", 1024 * 10], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "--image-opts", "-O", "raw", "file.driver=iscsi,file.transport=tcp,file.initiator-name=foobar,file.portal=127.0.0.1,file.target=iqn.2019-10.org.test:foobar,file.lun=1,driver=raw", "rbd:cpool/vm-$vmid-disk-0:mon_host=127.0.0.42;127.0.0.21;[\\:\\:1]:auth_supported=none", ], }, { name => "rbd2zos", parameters => ["rbd-store:vm-$vmid-disk-0", "zfs-over-iscsi:vm-$vmid-disk-0", 1024 * 10], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "--target-image-opts", "rbd:cpool/vm-$vmid-disk-0:mon_host=127.0.0.42;127.0.0.21;[\\:\\:1]:auth_supported=none", "file.driver=iscsi,file.transport=tcp,file.initiator-name=foobar,file.portal=127.0.0.1,file.target=iqn.2019-10.org.test:foobar,file.lun=1,driver=raw", ], }, { name => "local2lvmthin", parameters => ["local:$vmid/vm-$vmid-disk-0.raw", "local-lvm:vm-$vmid-disk-0", 1024 * 10], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", "/dev/pve/vm-$vmid-disk-0", ], }, { name => "lvmthin2local", parameters => ["local-lvm:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw", "/dev/pve/vm-$vmid-disk-0", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "zeroinit", parameters => [ "local-lvm:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10, { 'is-zero-initialized' => 1 }, ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw", "/dev/pve/vm-$vmid-disk-0", "zeroinit:/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "notexistingstorage", parameters => ["local-lvm:vm-$vmid-disk-0", "not-existing:$vmid/vm-$vmid-disk-0.raw", 1024 * 10], expected => "storage 'not-existing' does not exist\n", }, { name => "vmdkfile", parameters => ["./test.vmdk", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "vmdk", "-O", "raw", "./test.vmdk", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "notexistingfile", parameters => ["/foo/bar", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10], expected => "source '/foo/bar' is not a valid volid nor path for qemu-img convert\n", }, { name => "efidisk", parameters => ["/usr/share/kvm/OVMF_VARS-pure-efi.fd", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-O", "raw", "/usr/share/kvm/OVMF_VARS-pure-efi.fd", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "efi2zos", parameters => ["/usr/share/kvm/OVMF_VARS-pure-efi.fd", "zfs-over-iscsi:vm-$vmid-disk-0", 1024 * 10], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "--target-image-opts", "/usr/share/kvm/OVMF_VARS-pure-efi.fd", "file.driver=iscsi,file.transport=tcp,file.initiator-name=foobar,file.portal=127.0.0.1,file.target=iqn.2019-10.org.test:foobar,file.lun=1,driver=raw", ], }, { name => "bwlimit", parameters => [ "local-lvm:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10, { bwlimit => 1024 }, ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-r", "1024K", "-f", "raw", "-O", "raw", "/dev/pve/vm-$vmid-disk-0", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "krbdsnapshot", parameters => [ "krbd-store:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10, { snapname => 'foo' }, ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw", "/dev/rbd-pve/fc4181a6-56eb-4f68-b452-8ba1f381ca2a/apool/vm-$vmid-disk-0\@foo", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "rbdsnapshot", parameters => [ "rbd-store:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10, { snapname => 'foo' }, ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw", "rbd:cpool/vm-$vmid-disk-0\@foo:mon_host=127.0.0.42;127.0.0.21;[\\:\\:1]:auth_supported=none", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "btrfs_raw_snapshots", parameters => [ "btrfs:$vmid/vm-$vmid-disk-0.raw", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10, { snapname => 'foo' }, ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw", "/var/lib/btrfs/images/$vmid/vm-$vmid-disk-0\@foo/disk.raw", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "btrfs_qcow2_snapshots", parameters => [ "btrfs:$vmid/vm-$vmid-disk-0.qcow2", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10, { snapname => 'snap' }, ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-l", "snapshot.name=snap", "-f", "qcow2", "-O", "raw", "/var/lib/btrfs/images/$vmid/vm-$vmid-disk-0.qcow2", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "lvmsnapshot", parameters => [ "local-lvm:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10, { snapname => 'foo' }, ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw", "/dev/pve/snap_vm-$vmid-disk-0_foo", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "qcow2snapshot", parameters => [ "local:$vmid/vm-$vmid-disk-0.qcow2", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10, { snapname => 'snap' }, ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-l", "snapshot.name=snap", "-f", "qcow2", "-O", "raw", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.qcow2", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "qcow2_external_snapshot", parameters => [ "localsnapext:$vmid/vm-$vmid-disk-0.qcow2", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10, { snapname => 'foo' }, ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "qcow2", "-O", "raw", "/var/lib/vzsnapext/images/$vmid/snap-foo-vm-$vmid-disk-0.qcow2", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "lvmqcow2_external_snapshot", parameters => [ "lvm-store:vm-$vmid-disk-0.qcow2", "local:$vmid/vm-$vmid-disk-0.raw", 1024 * 10, { snapname => 'foo' }, ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "qcow2", "-O", "raw", "/dev/pve/snap_vm-$vmid-disk-0_foo.qcow2", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", ], }, { name => "qcow2_external_snapshot_target", parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "localsnapext:$vmid/vm-$vmid-disk-target.qcow2", 1024 * 10, ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "--target-image-opts", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", "discard-no-unref=true,driver=qcow2,file.driver=file" . ",file.filename=/var/lib/vzsnapext/images/$vmid/vm-$vmid-disk-target.qcow2", ], }, { name => "qcow2_external_snapshot_target_zeroinit", parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "localsnapext:$vmid/vm-$vmid-disk-target.qcow2", 1024 * 10, { 'is-zero-initialized' => 1 }, ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "--target-image-opts", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", "driver=zeroinit,file.discard-no-unref=true,file.driver=qcow2,file.file.driver=file" . ",file.file.filename=/var/lib/vzsnapext/images/$vmid/vm-$vmid-disk-target.qcow2", ], }, { name => "lvmqcow2_external_snapshot_target", parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "lvm-store:vm-$vmid-disk-target.qcow2", 1024 * 10, ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "--target-image-opts", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", "discard-no-unref=true,driver=qcow2,file.driver=host_device" . ",file.filename=/dev/pve/vm-$vmid-disk-target.qcow2", ], }, ]; my $command; my $file_stat_module = Test::MockModule->new("File::stat"); $file_stat_module->mock( stat => sub { my ($path) = @_; my $st = $file_stat_module->original('stat')->('./run_qemu_img_convert_tests.pl'); $st->[2] = 25008 if $path =~ m!/dev/!; # block device return $st; }, ); my $storage_module = Test::MockModule->new("PVE::Storage"); $storage_module->mock( config => sub { return $storage_config; }, activate_volumes => sub { return 1; }, volume_snapshot_info => sub { my ($cfg, $volid) = @_; if ( $volid eq "lvm-store:vm-$vmid-disk-target.qcow2" || $volid eq "localsnapext:$vmid/vm-$vmid-disk-target.qcow2" ) { # target volumes don't have snapshots return {}; } die "mocked volume_snapshot_info called with unexpected volid $volid\n"; }, ); my $lio_module = Test::MockModule->new("PVE::Storage::LunCmd::LIO"); $lio_module->mock( run_lun_command => sub { return 1; }, ); # we use the exported run_command so we have to mock it there my $zfsplugin_module = Test::MockModule->new("PVE::Storage::ZFSPlugin"); $zfsplugin_module->mock( run_command => sub { return 1; }, ); my $qemu_server_helpers_module = Test::MockModule->new("PVE::QemuServer::Helpers"); $qemu_server_helpers_module->mock( get_iscsi_initiator_name => sub { return "foobar"; }, ); my $tools_module = Test::MockModule->new("PVE::Tools"); $tools_module->mock( run_command => sub { $command = shift; }, ); foreach my $test (@$tests) { my $name = $test->{name}; my $expected = $test->{expected}; eval { PVE::QemuServer::QemuImage::convert(@{ $test->{parameters} }) }; if (my $err = $@) { is($err, $expected, $name); } elsif (defined($command)) { is_deeply($command, $expected, $name); $command = undef; } else { fail($name); note("no command"); } } done_testing(); ================================================ FILE: src/test/run_qemu_migrate_tests.pl ================================================ #!/usr/bin/perl use strict; use warnings; use JSON; use Test::More; use Test::MockModule; use PVE::JSONSchema; use PVE::Tools qw(file_set_contents file_get_contents run_command); my $QM_LIB_PATH = '..'; my $MIGRATE_LIB_PATH = '..'; my $RUN_DIR_PATH = './MigrationTest/run/'; # test configuration shared by all tests my $replication_config = { 'ids' => { '105-0' => { 'guest' => '105', 'id' => '105-0', 'jobnum' => '0', 'source' => 'pve0', 'target' => 'pve2', 'type' => 'local', }, }, 'order' => { '105-0' => 1, }, }; my $storage_config = { ids => { local => { content => { images => 1, }, path => "/var/lib/vz", type => "dir", shared => 0, }, "local-lvm" => { content => { images => 1, }, nodes => { pve0 => 1, pve1 => 1, }, type => "lvmthin", thinpool => "data", vgname => "pve", }, "local-zfs" => { content => { images => 1, rootdir => 1, }, pool => "rpool/data", sparse => 1, type => "zfspool", }, "rbd-store" => { monhost => "127.0.0.42,127.0.0.21,::1", fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a', content => { images => 1, }, type => "rbd", pool => "cpool", username => "admin", shared => 1, }, "local-dir" => { content => { images => 1, }, path => "/some/dir/", type => "dir", }, "other-dir" => { content => { images => 1, }, path => "/some/other/dir/", type => "dir", }, "zfs-alias-1" => { content => { images => 1, rootdir => 1, }, pool => "aliaspool", sparse => 1, type => "zfspool", }, "zfs-alias-2" => { content => { images => 1, rootdir => 1, }, pool => "aliaspool", sparse => 1, type => "zfspool", }, }, }; my $vm_configs = { 105 => { 'bootdisk' => 'scsi0', 'cores' => 1, 'ide0' => 'local-zfs:vm-105-disk-1,size=103M', 'ide2' => 'none,media=cdrom', 'memory' => 512, 'name' => 'Copy-of-VM-newapache', 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1', 'numa' => 0, 'ostype' => 'l26', 'parent' => 'ohsnap', 'pending' => {}, 'scsi0' => 'local-zfs:vm-105-disk-0,size=4G', 'scsihw' => 'virtio-scsi-pci', 'smbios1' => 'uuid=1ddfe18b-77e0-47f6-a4bd-f1761bf6d763', 'snapshots' => { 'ohsnap' => { 'bootdisk' => 'scsi0', 'cores' => 1, 'ide2' => 'none,media=cdrom', 'memory' => 512, 'name' => 'Copy-of-VM-newapache', 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1', 'numa' => 0, 'ostype' => 'l26', 'scsi0' => 'local-zfs:vm-105-disk-0,size=4G', 'scsihw' => 'virtio-scsi-pci', 'smbios1' => 'uuid=1ddfe18b-77e0-47f6-a4bd-f1761bf6d763', 'snaptime' => 1580976924, 'sockets' => 1, 'startup' => 'order=2', 'vmgenid' => '4eb1d535-9381-4ddc-a8aa-af50c4d9177b', }, }, 'sockets' => 1, 'startup' => 'order=2', 'vmgenid' => '4eb1d535-9381-4ddc-a8aa-af50c4d9177b', }, 111 => { 'bootdisk' => 'scsi0', 'cores' => 1, 'ide0' => 'local-lvm:vm-111-disk-0,size=4096M', 'ide2' => 'none,media=cdrom', 'memory' => 512, 'name' => 'pending-test', 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1', 'numa' => 0, 'ostype' => 'l26', 'pending' => { 'scsi0' => 'local-zfs:vm-111-disk-0,size=103M', }, 'scsihw' => 'virtio-scsi-pci', 'snapshots' => {}, 'smbios1' => 'uuid=5ad71d4d-8f73-4377-853e-2d22c10c96a5', 'sockets' => 1, 'vmgenid' => '2c00c030-0b5b-4988-a371-6ab259893f22', }, 123 => { 'bootdisk' => 'scsi0', 'cores' => 1, 'scsi0' => 'zfs-alias-1:vm-123-disk-0,size=4096M', 'scsi1' => 'zfs-alias-2:vm-123-disk-0,size=4096M', 'ide2' => 'none,media=cdrom', 'memory' => 512, 'name' => 'alias-test', 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1', 'numa' => 0, 'ostype' => 'l26', 'pending' => {}, 'scsihw' => 'virtio-scsi-pci', 'snapshots' => {}, 'smbios1' => 'uuid=5ad71d4d-8f73-4377-853e-2d22c10c96a5', 'sockets' => 1, 'vmgenid' => '2c00c030-0b5b-4988-a371-6ab259893f22', }, 149 => { 'agent' => '0', 'bootdisk' => 'scsi0', 'cores' => 1, 'hotplug' => 'disk,network,usb,memory,cpu', 'ide2' => 'none,media=cdrom', 'memory' => 4096, 'name' => 'asdf', 'net0' => 'virtio=52:5D:7E:62:85:97,bridge=vmbr1', 'numa' => 1, 'ostype' => 'l26', 'scsi0' => 'local-lvm:vm-149-disk-0,format=raw,size=4G', 'scsi1' => 'local-dir:149/vm-149-disk-0.qcow2,format=qcow2,size=1G', 'scsihw' => 'virtio-scsi-pci', 'snapshots' => {}, 'smbios1' => 'uuid=e980bd43-a405-42e2-b5f4-31efe6517460', 'sockets' => 1, 'startup' => 'order=2', 'vmgenid' => '36c6c50c-6ef5-4adc-9b6f-6ba9c8071db0', }, 341 => { 'arch' => 'aarch64', 'bootdisk' => 'scsi0', 'cores' => 1, 'efidisk0' => 'local-lvm:vm-341-disk-0', 'ide2' => 'none,media=cdrom', 'ipconfig0' => 'ip=103.214.69.10/25,gw=103.214.69.1', 'memory' => 4096, 'name' => 'VM1033', 'net0' => 'virtio=4E:F1:82:6D:D7:4B,bridge=vmbr0,firewall=1,rate=10', 'numa' => 0, 'ostype' => 'l26', 'scsi0' => 'rbd-store:vm-341-disk-0,size=1G', 'scsihw' => 'virtio-scsi-pci', 'snapshots' => {}, 'smbios1' => 'uuid=e01e4c73-46f1-47c8-af79-288fdf6b7462', 'sockets' => 2, 'vmgenid' => 'af47c000-eb0c-48e8-8991-ca4593cd6916', }, 1033 => { 'bootdisk' => 'scsi0', 'cores' => 1, 'ide0' => 'rbd-store:vm-1033-cloudinit,media=cdrom,size=4M', 'ide2' => 'none,media=cdrom', 'ipconfig0' => 'ip=103.214.69.10/25,gw=103.214.69.1', 'memory' => 4096, 'name' => 'VM1033', 'net0' => 'virtio=4E:F1:82:6D:D7:4B,bridge=vmbr0,firewall=1,rate=10', 'numa' => 0, 'ostype' => 'l26', 'scsi0' => 'rbd-store:vm-1033-disk-1,size=1G', 'scsihw' => 'virtio-scsi-pci', 'snapshots' => {}, 'smbios1' => 'uuid=e01e4c73-46f1-47c8-af79-288fdf6b7462', 'sockets' => 2, 'vmgenid' => 'af47c000-eb0c-48e8-8991-ca4593cd6916', }, 4567 => { 'bootdisk' => 'scsi0', 'cores' => 1, 'ide2' => 'none,media=cdrom', 'memory' => 512, 'name' => 'snapme', 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1', 'numa' => 0, 'ostype' => 'l26', 'parent' => 'snap1', 'pending' => {}, 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G', 'scsihw' => 'virtio-scsi-pci', 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74', 'snapshots' => { 'snap1' => { 'bootdisk' => 'scsi0', 'cores' => 1, 'ide2' => 'none,media=cdrom', 'memory' => 512, 'name' => 'snapme', 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1', 'numa' => 0, 'ostype' => 'l26', 'runningcpu' => 'kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep', 'runningmachine' => 'pc-i440fx-10.0+pve0', 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G', 'scsihw' => 'virtio-scsi-pci', 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74', 'snaptime' => 1595928799, 'sockets' => 1, 'startup' => 'order=2', 'vmgenid' => '932b227a-8a39-4ede-955a-dbd4bc4385ed', 'vmstate' => 'local-dir:4567/vm-4567-state-snap1.raw', }, 'snap2' => { 'bootdisk' => 'scsi0', 'cores' => 1, 'ide2' => 'none,media=cdrom', 'memory' => 512, 'name' => 'snapme', 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1', 'numa' => 0, 'ostype' => 'l26', 'parent' => 'snap1', 'runningcpu' => 'kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep', 'runningmachine' => 'pc-i440fx-10.0+pve0', 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G', 'scsi1' => 'local-zfs:vm-4567-disk-0,size=1G', 'scsihw' => 'virtio-scsi-pci', 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74', 'snaptime' => 1595928871, 'sockets' => 1, 'startup' => 'order=2', 'vmgenid' => '932b227a-8a39-4ede-955a-dbd4bc4385ed', 'vmstate' => 'local-dir:4567/vm-4567-state-snap2.raw', }, }, 'sockets' => 1, 'startup' => 'order=2', 'unused0' => 'local-zfs:vm-4567-disk-0', 'vmgenid' => 'e698e60c-9278-4dd9-941f-416075383f2a', }, }; my $source_vdisks = { 'local-dir' => [ { 'ctime' => 1589439681, 'format' => 'qcow2', 'parent' => undef, 'size' => 1073741824, 'used' => 335872, 'vmid' => '149', 'volid' => 'local-dir:149/vm-149-disk-0.qcow2', }, { 'ctime' => 1595928898, 'format' => 'qcow2', 'parent' => undef, 'size' => 4294967296, 'used' => 1811664896, 'vmid' => '4567', 'volid' => 'local-dir:4567/vm-4567-disk-0.qcow2', }, { 'ctime' => 1595928800, 'format' => 'raw', 'parent' => undef, 'size' => 274666496, 'used' => 274669568, 'vmid' => '4567', 'volid' => 'local-dir:4567/vm-4567-state-snap1.raw', }, { 'ctime' => 1595928872, 'format' => 'raw', 'parent' => undef, 'size' => 273258496, 'used' => 273260544, 'vmid' => '4567', 'volid' => 'local-dir:4567/vm-4567-state-snap2.raw', }, ], 'local-lvm' => [ { 'ctime' => '1589277334', 'format' => 'raw', 'size' => 4294967296, 'vmid' => '149', 'volid' => 'local-lvm:vm-149-disk-0', }, { 'ctime' => '1589277334', 'format' => 'raw', 'size' => 4194304, 'vmid' => '341', 'volid' => 'local-lvm:vm-341-disk-0', }, { 'ctime' => '1589277334', 'format' => 'raw', 'size' => 4294967296, 'vmid' => '111', 'volid' => 'local-lvm:vm-111-disk-0', }, ], 'local-zfs' => [ { 'ctime' => '1589277334', 'format' => 'raw', 'size' => 4294967296, 'vmid' => '105', 'volid' => 'local-zfs:vm-105-disk-0', }, { 'ctime' => '1589277334', 'format' => 'raw', 'size' => 108003328, 'vmid' => '105', 'volid' => 'local-zfs:vm-105-disk-1', }, { 'ctime' => '1589277334', 'format' => 'raw', 'size' => 108003328, 'vmid' => '111', 'volid' => 'local-zfs:vm-111-disk-0', }, { 'format' => 'raw', 'name' => 'vm-4567-disk-0', 'parent' => undef, 'size' => 1073741824, 'vmid' => '4567', 'volid' => 'local-zfs:vm-4567-disk-0', }, ], 'rbd-store' => [ { 'ctime' => '1589277334', 'format' => 'raw', 'size' => 1073741824, 'vmid' => '1033', 'volid' => 'rbd-store:vm-1033-disk-1', }, { 'ctime' => '1589277334', 'format' => 'raw', 'size' => 1073741824, 'vmid' => '1033', 'volid' => 'rbd-store:vm-1033-cloudinit', }, ], 'zfs-alias-1' => [ { 'ctime' => '1589277334', 'format' => 'raw', 'size' => 4294967296, 'vmid' => '123', 'volid' => 'zfs-alias-1:vm-123-disk-0', }, ], 'zfs-alias-2' => [ { 'ctime' => '1589277334', 'format' => 'raw', 'size' => 4294967296, 'vmid' => '123', 'volid' => 'zfs-alias-2:vm-123-disk-0', }, ], }; my $default_expected_calls_online = { move_config_to_node => 1, ssh_qm_start => 1, vm_stop => 1, }; my $default_expected_calls_offline = { move_config_to_node => 1, }; my $replicated_expected_calls_online = { %{$default_expected_calls_online}, transfer_replication_state => 1, switch_replication_job_target => 1, }; my $replicated_expected_calls_offline = { %{$default_expected_calls_offline}, transfer_replication_state => 1, switch_replication_job_target => 1, }; # helpers sub get_patched_config { my ($vmid, $patch) = @_; my $new_config = { %{ $vm_configs->{$vmid} } }; patch_config($new_config, $patch) if defined($patch); return $new_config; } sub patch_config { my ($config, $patch) = @_; foreach my $key (keys %{$patch}) { if ($key eq 'snapshots' && defined($patch->{$key})) { my $new_snapshot_configs = {}; foreach my $snap (keys %{ $patch->{snapshots} }) { my $new_snapshot_config = { %{ $config->{snapshots}->{$snap} } }; patch_config($new_snapshot_config, $patch->{snapshots}->{$snap}); $new_snapshot_configs->{$snap} = $new_snapshot_config; } $config->{snapshots} = $new_snapshot_configs; } elsif (defined($patch->{$key})) { $config->{$key} = $patch->{$key}; } else { # use undef value for deletion delete $config->{$key}; } } } sub local_volids_for_vm { my ($vmid) = @_; my $res = {}; foreach my $storeid (keys %{$source_vdisks}) { next if $storage_config->{ids}->{$storeid}->{shared}; $res = { %{$res}, map { $_->{vmid} eq $vmid ? ($_->{volid} => 1) : () } @{ $source_vdisks->{$storeid} }, }; } return $res; } my $tests = [ # each test consists of the following: # name - unique name for the test which also serves as a dir name. # NOTE: gets passed to make, so don't use whitespace or slash # and adapt buildsys (regex) on code structure changes # target - hostname of target node # vmid - ID of the VM to migrate # opts - options for the migrate() call # target_volids - hash of volids on the target at the beginning # vm_status - hash with running, runningmachine and optionally runningcpu # expected_calls - hash whose keys are calls which are required # to be made if the migration gets far enough # expect_die - expect the migration call to fail, and an error message # matching the specified text in the log # expected - hash consisting of: # source_volids - hash of volids expected on the source # target_volids - hash of volids expected on the target # vm_config - vm configuration hash # vm_status - hash with running, runningmachine and optionally runningcpu { # NOTE get_efivars_size is mocked and returns 128K name => '341_running_efidisk_targetstorage_dir', target => 'pve1', vmid => 341, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, targetstorage => 'local-dir', }, expected_calls => $default_expected_calls_online, expected => { source_volids => {}, target_volids => { 'local-dir:341/vm-341-disk-10.raw' => 1, }, vm_config => get_patched_config( 341, { efidisk0 => 'local-dir:341/vm-341-disk-10.raw,format=raw,size=128K', }, ), vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { # NOTE get_efivars_size is mocked and returns 128K name => '341_running_efidisk', target => 'pve1', vmid => 341, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, }, expected_calls => $default_expected_calls_online, expected => { source_volids => {}, target_volids => { 'local-lvm:vm-341-disk-10' => 1, }, vm_config => get_patched_config( 341, { efidisk0 => 'local-lvm:vm-341-disk-10,format=raw,size=128K', }, ), vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '149_running_vdisk_alloc_and_pvesm_free_fail', target => 'pve1', vmid => 149, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, }, fail_config => { vdisk_alloc => 'local-dir:149/vm-149-disk-11.qcow2', pvesm_free => 'local-lvm:vm-149-disk-10', }, expected_calls => {}, expect_die => "remote command failed with exit code", expected => { source_volids => local_volids_for_vm(149), target_volids => { 'local-lvm:vm-149-disk-10' => 1, }, vm_config => $vm_configs->{149}, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '149_running_vdisk_alloc_fail', target => 'pve1', vmid => 149, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, }, fail_config => { vdisk_alloc => 'local-lvm:vm-149-disk-10', }, expected_calls => {}, expect_die => "remote command failed with exit code", expected => { source_volids => local_volids_for_vm(149), target_volids => {}, vm_config => $vm_configs->{149}, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '149_vdisk_free_fail', target => 'pve1', vmid => 149, vm_status => { running => 0, }, opts => { 'with-local-disks' => 1, }, fail_config => { 'vdisk_free' => 'local-lvm:vm-149-disk-0', }, expected_calls => $default_expected_calls_offline, expect_die => "vdisk_free 'local-lvm:vm-149-disk-0' error", expected => { source_volids => { 'local-lvm:vm-149-disk-0' => 1, }, target_volids => local_volids_for_vm(149), vm_config => $vm_configs->{149}, vm_status => { running => 0, }, }, }, { name => '105_replicated_run_replication_fail', target => 'pve2', vmid => 105, vm_status => { running => 0, }, target_volids => local_volids_for_vm(105), fail_config => { run_replication => 1, }, expected_calls => {}, expect_die => 'run_replication error', expected => { source_volids => local_volids_for_vm(105), target_volids => local_volids_for_vm(105), vm_config => $vm_configs->{105}, vm_status => { running => 0, }, }, }, { name => '1033_running_query_migrate_fail', target => 'pve2', vmid => 1033, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, }, fail_config => { 'query-migrate' => 1, }, expected_calls => {}, expect_die => 'online migrate failure - aborting', expected => { source_volids => {}, target_volids => {}, vm_config => $vm_configs->{1033}, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '4567_targetstorage_dirotherdir', target => 'pve1', vmid => 4567, vm_status => { running => 0, }, opts => { targetstorage => 'local-dir:other-dir,local-zfs:local-zfs', }, storage_migrate_map => { 'local-dir:4567/vm-4567-disk-0.qcow2' => '4567/vm-4567-disk-0.qcow2', 'local-dir:4567/vm-4567-state-snap1.raw' => '4567/vm-4567-state-snap1.raw', 'local-dir:4567/vm-4567-state-snap2.raw' => '4567/vm-4567-state-snap2.raw', }, expected_calls => $default_expected_calls_offline, expected => { source_volids => {}, target_volids => { 'other-dir:4567/vm-4567-disk-0.qcow2' => 1, 'other-dir:4567/vm-4567-state-snap1.raw' => 1, 'other-dir:4567/vm-4567-state-snap2.raw' => 1, 'local-zfs:vm-4567-disk-0' => 1, }, vm_config => get_patched_config( 4567, { 'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G', snapshots => { snap1 => { 'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G', 'vmstate' => 'other-dir:4567/vm-4567-state-snap1.raw', }, snap2 => { 'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G', 'scsi1' => 'local-zfs:vm-4567-disk-0,size=1G', 'vmstate' => 'other-dir:4567/vm-4567-state-snap2.raw', }, }, }, ), vm_status => { running => 0, }, }, }, { name => '4567_running', target => 'pve1', vmid => 4567, vm_status => { running => 1, runningmachine => 'pc-i440fx-10.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, }, expected_calls => {}, expect_die => 'online storage migration not possible if non-replicated snapshot exists', expected => { source_volids => local_volids_for_vm(4567), target_volids => {}, vm_config => $vm_configs->{4567}, vm_status => { running => 1, runningmachine => 'pc-i440fx-10.0+pve0', }, }, }, { name => '4567_offline', target => 'pve1', vmid => 4567, vm_status => { running => 0, }, expected_calls => $default_expected_calls_offline, expected => { source_volids => {}, target_volids => local_volids_for_vm(4567), vm_config => $vm_configs->{4567}, vm_status => { running => 0, }, }, }, { name => '149_running_orphaned_disk_targetstorage_zfs', target => 'pve1', vmid => 149, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, targetstorage => 'local-zfs', }, config_patch => { scsi1 => undef, }, storage_migrate_map => { 'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0', }, expected_calls => $default_expected_calls_online, expected => { source_volids => { 'local-dir:149/vm-149-disk-0.qcow2' => 1, }, target_volids => { 'local-zfs:vm-149-disk-10' => 1, }, vm_config => get_patched_config( 149, { scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G', scsi1 => undef, }, ), vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '149_running_orphaned_disk', target => 'pve1', vmid => 149, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, }, config_patch => { scsi1 => undef, }, storage_migrate_map => { 'local-dir:149/vm-149-disk-0.qcow2' => '149/vm-149-disk-0.qcow2', }, expected_calls => $default_expected_calls_online, expected => { source_volids => { 'local-dir:149/vm-149-disk-0.qcow2' => 1, }, target_volids => { 'local-lvm:vm-149-disk-10' => 1, }, vm_config => get_patched_config( 149, { scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G', scsi1 => undef, }, ), vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { # FIXME: This test is not (yet) a realistic situation, because # storage_migrate currently never changes the format (AFAICT) # But if such migrations become possible, we need to either update # the 'format' property or simply remove it for drives migrated # with storage_migrate (the property is optional, so it shouldn't be a problem) name => '149_targetstorage_map_lvmzfs_defaultlvm', target => 'pve1', vmid => 149, vm_status => { running => 0, }, opts => { targetstorage => 'local-lvm:local-zfs,local-lvm', }, storage_migrate_map => { 'local-lvm:vm-149-disk-0' => 'vm-149-disk-0', 'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0', }, expected_calls => $default_expected_calls_offline, expected => { source_volids => {}, target_volids => { 'local-zfs:vm-149-disk-0' => 1, 'local-lvm:vm-149-disk-0' => 1, }, vm_config => get_patched_config( 149, { scsi0 => 'local-zfs:vm-149-disk-0,format=raw,size=4G', scsi1 => 'local-lvm:vm-149-disk-0,format=qcow2,size=1G', }, ), vm_status => { running => 0, }, }, }, { # FIXME same as for the previous test name => '149_targetstorage_map_dirzfs_lvmdir', target => 'pve1', vmid => 149, vm_status => { running => 0, }, opts => { online => 1, 'with-local-disks' => 1, targetstorage => 'local-dir:local-zfs,local-lvm:local-dir', }, storage_migrate_map => { 'local-lvm:vm-149-disk-0' => '149/vm-149-disk-0.raw', 'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0', }, expected_calls => $default_expected_calls_offline, expected => { source_volids => {}, target_volids => { 'local-dir:149/vm-149-disk-0.raw' => 1, 'local-zfs:vm-149-disk-0' => 1, }, vm_config => get_patched_config( 149, { scsi0 => 'local-dir:149/vm-149-disk-0.raw,format=raw,size=4G', scsi1 => 'local-zfs:vm-149-disk-0,format=qcow2,size=1G', }, ), vm_status => { running => 0, }, }, }, { name => '149_running_targetstorage_map_lvmzfs_defaultlvm', target => 'pve1', vmid => 149, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, targetstorage => 'local-lvm:local-zfs,local-lvm', }, expected_calls => $default_expected_calls_online, expected => { source_volids => {}, target_volids => { 'local-zfs:vm-149-disk-10' => 1, 'local-lvm:vm-149-disk-11' => 1, }, vm_config => get_patched_config( 149, { scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G', scsi1 => 'local-lvm:vm-149-disk-11,format=raw,size=1G', }, ), vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '149_running_targetstorage_map_lvmzfs_dirdir', target => 'pve1', vmid => 149, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, targetstorage => 'local-lvm:local-zfs,local-dir:local-dir', }, expected_calls => $default_expected_calls_online, expected => { source_volids => {}, target_volids => { 'local-zfs:vm-149-disk-10' => 1, 'local-dir:149/vm-149-disk-11.qcow2' => 1, }, vm_config => get_patched_config( 149, { scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G', scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G', }, ), vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '149_running_targetstorage_zfs', target => 'pve1', vmid => 149, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, targetstorage => 'local-zfs', }, expected_calls => $default_expected_calls_online, expected => { source_volids => {}, target_volids => { 'local-zfs:vm-149-disk-10' => 1, 'local-zfs:vm-149-disk-11' => 1, }, vm_config => get_patched_config( 149, { scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G', scsi1 => 'local-zfs:vm-149-disk-11,format=raw,size=1G', }, ), vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '149_running_wrong_size', target => 'pve1', vmid => 149, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, }, config_patch => { scsi0 => 'local-lvm:vm-149-disk-0,size=123T', }, expected_calls => $default_expected_calls_online, expected => { source_volids => {}, target_volids => { 'local-lvm:vm-149-disk-10' => 1, 'local-dir:149/vm-149-disk-11.qcow2' => 1, }, vm_config => get_patched_config( 149, { scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G', scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G', }, ), vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '149_running_missing_size', target => 'pve1', vmid => 149, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, }, config_patch => { scsi0 => 'local-lvm:vm-149-disk-0', }, expected_calls => $default_expected_calls_online, expected => { source_volids => {}, target_volids => { 'local-lvm:vm-149-disk-10' => 1, 'local-dir:149/vm-149-disk-11.qcow2' => 1, }, vm_config => get_patched_config( 149, { scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G', scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G', }, ), vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '105_local_device_shared', target => 'pve1', vmid => 105, vm_status => { running => 0, }, config_patch => { ide2 => '/dev/sde,shared=1', }, expected_calls => $default_expected_calls_offline, expected => { source_volids => {}, target_volids => local_volids_for_vm(105), vm_config => get_patched_config(105, { ide2 => '/dev/sde,shared=1', }), vm_status => { running => 0, }, }, }, { name => '105_local_device_in_snapshot', target => 'pve1', vmid => 105, vm_status => { running => 0, }, config_patch => { snapshots => { ohsnap => { ide2 => '/dev/sde', }, }, }, expected_calls => {}, expect_die => "can't migrate local disk '/dev/sde': local file/device", expected => { source_volids => local_volids_for_vm(105), target_volids => {}, vm_config => get_patched_config(105, { snapshots => { ohsnap => { ide2 => '/dev/sde', }, }, }), vm_status => { running => 0, }, }, }, { name => '105_local_device', target => 'pve1', vmid => 105, vm_status => { running => 0, }, config_patch => { ide2 => '/dev/sde', }, expected_calls => {}, expect_die => "can't migrate local disk '/dev/sde': local file/device", expected => { source_volids => local_volids_for_vm(105), target_volids => {}, vm_config => get_patched_config(105, { ide2 => '/dev/sde', }), vm_status => { running => 0, }, }, }, { name => '105_cdrom_in_snapshot', target => 'pve1', vmid => 105, vm_status => { running => 0, }, config_patch => { snapshots => { ohsnap => { ide2 => 'cdrom,media=cdrom', }, }, }, expected_calls => {}, expect_die => "can't migrate local cdrom drive (referenced in snapshot - ohsnap", expected => { source_volids => local_volids_for_vm(105), target_volids => {}, vm_config => get_patched_config( 105, { snapshots => { ohsnap => { ide2 => 'cdrom,media=cdrom', }, }, }, ), vm_status => { running => 0, }, }, }, { name => '105_cdrom', target => 'pve1', vmid => 105, vm_status => { running => 0, }, config_patch => { ide2 => 'cdrom,media=cdrom', }, expected_calls => {}, expect_die => "can't migrate local cdrom drive", expected => { source_volids => local_volids_for_vm(105), target_volids => {}, vm_config => get_patched_config(105, { ide2 => 'cdrom,media=cdrom', }), vm_status => { running => 0, }, }, }, { name => '149_running_missing_option_withlocaldisks', target => 'pve1', vmid => 149, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, }, expected_calls => {}, expect_die => "can't live migrate attached local disks without with-local-disks option", expected => { source_volids => local_volids_for_vm(149), target_volids => {}, vm_config => $vm_configs->{149}, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '149_running_missing_option_online', target => 'pve1', vmid => 149, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { 'with-local-disks' => 1, }, expected_calls => {}, expect_die => "can't migrate running VM without --online", expected => { source_volids => local_volids_for_vm(149), target_volids => {}, vm_config => $vm_configs->{149}, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '1033_running_customcpu', target => 'pve1', vmid => 1033, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', runningcpu => 'host,+kvm_pv_eoi,+kvm_pv_unhalt', }, opts => { online => 1, }, config_patch => { cpu => 'custom-mycpu', }, expected_calls => $default_expected_calls_online, expected => { source_volids => {}, target_volids => {}, vm_config => get_patched_config(1033, { cpu => 'custom-mycpu', }), vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', runningcpu => 'host,+kvm_pv_eoi,+kvm_pv_unhalt', }, }, }, { name => '105_replicated_to_non_replication_target', target => 'pve1', vmid => 105, vm_status => { running => 0, }, target_volids => {}, expected_calls => $replicated_expected_calls_offline, expected => { source_volids => {}, target_volids => local_volids_for_vm(105), vm_config => $vm_configs->{105}, vm_status => { running => 0, }, }, }, { name => '105_running_replicated', target => 'pve2', vmid => 105, vm_status => { running => 1, runningmachine => 'pc-i440fx-10.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, }, target_volids => local_volids_for_vm(105), expected_calls => { %{$replicated_expected_calls_online}, 'block-dirty-bitmap-add-drive-scsi0' => 1, 'block-dirty-bitmap-add-drive-ide0' => 1, }, expected => { source_volids => local_volids_for_vm(105), target_volids => local_volids_for_vm(105), vm_config => $vm_configs->{105}, vm_status => { running => 1, runningmachine => 'pc-i440fx-10.0+pve0', }, }, }, { name => '105_replicated', target => 'pve2', vmid => 105, vm_status => { running => 0, }, target_volids => local_volids_for_vm(105), expected_calls => $replicated_expected_calls_offline, expected => { source_volids => local_volids_for_vm(105), target_volids => local_volids_for_vm(105), vm_config => $vm_configs->{105}, vm_status => { running => 0, }, }, }, { name => '105_running_replicated_without_snapshot', target => 'pve2', vmid => 105, vm_status => { running => 1, runningmachine => 'pc-i440fx-10.0+pve0', }, config_patch => { snapshots => undef, }, opts => { online => 1, 'with-local-disks' => 1, }, target_volids => local_volids_for_vm(105), expected_calls => { %{$replicated_expected_calls_online}, 'block-dirty-bitmap-add-drive-scsi0' => 1, 'block-dirty-bitmap-add-drive-ide0' => 1, }, expected => { source_volids => local_volids_for_vm(105), target_volids => local_volids_for_vm(105), vm_config => get_patched_config(105, { snapshots => {}, }), vm_status => { running => 1, runningmachine => 'pc-i440fx-10.0+pve0', }, }, }, { name => '105_replicated_without_snapshot', target => 'pve2', vmid => 105, vm_status => { running => 0, }, config_patch => { snapshots => undef, }, opts => { online => 1, }, target_volids => local_volids_for_vm(105), expected_calls => $replicated_expected_calls_offline, expected => { source_volids => local_volids_for_vm(105), target_volids => local_volids_for_vm(105), vm_config => get_patched_config(105, { snapshots => {}, }), vm_status => { running => 0, }, }, }, { name => '1033_running', target => 'pve2', vmid => 1033, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, }, expected_calls => $default_expected_calls_online, expected => { source_volids => {}, target_volids => {}, vm_config => $vm_configs->{1033}, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '149_locked', target => 'pve2', vmid => 149, vm_status => { running => 0, }, config_patch => { lock => 'locked', }, expected_calls => {}, expect_die => "VM is locked", expected => { source_volids => local_volids_for_vm(149), target_volids => {}, vm_config => get_patched_config(149, { lock => 'locked', }), vm_status => { running => 0, }, }, }, { name => '149_storage_not_available', target => 'pve2', vmid => 149, vm_status => { running => 0, }, expected_calls => {}, expect_die => "storage 'local-lvm' is not available on node 'pve2'", expected => { source_volids => local_volids_for_vm(149), target_volids => {}, vm_config => $vm_configs->{149}, vm_status => { running => 0, }, }, }, { name => '149_running', target => 'pve1', vmid => 149, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, }, expected_calls => $default_expected_calls_online, expected => { source_volids => {}, target_volids => { 'local-lvm:vm-149-disk-10' => 1, 'local-dir:149/vm-149-disk-11.qcow2' => 1, }, vm_config => get_patched_config( 149, { scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G', scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G', }, ), vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '149_running_drive_mirror_fail', target => 'pve1', vmid => 149, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, }, expected_calls => {}, expect_die => "qemu_drive_mirror 'scsi1' error", fail_config => { 'qemu_drive_mirror' => 'scsi1', }, expected => { source_volids => local_volids_for_vm(149), target_volids => {}, vm_config => $vm_configs->{149}, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '149_running_unused_block_job_cancel_fail', target => 'pve1', vmid => 149, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, }, config_patch => { scsi1 => undef, unused0 => 'local-dir:149/vm-149-disk-0.qcow2', }, expected_calls => {}, expect_die => "block_job_monitor 'cancel' error", # note that 'cancel' is also used to finish and that's what this test is about fail_config => { 'block_job_monitor' => 'cancel', }, expected => { source_volids => local_volids_for_vm(149), target_volids => {}, vm_config => get_patched_config( 149, { scsi1 => undef, unused0 => 'local-dir:149/vm-149-disk-0.qcow2', }, ), vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '149_offline', target => 'pve1', vmid => 149, vm_status => { running => 0, }, opts => { 'with-local-disks' => 1, }, expected_calls => $default_expected_calls_offline, expected => { source_volids => {}, target_volids => local_volids_for_vm(149), vm_config => $vm_configs->{149}, vm_status => { running => 0, }, }, }, { name => '149_storage_migrate_fail', target => 'pve1', vmid => 149, vm_status => { running => 0, }, opts => { 'with-local-disks' => 1, }, fail_config => { 'storage_migrate' => 'local-lvm:vm-149-disk-0', }, expected_calls => {}, expect_die => "storage_migrate 'local-lvm:vm-149-disk-0' error", expected => { source_volids => local_volids_for_vm(149), target_volids => {}, vm_config => $vm_configs->{149}, vm_status => { running => 0, }, }, }, { name => '111_running_pending', target => 'pve1', vmid => 111, vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, opts => { online => 1, 'with-local-disks' => 1, }, expected_calls => $default_expected_calls_online, expected => { source_volids => {}, target_volids => { 'local-zfs:vm-111-disk-0' => 1, 'local-lvm:vm-111-disk-10' => 1, }, vm_config => get_patched_config( 111, { ide0 => 'local-lvm:vm-111-disk-10,format=raw,size=4G', pending => { scsi0 => 'local-zfs:vm-111-disk-0,size=103M', }, }, ), vm_status => { running => 1, runningmachine => 'pc-q35-5.0+pve0', }, }, }, { name => '123_alias_fail', target => 'pve1', vmid => 123, vm_status => { running => 0, }, opts => { 'with-local-disks' => 1, }, expected_calls => {}, expect_die => "detected not supported aliased volumes", expected => { source_volids => local_volids_for_vm(123), target_volids => {}, vm_config => $vm_configs->{123}, vm_status => { running => 0, }, }, }, ]; my $single_test_name = shift; mkdir $RUN_DIR_PATH; foreach my $test (@{$tests}) { my $name = $test->{name}; next if defined($single_test_name) && $name ne $single_test_name; my $run_dir = "${RUN_DIR_PATH}/${name}"; mkdir $run_dir; file_set_contents("${run_dir}/replication_config", to_json($replication_config)); file_set_contents("${run_dir}/storage_config", to_json($storage_config)); file_set_contents("${run_dir}/source_vdisks", to_json($source_vdisks)); my $expect_die = $test->{expect_die}; my $expected = $test->{expected}; my $source_volids = local_volids_for_vm($test->{vmid}); my $target_volids = $test->{target_volids} // {}; my $config_patch = $test->{config_patch}; my $vm_config = get_patched_config($test->{vmid}, $test->{config_patch}); my $fail_config = $test->{fail_config} // {}; my $storage_migrate_map = $test->{storage_migrate_map} // {}; if (my $targetstorage = $test->{opts}->{targetstorage}) { $test->{opts}->{storagemap} = PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id'); } my $migrate_params = { target => $test->{target}, vmid => $test->{vmid}, opts => $test->{opts}, }; file_set_contents("${run_dir}/nbd_info", to_json({})); file_set_contents("${run_dir}/source_volids", to_json($source_volids)); file_set_contents("${run_dir}/target_volids", to_json($target_volids)); file_set_contents("${run_dir}/vm_config", to_json($vm_config)); file_set_contents("${run_dir}/vm_status", to_json($test->{vm_status})); file_set_contents("${run_dir}/expected_calls", to_json($test->{expected_calls})); file_set_contents("${run_dir}/fail_config", to_json($fail_config)); file_set_contents("${run_dir}/storage_migrate_map", to_json($storage_migrate_map)); file_set_contents("${run_dir}/migrate_params", to_json($migrate_params)); $ENV{QM_LIB_PATH} = $QM_LIB_PATH; $ENV{RUN_DIR_PATH} = $run_dir; my $exitcode = run_command( [ '/usr/bin/perl', "-I${MIGRATE_LIB_PATH}", "-I${MIGRATE_LIB_PATH}/test", "${MIGRATE_LIB_PATH}/test/MigrationTest/QemuMigrateMock.pm", ], noerr => 1, errfunc => sub { print "#$name - $_[0]\n" }, ); if (defined($expect_die) && $exitcode) { my $log = file_get_contents("${run_dir}/log"); my @lines = split /\n/, $log; my $matched = 0; foreach my $line (@lines) { $matched = 1 if $line =~ m/^err:.*\Q${expect_die}\E/; $matched = 1 if $line =~ m/^warn:.*\Q${expect_die}\E/; } if (!$matched) { fail($name); note("expected error message is not present in log"); } } elsif (defined($expect_die) && !$exitcode) { fail($name); note("mocked migrate call didn't fail, but it was expected to - check log"); } elsif (!defined($expect_die) && $exitcode) { fail($name); note("mocked migrate call failed, but it was not expected - check log"); } my $expected_calls = decode_json(file_get_contents("${run_dir}/expected_calls")); foreach my $call (keys %{$expected_calls}) { fail($name); note("expected call '$call' was not made"); } if (!defined($expect_die)) { my $nbd_info = decode_json(file_get_contents("${run_dir}/nbd_info")); foreach my $drive (keys %{$nbd_info}) { fail($name); note("drive '$drive' was not mirrored"); } } my $actual = { source_volids => decode_json(file_get_contents("${run_dir}/source_volids")), target_volids => decode_json(file_get_contents("${run_dir}/target_volids")), vm_config => decode_json(file_get_contents("${run_dir}/vm_config")), vm_status => decode_json(file_get_contents("${run_dir}/vm_status")), }; is_deeply($actual, $expected, $name); } done_testing(); ================================================ FILE: src/test/run_qemu_restore_config_tests.pl ================================================ #!/usr/bin/perl use strict; use warnings; use lib qw(..); use Test::MockModule; use Test::More; use File::Basename; use PVE::QemuServer; use PVE::Tools qw(dir_glob_foreach file_get_contents); my $INPUT_DIR = './restore-config-input'; my $EXPECTED_DIR = './restore-config-expected'; # NOTE update when you add/remove tests plan tests => 4; my $pve_cluster_module = Test::MockModule->new("PVE::Cluster"); $pve_cluster_module->mock( cfs_read_file => sub { my ($file) = @_; if ($file eq 'datacenter.cfg') { return {}; } else { die "'cfs_read_file' called - missing mock?\n"; } }, ); dir_glob_foreach( './restore-config-input', '[0-9]+.conf', sub { my ($file) = @_; my $vmid = basename($file, ('.conf')); my $fh = IO::File->new("${INPUT_DIR}/${file}", "r") or die "unable to read '$file' - $!\n"; my $map = {}; my $disknum = 0; # NOTE For now, the map is hardcoded to a file-based 'target' storage. # In the future, the test could be extended to include parse_backup_hints # and restore_allocate_devices. Even better if the config-related logic from # the restore_XYZ_archive functions could become a separate function. while (defined(my $line = <$fh>)) { if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) { my ($drive, undef, $storeid, $fmt) = ($1, $2, $3, $4); $fmt ||= 'raw'; $map->{$drive} = "target:${vmid}/vm-${vmid}-disk-${disknum}.${fmt}"; $disknum++; } } $fh->seek(0, 0) or die "seek failed - $!\n"; my $got = ''; my $cookie = { netcount => 0 }; while (defined(my $line = <$fh>)) { $got .= PVE::QemuServer::restore_update_config_line( $cookie, $map, $line, 0, ); } my $expected = file_get_contents("${EXPECTED_DIR}/${file}"); is_deeply($got, $expected, $file); }, ); done_testing(); ================================================ FILE: src/test/run_snapshot_tests.pl ================================================ #!/usr/bin/perl use strict; use warnings; use TAP::Harness; my $harness = TAP::Harness->new({ "verbosity" => -2 }); my $res = $harness->runtests("snapshot-test.pm"); system("rm -rf snapshot-working/"); exit -1 if $res->{failed}; ================================================ FILE: src/test/snapshot-expected/commit/qemu-server/101.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/commit/qemu-server/102.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/commit/qemu-server/201.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: prepare snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/commit/qemu-server/202.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom lock: snapshot memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: abcdefg smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [abcdefg] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [abcdefg2] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: abcdefg smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: prepare snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/commit/qemu-server/203.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom lock: snapshot memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: delete snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/create/qemu-server/101.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/create/qemu-server/102.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/create/qemu-server/103.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/create/qemu-server/104.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume [test2] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/create/qemu-server/105.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test sata0: local:snapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 sata0: local:snapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G ================================================ FILE: src/test/snapshot-expected/create/qemu-server/106.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/create/qemu-server/201.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/create/qemu-server/202.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 sata0: local:snapshotable-disk-2,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl virtio0: local:unsnapshotable-disk-1,discard=on,size=32G ================================================ FILE: src/test/snapshot-expected/create/qemu-server/203.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/create/qemu-server/301.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/create/qemu-server/302.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/create/qemu-server/303.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/delete/qemu-server/101.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/delete/qemu-server/102.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/delete/qemu-server/103.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/delete/qemu-server/104.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test3 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test3] #another test comment bootdisk: ide0 cores: 2 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/delete/qemu-server/105.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/delete/qemu-server/106.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 sata0: local:snapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl virtio0: local:unsnapshotable-disk-2,discard=on,size=32G ================================================ FILE: src/test/snapshot-expected/delete/qemu-server/201.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom lock: snapshot-delete memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: delete snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/delete/qemu-server/202.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom lock: snapshot-delete memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test sata0: local:snapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl virtio0: local:unsnapshotable-disk-2,discard=on,size=32G [test] #test comment bootdisk: ide0 cores: 4 ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 sata0: local:snapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: delete snaptime: 1234567890 sockets: 1 unused0: local:snapshotable-disk-1 vga: qxl virtio0: local:unsnapshotable-disk-2,discard=on,size=32G ================================================ FILE: src/test/snapshot-expected/delete/qemu-server/203.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom lock: backup memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/delete/qemu-server/204.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/prepare/qemu-server/101.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom lock: snapshot memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: prepare snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/prepare/qemu-server/102.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom lock: snapshot memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: prepare snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/prepare/qemu-server/103.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom lock: snapshot memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: prepare snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/prepare/qemu-server/104.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom lock: snapshot memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: prepare snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/prepare/qemu-server/200.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom lock: snapshot memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/prepare/qemu-server/201.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/prepare/qemu-server/202.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/prepare/qemu-server/300.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/101.conf ================================================ # this is a description agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/102.conf ================================================ # this is a description bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/103.conf ================================================ # this is a description bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/104.conf ================================================ # this is a description bootdisk: ide0 cores: 3 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 3 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test3] #another test comment bootdisk: ide0 cores: 2 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/105.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test sata0: local:snapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 sata0: local:snapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/106.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/201.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/202.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/203.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: delete snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/204.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom lock: backup memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/205.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/206.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test sata0: local:unsnapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 sata0: local:unsnapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/207.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom lock: rollback memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test sata0: local:snapshotable-disk-4,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 sata0: local:snapshotable-disk-4,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/301.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/302.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-expected/rollback/qemu-server/303.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/commit/qemu-server/101.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom lock: snapshot memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: prepare snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/commit/qemu-server/102.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom lock: snapshot memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: prepare snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/commit/qemu-server/201.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: prepare snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/commit/qemu-server/202.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom lock: snapshot memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: abcdefg smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [abcdefg] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [abcdefg2] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: abcdefg smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: prepare snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/commit/qemu-server/203.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom lock: snapshot memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: delete snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/create/qemu-server/101.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/create/qemu-server/102.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/create/qemu-server/103.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/create/qemu-server/104.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/create/qemu-server/105.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 sata0: local:snapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G ================================================ FILE: src/test/snapshot-input/create/qemu-server/106.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/create/qemu-server/201.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/create/qemu-server/202.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 sata0: local:snapshotable-disk-2,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl virtio0: local:unsnapshotable-disk-1,discard=on,size=32G ================================================ FILE: src/test/snapshot-input/create/qemu-server/203.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/create/qemu-server/301.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/create/qemu-server/302.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/create/qemu-server/303.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/delete/qemu-server/101.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/delete/qemu-server/102.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/delete/qemu-server/103.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/delete/qemu-server/104.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test3 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 3 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test3] #another test comment bootdisk: ide0 cores: 2 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/delete/qemu-server/105.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/delete/qemu-server/106.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test sata0: local:snapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl virtio0: local:unsnapshotable-disk-2,discard=on,size=32G [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 sata0: local:snapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl virtio0: local:unsnapshotable-disk-2,discard=on,size=32G ================================================ FILE: src/test/snapshot-input/delete/qemu-server/201.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/delete/qemu-server/202.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test sata0: local:snapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl virtio0: local:unsnapshotable-disk-2,discard=on,size=32G [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 sata0: local:snapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl virtio0: local:unsnapshotable-disk-2,discard=on,size=32G ================================================ FILE: src/test/snapshot-input/delete/qemu-server/203.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom lock: backup memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/delete/qemu-server/204.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/prepare/qemu-server/101.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/prepare/qemu-server/102.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/prepare/qemu-server/103.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/prepare/qemu-server/104.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/prepare/qemu-server/200.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom lock: snapshot memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/prepare/qemu-server/201.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/prepare/qemu-server/202.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/prepare/qemu-server/300.conf ================================================ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/101.conf ================================================ # this is a description agent: 1 bootdisk: ide2 cores: 2 ide0: local:snapshotable-disk-1,size=32G ide2: none,media=cdrom memory: 4096 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 2 [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/102.conf ================================================ # this is a description agent: 1 bootdisk: ide2 cores: 2 ide0: local:snapshotable-disk-1,size=32G ide2: none,media=cdrom machine: pc memory: 4096 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 2 [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/103.conf ================================================ # this is a description agent: 1 bootdisk: ide2 cores: 2 ide0: local:snapshotable-disk-1,size=32G ide2: none,media=cdrom machine: pc memory: 4096 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 2 [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/104.conf ================================================ # this is a description agent: 1 bootdisk: ide2 cores: 2 ide0: local:snapshotable-disk-1,size=32G ide2: none,media=cdrom machine: pc memory: 4096 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 2 [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test2] #test comment bootdisk: ide0 cores: 3 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl [test3] #another test comment bootdisk: ide0 cores: 2 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test2 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/105.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test sata0: local:snapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 sata0: local:snapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/106.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: pc memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/201.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/202.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/203.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: delete snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/204.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom lock: backup memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/205.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/206.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test sata0: local:unsnapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 sata0: local:unsnapshotable-disk-3,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/207.conf ================================================ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test sata0: local:snapshotable-disk-4,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G [test] #test comment bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 sata0: local:snapshotable-disk-4,discard=on,size=32G smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl virtio0: local:snapshotable-disk-2,discard=on,size=32G ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/301.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/302.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-input/rollback/qemu-server/303.conf ================================================ agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test smbios1: uuid=01234567-890a-bcde-f012-34567890abcd sockets: 1 vga: qxl [test] #test comment agent: 1 bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 vga: qxl vmstate: somestorage:state-volume ================================================ FILE: src/test/snapshot-test.pm ================================================ package PVE::QemuServer; ## no critic use strict; use warnings; use lib qw(..); use PVE::Storage; use PVE::Storage::Plugin; use PVE::QemuServer; use PVE::QemuConfig; use PVE::Tools; use PVE::ReplicationConfig; use Test::MockModule; use Test::More; my $activate_storage_possible = 1; my $nodename; my $snapshot_possible; my $vol_snapshot_possible = {}; my $vol_snapshot_delete_possible = {}; my $vol_snapshot_rollback_possible = {}; my $vol_snapshot_rollback_enabled = {}; my $vol_snapshot = {}; my $vol_snapshot_delete = {}; my $vol_snapshot_rollback = {}; my $running; my $freeze_possible; my $stop_possible; my $save_vmstate_works; my $vm_mon = {}; # Mocked methods sub mocked_volume_snapshot { my ($storecfg, $volid, $snapname) = @_; die "Storage config not mocked! aborting\n" if defined($storecfg); die "volid undefined\n" if !defined($volid); die "snapname undefined\n" if !defined($snapname); if ($vol_snapshot_possible->{$volid}) { if (defined($vol_snapshot->{$volid})) { $vol_snapshot->{$volid} .= ",$snapname"; } else { $vol_snapshot->{$volid} = $snapname; } return 1; } else { die "volume snapshot disabled\n"; } } sub mocked_volume_snapshot_delete { my ($storecfg, $volid, $snapname) = @_; die "Storage config not mocked! aborting\n" if defined($storecfg); die "volid undefined\n" if !defined($volid); die "snapname undefined\n" if !defined($snapname); if ($vol_snapshot_delete_possible->{$volid}) { if (defined($vol_snapshot_delete->{$volid})) { $vol_snapshot_delete->{$volid} .= ",$snapname"; } else { $vol_snapshot_delete->{$volid} = $snapname; } return 1; } else { die "volume snapshot delete disabled\n"; } } sub mocked_volume_snapshot_rollback { my ($storecfg, $volid, $snapname) = @_; die "Storage config not mocked! aborting\n" if defined($storecfg); die "volid undefined\n" if !defined($volid); die "snapname undefined\n" if !defined($snapname); if ($vol_snapshot_rollback_enabled->{$volid}) { if (defined($vol_snapshot_rollback->{$volid})) { $vol_snapshot_rollback->{$volid} .= ",$snapname"; } else { $vol_snapshot_rollback->{$volid} = $snapname; } return 1; } else { die "volume snapshot rollback disabled\n"; } } sub mocked_volume_rollback_is_possible { my ($storecfg, $volid, $snapname) = @_; die "Storage config not mocked! aborting\n" if defined($storecfg); die "volid undefined\n" if !defined($volid); die "snapname undefined\n" if !defined($snapname); return $vol_snapshot_rollback_possible->{$volid} if ($vol_snapshot_rollback_possible->{$volid}); die "volume_rollback_is_possible failed\n"; } sub mocked_activate_storage { my ($storecfg, $storeid) = @_; die "Storage config not mocked! aborting\n" if defined($storecfg); die "storage activation failed\n" if !$activate_storage_possible; return; } sub mocked_activate_volumes { my ($storecfg, $volumes) = @_; die "Storage config not mocked! aborting\n" if defined($storecfg); die "wrong volume - fake vmstate expected!\n" if ((scalar @$volumes != 1) || @$volumes[0] ne "somestorage:state-volume"); return; } sub mocked_deactivate_volumes { my ($storecfg, $volumes) = @_; die "Storage config not mocked! aborting\n" if defined($storecfg); die "wrong volume - fake vmstate expected!\n" if ((scalar @$volumes != 1) || @$volumes[0] ne "somestorage:state-volume"); return; } sub mocked_vdisk_free { my ($storecfg, $vmstate) = @_; die "Storage config not mocked! aborting\n" if defined($storecfg); die "wrong vdisk - fake vmstate expected!\n" if ($vmstate ne "somestorage:state-volume"); return; } sub mocked_run_command { my ($cmd, %param) = @_; my $cmdstring; if (my $ref = ref($cmd)) { $cmdstring = PVE::Tools::cmd2string($cmd); if ($cmdstring =~ m/.*\/qemu-(un)?freeze.*/) { return 1 if $freeze_possible; die "qemu-[un]freeze disabled\n"; } if ($cmdstring =~ m/.*\/qemu-stop.*--kill.*/) { if ($stop_possible) { $running = 0; return 1; } else { return 0; } } } die "unexpected run_command call: '$cmdstring', aborting\n"; } # Testing methods sub test_file { my ($exp_fn, $real_fn) = @_; my $ret; eval { $ret = system("diff -u '$exp_fn' '$real_fn'"); }; die if $@; return !$ret; } sub testcase_prepare { my ($vmid, $snapname, $save_vmstate, $comment, $exp_err) = @_; subtest "Preparing snapshot '$snapname' for vm '$vmid'" => sub { plan tests => 2; $@ = undef; eval { PVE::QemuConfig->__snapshot_prepare($vmid, $snapname, $save_vmstate, $comment); }; is($@, $exp_err, "\$@ correct"); ok( test_file( "snapshot-expected/prepare/qemu-server/$vmid.conf", "snapshot-working/prepare/qemu-server/$vmid.conf", ), "config file correct", ); }; } sub testcase_commit { my ($vmid, $snapname, $exp_err) = @_; subtest "Committing snapshot '$snapname' for vm '$vmid'" => sub { plan tests => 2; $@ = undef; eval { PVE::QemuConfig->__snapshot_commit($vmid, $snapname); }; is($@, $exp_err, "\$@ correct"); ok( test_file( "snapshot-expected/commit/qemu-server/$vmid.conf", "snapshot-working/commit/qemu-server/$vmid.conf", ), "config file correct", ); } } sub testcase_create { my ($vmid, $snapname, $save_vmstate, $comment, $exp_err, $exp_vol_snap, $exp_vol_snap_delete) = @_; subtest "Creating snapshot '$snapname' for vm '$vmid'" => sub { plan tests => 4; $vol_snapshot = {}; $vol_snapshot_delete = {}; $exp_vol_snap = {} if !defined($exp_vol_snap); $exp_vol_snap_delete = {} if !defined($exp_vol_snap_delete); $@ = undef; eval { PVE::QemuConfig->snapshot_create($vmid, $snapname, $save_vmstate, $comment); }; is($@, $exp_err, "\$@ correct"); is_deeply($vol_snapshot, $exp_vol_snap, "created correct volume snapshots"); is_deeply( $vol_snapshot_delete, $exp_vol_snap_delete, "deleted correct volume snapshots", ); ok( test_file( "snapshot-expected/create/qemu-server/$vmid.conf", "snapshot-working/create/qemu-server/$vmid.conf", ), "config file correct", ); }; } sub testcase_delete { my ($vmid, $snapname, $force, $exp_err, $exp_vol_snap_delete) = @_; subtest "Deleting snapshot '$snapname' of vm '$vmid'" => sub { plan tests => 3; $vol_snapshot_delete = {}; $exp_vol_snap_delete = {} if !defined($exp_vol_snap_delete); $@ = undef; eval { PVE::QemuConfig->snapshot_delete($vmid, $snapname, $force); }; is($@, $exp_err, "\$@ correct"); is_deeply( $vol_snapshot_delete, $exp_vol_snap_delete, "deleted correct volume snapshots", ); ok( test_file( "snapshot-expected/delete/qemu-server/$vmid.conf", "snapshot-working/delete/qemu-server/$vmid.conf", ), "config file correct", ); }; } sub testcase_rollback { my ($vmid, $snapname, $exp_err, $exp_vol_snap_rollback) = @_; subtest "Rolling back to snapshot '$snapname' of vm '$vmid'" => sub { plan tests => 3; $vol_snapshot_rollback = {}; $running = 1; $exp_vol_snap_rollback = {} if !defined($exp_vol_snap_rollback); $@ = undef; eval { PVE::QemuConfig->snapshot_rollback($vmid, $snapname); }; is($@, $exp_err, "\$@ correct"); is_deeply( $vol_snapshot_rollback, $exp_vol_snap_rollback, "rolled back to correct volume snapshots", ); ok( test_file( "snapshot-expected/rollback/qemu-server/$vmid.conf", "snapshot-working/rollback/qemu-server/$vmid.conf", ), "config file correct", ); }; } # BEGIN mocked PVE::QemuConfig methods sub config_file_lock { return "snapshot-working/pve-test.lock"; } sub cfs_config_path { my ($class, $vmid, $node) = @_; $node = $nodename if !$node; return "snapshot-working/$node/qemu-server/$vmid.conf"; } sub load_config { my ($class, $vmid, $node) = @_; my $filename = $class->cfs_config_path($vmid, $node); my $raw = PVE::Tools::file_get_contents($filename); my $conf = PVE::QemuServer::parse_vm_config($filename, $raw); return $conf; } sub write_config { my ($class, $vmid, $conf) = @_; my $filename = $class->cfs_config_path($vmid); if ($conf->{snapshots}) { foreach my $snapname (keys %{ $conf->{snapshots} }) { $conf->{snapshots}->{$snapname}->{snaptime} = "1234567890" if $conf->{snapshots}->{$snapname}->{snaptime}; } } my $raw = PVE::QemuServer::write_vm_config($filename, $conf); PVE::Tools::file_set_contents($filename, $raw); } sub has_feature { my ($class, $feature, $conf, $storecfg, $snapname, $running, $backup_only) = @_; return $snapshot_possible; } sub __snapshot_save_vmstate { my ($class, $vmid, $conf, $snapname, $storecfg) = @_; die "save_vmstate failed\n" if !$save_vmstate_works; my $snap = $conf->{snapshots}->{$snapname}; $snap->{vmstate} = "somestorage:state-volume"; $snap->{runningmachine} = "q35"; } sub assert_config_exists_on_node { my ($vmid, $node) = @_; return -f cfs_config_path("PVE::QemuConfig", $vmid, $node); } # END mocked PVE::QemuConfig methods # BEGIN mocked PVE::QemuServer::Helpers methods sub vm_running_locally { return $running; } # END mocked PVE::QemuServer::Helpers methods # BEGIN mocked PVE::QemuServer::Monitor methods sub qmp_cmd { my ($peer, $execute, %arguments) = @_; my $cmd = { execute => $execute, arguments => \%arguments }; my $exec = $cmd->{execute}; if ($exec eq "guest-ping") { die "guest-ping disabled\n" if !$vm_mon->{guest_ping}; return; } if ($exec eq "guest-fsfreeze-freeze" || $exec eq "guest-fsfreeze-thaw") { die "freeze disabled\n" if !$freeze_possible; return; } if ($exec eq "savevm-start") { die "savevm-start disabled\n" if !$vm_mon->{savevm_start}; return; } if ($exec eq "savevm-end") { die "savevm-end disabled\n" if !$vm_mon->{savevm_end}; return; } if ($exec eq "query-savevm") { return { "status" => "completed", "bytes" => 1024 * 1024 * 1024, "total-time" => 5000, }; } die "unexpected vm_qmp_command!\n"; } # END mocked PVE::QemuServer::Monitor methods # # BEGIN mocked PVE::QemuMigrate::Helpers methods sub set_migration_caps { } # ignored # END mocked PVE::QemuMigrate::Helpers methods # BEGIN redefine PVE::QemuServer methods sub do_snapshots_type { return 'storage'; } sub vm_start { my ($storecfg, $vmid, $params, $migrate_opts) = @_; die "Storage config not mocked! aborting\n" if defined($storecfg); die "statefile and forcemachine must be both defined or undefined! aborting\n" if defined($params->{statefile}) xor defined($params->{forcemachine}); return; } sub vm_stop { my ( $storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom, ) = @_; $running = 0 if $stop_possible; return; } # END redefine PVE::QemuServer methods PVE::Tools::run_command("rm -rf snapshot-working"); PVE::Tools::run_command("cp -a snapshot-input snapshot-working"); my $qemu_migrate_helpers_module = Test::MockModule->new('PVE::QemuMigrate::Helpers'); $qemu_migrate_helpers_module->mock('set_migration_caps', \&set_migration_caps); my $qemu_helpers_module = Test::MockModule->new('PVE::QemuServer::Helpers'); $qemu_helpers_module->mock('vm_running_locally', \&vm_running_locally); my $qemu_monitor_module = Test::MockModule->new('PVE::QemuServer::Monitor'); $qemu_monitor_module->mock('qmp_cmd', \&qmp_cmd); my $qemu_config_module = Test::MockModule->new('PVE::QemuConfig'); $qemu_config_module->mock('config_file_lock', \&config_file_lock); $qemu_config_module->mock('cfs_config_path', \&cfs_config_path); $qemu_config_module->mock('load_config', \&load_config); $qemu_config_module->mock('write_config', \&write_config); $qemu_config_module->mock('has_feature', \&has_feature); $qemu_config_module->mock('__snapshot_save_vmstate', \&__snapshot_save_vmstate); $qemu_config_module->mock('assert_config_exists_on_node', \&assert_config_exists_on_node); # ignore existing replication config my $repl_config_module = Test::MockModule->new('PVE::ReplicationConfig'); $repl_config_module->mock('new' => sub { return bless {}, "PVE::ReplicationConfig" }); $repl_config_module->mock('check_for_existing_jobs' => sub { return }); my $storage_module = Test::MockModule->new('PVE::Storage'); $storage_module->mock('config', sub { return; }); $storage_module->mock('path', sub { return "/some/store/statefile/path"; }); $storage_module->mock('activate_storage', \&mocked_activate_storage); $storage_module->mock('activate_volumes', \&mocked_activate_volumes); $storage_module->mock('deactivate_volumes', \&mocked_deactivate_volumes); $storage_module->mock('vdisk_free', \&mocked_vdisk_free); $storage_module->mock('volume_snapshot', \&mocked_volume_snapshot); $storage_module->mock('volume_snapshot_delete', \&mocked_volume_snapshot_delete); $storage_module->mock('volume_snapshot_rollback', \&mocked_volume_snapshot_rollback); $storage_module->mock('volume_rollback_is_possible', \&mocked_volume_rollback_is_possible); $running = 1; $freeze_possible = 1; $save_vmstate_works = 1; printf("\n"); printf("Running prepare tests\n"); printf("\n"); $nodename = "prepare"; printf("\n"); printf("Setting has_feature to return true\n"); printf("\n"); $snapshot_possible = 1; printf("Successful snapshot_prepare with no existing snapshots\n"); testcase_prepare("101", "test", 0, "test comment", ''); printf("Successful snapshot_prepare with no existing snapshots, including vmstate\n"); testcase_prepare("102", "test", 1, "test comment", ''); printf("Successful snapshot_prepare with one existing snapshot\n"); testcase_prepare("103", "test2", 0, "test comment", ""); printf("Successful snapshot_prepare with one existing snapshot, including vmstate\n"); testcase_prepare("104", "test2", 1, "test comment", ""); printf("Expected error for snapshot_prepare on locked container\n"); testcase_prepare("200", "test", 0, "test comment", "VM is locked (snapshot)\n"); printf("Expected error for snapshot_prepare with duplicate snapshot name\n"); testcase_prepare("201", "test", 0, "test comment", "snapshot name 'test' already used\n"); $save_vmstate_works = 0; printf("Expected error for snapshot_prepare with failing save_vmstate\n"); testcase_prepare("202", "test", 1, "test comment", "save_vmstate failed\n"); $save_vmstate_works = 1; printf("\n"); printf("Setting has_feature to return false\n"); printf("\n"); $snapshot_possible = 0; printf("Expected error for snapshot_prepare if snapshots not possible\n"); testcase_prepare("300", "test", 0, "test comment", "snapshot feature is not available\n"); printf("\n"); printf("Running commit tests\n"); printf("\n"); $nodename = "commit"; printf("\n"); printf("Setting has_feature to return true\n"); printf("\n"); $snapshot_possible = 1; printf("Successful snapshot_commit with one prepared snapshot\n"); testcase_commit("101", "test", ""); printf("Successful snapshot_commit with one committed and one prepared snapshot\n"); testcase_commit("102", "test2", ""); printf("Expected error for snapshot_commit with no snapshot lock\n"); testcase_commit("201", "test", "missing snapshot lock\n"); printf("Expected error for snapshot_commit with invalid snapshot name\n"); testcase_commit("202", "test", "snapshot 'test' does not exist\n"); printf("Expected error for snapshot_commit with invalid snapshot state\n"); testcase_commit("203", "test", "wrong snapshot state\n"); $vol_snapshot_possible->{"local:snapshotable-disk-1"} = 1; $vol_snapshot_possible->{"local:snapshotable-disk-2"} = 1; $vol_snapshot_possible->{"local:snapshotable-disk-3"} = 1; $vol_snapshot_delete_possible->{"local:snapshotable-disk-1"} = 1; $vol_snapshot_delete_possible->{"local:snapshotable-disk-3"} = 1; $vol_snapshot_rollback_enabled->{"local:snapshotable-disk-1"} = 1; $vol_snapshot_rollback_enabled->{"local:snapshotable-disk-2"} = 1; $vol_snapshot_rollback_enabled->{"local:snapshotable-disk-3"} = 1; $vol_snapshot_rollback_possible->{"local:snapshotable-disk-1"} = 1; $vol_snapshot_rollback_possible->{"local:snapshotable-disk-2"} = 1; $vol_snapshot_rollback_possible->{"local:snapshotable-disk-3"} = 1; $vol_snapshot_rollback_possible->{"local:snapshotable-disk-4"} = 1; $vm_mon->{guest_ping} = 1; $vm_mon->{savevm_start} = 1; $vm_mon->{savevm_end} = 1; # possible, but fails $vol_snapshot_rollback_possible->{"local:snapshotable-disk-4"} = 1; #printf("\n"); #printf("Setting up Mocking for PVE::Tools\n"); #my $tools_module = Test::MockModule->new('PVE::Tools'); #$tools_module->mock('run_command' => \&mocked_run_command); #printf("\trun_command() mocked\n"); # $nodename = "create"; printf("\n"); printf("Running create tests\n"); printf("\n"); printf("Successful snapshot_create with no existing snapshots\n"); testcase_create("101", "test", 0, "test comment", "", { "local:snapshotable-disk-1" => "test" }); printf("Successful snapshot_create with no existing snapshots, including vmstate\n"); testcase_create("102", "test", 1, "test comment", "", { "local:snapshotable-disk-1" => "test" }); printf("Successful snapshot_create with one existing snapshots\n"); testcase_create("103", "test2", 0, "test comment", "", { "local:snapshotable-disk-1" => "test2" }); printf("Successful snapshot_create with one existing snapshots, including vmstate\n"); testcase_create("104", "test2", 1, "test comment", "", { "local:snapshotable-disk-1" => "test2" }); printf("Successful snapshot_create with multiple mps\n"); testcase_create( "105", "test", 0, "test comment", "", { "local:snapshotable-disk-1" => "test", "local:snapshotable-disk-2" => "test", "local:snapshotable-disk-3" => "test", }, ); $freeze_possible = 0; printf("Successful snapshot_create with no existing snapshots and broken freeze\n"); testcase_create("106", "test", 1, "test comment", "", { "local:snapshotable-disk-1" => "test" }); $freeze_possible = 1; printf("Expected error for snapshot_create when volume snapshot is not possible\n"); testcase_create("201", "test", 0, "test comment", "volume snapshot disabled\n\n"); printf("Expected error for snapshot_create when volume snapshot is not possible for one drive\n"); testcase_create( "202", "test", 0, "test comment", "volume snapshot disabled\n\n", { "local:snapshotable-disk-1" => "test" }, { "local:snapshotable-disk-1" => "test" }, ); $vm_mon->{savevm_start} = 0; printf("Expected error for snapshot_create when QEMU mon command 'savevm-start' fails\n"); testcase_create("203", "test", 0, "test comment", "savevm-start disabled\n\n"); $vm_mon->{savevm_start} = 1; printf("Successful snapshot_create with no existing snapshots but set machine type\n"); testcase_create("301", "test", 1, "test comment", "", { "local:snapshotable-disk-1" => "test" }); $activate_storage_possible = 0; printf("Expected error for snapshot_create when storage activation is not possible\n"); testcase_create("303", "test", 1, "test comment", "storage activation failed\n\n"); $activate_storage_possible = 1; $nodename = "delete"; printf("\n"); printf("Running delete tests\n"); printf("\n"); printf("Successful snapshot_delete of only existing snapshot\n"); testcase_delete("101", "test", 0, "", { "local:snapshotable-disk-1" => "test" }); printf("Successful snapshot_delete of leaf snapshot\n"); testcase_delete("102", "test2", 0, "", { "local:snapshotable-disk-1" => "test2" }); printf("Successful snapshot_delete of root snapshot\n"); testcase_delete("103", "test", 0, "", { "local:snapshotable-disk-1" => "test" }); printf("Successful snapshot_delete of intermediate snapshot\n"); testcase_delete("104", "test2", 0, "", { "local:snapshotable-disk-1" => "test2" }); printf("Successful snapshot_delete with broken volume_snapshot_delete and force=1\n"); testcase_delete("105", "test", 1, ""); printf("Successful snapshot_delete with mp broken volume_snapshot_delete and force=1\n"); testcase_delete( "106", "test", 1, "", { "local:snapshotable-disk-1" => "test", "local:snapshotable-disk-3" => "test" }, ); printf( "Expected error when snapshot_delete fails with broken volume_snapshot_delete and force=0\n"); testcase_delete("201", "test", 0, "volume snapshot delete disabled\n"); printf( "Expected error when snapshot_delete fails with broken mp volume_snapshot_delete and force=0\n" ); testcase_delete( "202", "test", 0, "volume snapshot delete disabled\n", { "local:snapshotable-disk-1" => "test" }, ); printf("Expected error for snapshot_delete with locked config\n"); testcase_delete("203", "test", 0, "VM is locked (backup)\n"); $activate_storage_possible = 0; printf("Expected error for snapshot_delete when storage activation is not possible\n"); testcase_delete("204", "test", 0, "storage activation failed\n"); $activate_storage_possible = 1; $nodename = "rollback"; printf("\n"); printf("Running rollback tests\n"); printf("\n"); $stop_possible = 1; printf("Successful snapshot_rollback to only existing snapshot\n"); testcase_rollback("101", "test", "", { "local:snapshotable-disk-1" => "test" }); printf("Successful snapshot_rollback to leaf snapshot\n"); testcase_rollback("102", "test2", "", { "local:snapshotable-disk-1" => "test2" }); printf("Successful snapshot_rollback to root snapshot\n"); testcase_rollback("103", "test", "", { "local:snapshotable-disk-1" => "test" }); printf("Successful snapshot_rollback to intermediate snapshot\n"); testcase_rollback("104", "test2", "", { "local:snapshotable-disk-1" => "test2" }); printf("Successful snapshot_rollback with multiple mp\n"); testcase_rollback( "105", "test", "", { "local:snapshotable-disk-1" => "test", "local:snapshotable-disk-2" => "test", "local:snapshotable-disk-3" => "test", }, ); printf( "Successful snapshot_rollback to only existing snapshot, with saved vmstate and machine config\n" ); testcase_rollback("106", "test", "", { "local:snapshotable-disk-1" => "test" }); printf("Expected error for snapshot_rollback with non-existing snapshot\n"); testcase_rollback("201", "test2", "snapshot 'test2' does not exist\n"); printf("Expected error for snapshot_rollback if volume rollback not possible\n"); testcase_rollback("202", "test", "volume_rollback_is_possible failed\n"); printf("Expected error for snapshot_rollback with incomplete snapshot\n"); testcase_rollback("203", "test", "unable to rollback to incomplete snapshot (snapstate = delete)\n"); printf("Expected error for snapshot_rollback with lock\n"); testcase_rollback("204", "test", "VM is locked (backup)\n"); $stop_possible = 0; printf("Expected error for snapshot_rollback with unkillable container\n"); testcase_rollback("205", "test", "unable to rollback vm 205: vm is running\n"); $stop_possible = 1; printf("Expected error for snapshot_rollback with mp rollback_is_possible failure\n"); testcase_rollback("206", "test", "volume_rollback_is_possible failed\n"); printf( "Expected error for snapshot_rollback with mp rollback failure (results in inconsistent state)\n" ); testcase_rollback( "207", "test", "volume snapshot rollback disabled\n", { "local:snapshotable-disk-1" => "test", "local:snapshotable-disk-2" => "test" }, ); printf("Successful snapshot_rollback with saved vmstate and machine config only in snapshot\n"); testcase_rollback("301", "test", "", { "local:snapshotable-disk-1" => "test" }); printf("Successful snapshot_rollback with saved vmstate and machine config and runningmachine \n"); testcase_rollback("302", "test", "", { "local:snapshotable-disk-1" => "test" }); $activate_storage_possible = 0; printf("Expected error for snapshot_rollback when storage activation is not possible\n"); testcase_rollback("303", "test", "storage activation failed\n"); $activate_storage_possible = 1; done_testing(); 1; ================================================ FILE: src/test/test.vmdk ================================================ ================================================ FILE: src/test/test_get_replicatable_volumes.pl ================================================ #!/usr/bin/perl use strict; use warnings; use lib ('..'); use Data::Dumper; use PVE::Storage; use PVE::QemuConfig; use Test::More; my $storecfg = { ids => { local => { type => 'dir', shared => 0, content => { 'iso' => 1, 'backup' => 1, 'images' => 1, 'rootdir' => 1, }, path => "/var/lib/vz", }, 'local-zfs' => { type => 'zfspool', pool => 'nonexistent-testpool', shared => 0, content => { 'images' => 1, 'rootdir' => 1, }, }, }, }; my $vmid = 900; my $rawconf = "scsi0: non-existent-store:vm-103-disk-1,size=8G\n"; my $conf = PVE::QemuServer::parse_vm_config("/qemu-server/$vmid.conf", $rawconf); my $volumes; my $expect; my $test_name = "test non existent storage"; eval { $volumes = PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 0); }; is($@, "storage 'non-existent-store' does not exist\n", $test_name); $test_name = "test with disk from other VM (not owner)"; $rawconf = "scsi0: local:103/vm-103-disk-1.qcow2,size=8G\n"; $conf = PVE::QemuServer::parse_vm_config("/qemu-server/$vmid.conf", $rawconf); $volumes = PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 0); is_deeply($volumes, {}, $test_name); $test_name = "test missing replicate feature"; $rawconf = "scsi0: local:$vmid/vm-$vmid-disk-1.qcow2,size=8G\n"; $conf = PVE::QemuServer::parse_vm_config("/qemu-server/$vmid.conf", $rawconf); eval { $volumes = PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 0); }; is($@, "missing replicate feature on volume 'local:900/vm-900-disk-1.qcow2'\n", $test_name); $test_name = "test raw path disk with replicate enabled"; $rawconf = "scsi0: /dev/disk/abcdefg,size=8G\n"; $conf = PVE::QemuServer::parse_vm_config("/qemu-server/$vmid.conf", $rawconf); eval { $volumes = PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 0); }; is($@, "unable to replicate local file/device '/dev/disk/abcdefg'\n", $test_name); $test_name = "test raw path disk with replicate disabled"; $rawconf = "scsi0: /dev/disk/abcdefg,size=8G,replicate=0\n"; $conf = PVE::QemuServer::parse_vm_config("/qemu-server/$vmid.conf", $rawconf); $volumes = PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 0); is_deeply($volumes, {}, $test_name); $test_name = "test CDROM with iso file"; $rawconf = "ide2: local:iso/pve-cd.iso,media=cdrom\n"; $conf = PVE::QemuServer::parse_vm_config("/qemu-server/$vmid.conf", $rawconf); $volumes = PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 0); is_deeply($volumes, {}, $test_name); $test_name = "test CDROM with access to physical 'cdrom' device"; $rawconf = "ide2: cdrom,media=cdrom\n"; $conf = PVE::QemuServer::parse_vm_config("/qemu-server/$vmid.conf", $rawconf); $volumes = PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 0); is_deeply($volumes, {}, $test_name); $test_name = "test hidden volid in snapshot"; $rawconf = <<__EOD__; memory: 1024 scsi0: local-zfs:vm-$vmid-disk-2,size=8G [snap1] memory: 512 scsi0: local-zfs:vm-$vmid-disk-1,size=8G __EOD__ $conf = PVE::QemuServer::parse_vm_config("/qemu-server/$vmid.conf", $rawconf); $volumes = PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 0); $expect = { "local-zfs:vm-$vmid-disk-1" => 1, "local-zfs:vm-$vmid-disk-2" => 1, }; is_deeply($volumes, $expect, $test_name); $test_name = "test volid with different replicate setting in snapshot"; $rawconf = <<__EOD__; memory: 1024 scsi0: local-zfs:vm-$vmid-disk-1,size=8G,replicate=0 [snap1] memory: 512 scsi0: local-zfs:vm-$vmid-disk-1,size=8G __EOD__ $conf = PVE::QemuServer::parse_vm_config("/qemu-server/$vmid.conf", $rawconf); $volumes = PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 0); $expect = { "local-zfs:vm-$vmid-disk-1" => 1, }; is_deeply($volumes, $expect, $test_name); $test_name = "test vm with replicatable unused volumes"; $rawconf = <<__EOD__; scsi0: local-zfs:vm-$vmid-disk-1,size=8G unused1: local-zfs:vm-$vmid-disk-2 unused5: local-zfs:vm-$vmid-disk-3 __EOD__ $conf = PVE::QemuServer::parse_vm_config("/qemu-server/$vmid.conf", $rawconf); $volumes = PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 0); $expect = { "local-zfs:vm-$vmid-disk-1" => 1, "local-zfs:vm-$vmid-disk-2" => 1, "local-zfs:vm-$vmid-disk-3" => 1, }; is_deeply($volumes, $expect, $test_name); $test_name = "test vm with non-replicatable unused volumes"; $rawconf = <<__EOD__; scsi0: local-zfs:vm-$vmid-disk-1,size=8G unused1: local:$vmid/vm-$vmid-disk-2.raw __EOD__ $conf = PVE::QemuServer::parse_vm_config("/qemu-server/$vmid.conf", $rawconf); eval { $volumes = PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 0); }; is($@, "missing replicate feature on volume 'local:900/vm-900-disk-2.raw'\n", $test_name); done_testing(); exit(0); ================================================ FILE: src/usr/Makefile ================================================ PACKAGE ?= qemu-server DESTDIR= PREFIX=/usr LIBDIR=$(DESTDIR)/$(PREFIX)/lib LIBEXECDIR=$(DESTDIR)/$(PREFIX)/libexec/$(PACKAGE) LIBSYSTEMDDIR := $(DESTDIR)/usr/lib/systemd DBUSDIR := $(DESTDIR)/usr/share/dbus-1 SHAREDIR=$(DESTDIR)/$(PREFIX)/share/$(PACKAGE) .PHONY: install install: pve-usb.cfg pve-q35.cfg pve-q35-4.0.cfg bootsplash.jpg modules-load.conf pve-bridge pve-bridge-hotplug pve-bridgedown install -d $(SHAREDIR) install -m 0644 pve-usb.cfg $(SHAREDIR) install -m 0644 pve-q35.cfg $(SHAREDIR) install -m 0644 pve-q35-4.0.cfg $(SHAREDIR) install -m 0644 -D bootsplash.jpg $(SHAREDIR) install -D -m 0644 modules-load.conf $(LIBDIR)/modules-load.d/qemu-server.conf install -d $(LIBEXECDIR) install -m 0755 pve-bridge $(LIBEXECDIR)/pve-bridge install -m 0755 pve-bridge-hotplug $(LIBEXECDIR)/pve-bridge-hotplug install -m 0755 pve-bridgedown $(LIBEXECDIR)/pve-bridgedown install -D -m 0755 dbus-vmstate $(LIBEXECDIR)/dbus-vmstate install -d $(LIBSYSTEMDDIR) install -D -m 0644 pve-dbus-vmstate@.service $(LIBSYSTEMDDIR)/system/pve-dbus-vmstate@.service install -d $(DBUSDIR) install -D -m 0644 org.qemu.VMState1.conf $(DBUSDIR)/system.d/org.qemu.VMState1.conf .PHONY: clean clean: ================================================ FILE: src/usr/dbus-vmstate ================================================ #!/usr/bin/perl # Exports an DBus object implementing # https://www.qemu.org/docs/master/interop/dbus-vmstate.html package PVE::QemuServer::DBusVMState; use warnings; use strict; use Carp; use Net::DBus; use Net::DBus::Exporter qw(org.qemu.VMState1); use Net::DBus::Reactor; use PVE::QemuServer::Helpers; use PVE::QemuServer::QMPHelpers qw(qemu_objectadd qemu_objectdel); use PVE::SafeSyslog; use PVE::Systemd; use PVE::Tools; use base qw(Net::DBus::Object); use Class::MethodMaker [ scalar => [ qw(Id NumMigratedEntries) ]]; dbus_property('Id', 'string', 'read'); dbus_property('NumMigratedEntries', 'uint32', 'read', 'com.proxmox.VMStateHelper'); sub new { my ($class, $service, $vmid) = @_; my $self = $class->SUPER::new($service, '/org/qemu/VMState1'); $self->{vmid} = $vmid; $self->Id("pve-vmstate-$vmid"); $self->NumMigratedEntries(0); bless $self, $class; return $self; } sub Load { my ($self, $bytes) = @_; my $len = scalar(@$bytes); return if $len <= 1; # see also the `Save` method my $text = pack('c*', @$bytes); eval { PVE::Tools::run_command( ['conntrack', '--load-file', '-'], input => $text, ); }; if (my $err = $@) { syslog('warn', "failed to restore conntrack state: $err\n"); } else { syslog('info', "restored $len bytes of conntrack state\n"); } } dbus_method('Load', [['array', 'byte']], []); use constant { # From the documentation: # https://www.qemu.org/docs/master/interop/dbus-vmstate.html), # > For now, the data amount to be transferred is arbitrarily limited to 1Mb. # # See also qemu/backends/dbus-vmstate.c:DBUS_VMSTATE_SIZE_LIMIT DBUS_VMSTATE_SIZE_LIMIT => 1024 * 1024, }; sub Save { my ($self) = @_; my $text = ''; my $truncated = 0; my $num_entries = 0; eval { PVE::Tools::run_command( ['conntrack', '--dump', '--mark', $self->{vmid}, '--output', 'save'], outfunc => sub { my ($line) = @_; return if $truncated; if ((length($text) + length($line)) > DBUS_VMSTATE_SIZE_LIMIT) { syslog('warn', 'conntrack state too large, ignoring further entries'); $truncated = 1; return; } # conntrack(8) apparently does not preserve the `--mark` option, # add it back ourselves $text .= "$line --mark $self->{vmid}\n"; }, errfunc => sub { my ($line) = @_; if ($line =~ /(\d) flow entries/) { syslog('info', "received $1 conntrack entries"); # conntrack reports the number of displayed entries on stderr, # which shouldn't be considered an error. $self->NumMigratedEntries($1); return; } syslog('err', $line); } ); }; if (my $err = $@) { syslog('warn', "failed to save conntrack state: $err\n"); # Apparently either Net::DBus does not correctly zero-sized (byte) # arrays correctly - returning [] yields QEMU failing with # # "kvm: dbus_save_state_proxy: Failed to Save: not a byte array" # # Thus, just return an array with a single element and detect that # appropriately in the `Load`. A valid conntrack state can *never* be # just a single byte, so it is safe to rely on that. return [0]; } my @bytes = unpack('c*', $text); my $len = scalar(@bytes); syslog('info', "transferring $len bytes of conntrack state\n"); # Same as above w.r.t. returning as single-element array. return $len == 0 ? [0] : \@bytes; } dbus_method('Save', [], [['array', 'byte']]); # Additional method for cleanly shutting down the service. sub Quit { my ($self) = @_; syslog('info', "shutting down gracefully ..\n"); # On the source side, the VM won't exist anymore, so no need to remove # anything. if (PVE::QemuServer::Helpers::vm_running_locally($self->{vmid})) { eval { qemu_objectdel($self->{vmid}, 'pve-vmstate') }; if (my $err = $@) { syslog('warn', "failed to remove object: $err\n"); } } Net::DBus::Reactor->main()->shutdown(); } dbus_method('Quit', [], [], 'com.proxmox.VMStateHelper', { no_return => 1 }); my $vmid = shift; my $dbus = Net::DBus->system(); my $service = $dbus->export_service('org.qemu.VMState1'); my $obj = PVE::QemuServer::DBusVMState->new($service, $vmid); $SIG{TERM} = sub { $obj->Quit(); }; my $addr = $dbus->get_unique_name(); syslog('info', "pve-vmstate-$vmid listening on $addr\n"); # Inform QEMU about our running dbus-vmstate helper qemu_objectadd($vmid, 'pve-vmstate', 'dbus-vmstate', addr => 'unix:path=/run/dbus/system_bus_socket', 'id-list' => "pve-vmstate-$vmid", ); PVE::Systemd::notify("READY=1\n"); Net::DBus::Reactor->main()->run(); ================================================ FILE: src/usr/modules-load.conf ================================================ vhost_net msr ================================================ FILE: src/usr/org.qemu.VMState1.conf ================================================ ================================================ FILE: src/usr/pve-bridge ================================================ #!/usr/bin/perl use strict; use warnings; use PVE::Tools qw(run_command); use PVE::Network::SDN::Vnets; use PVE::Network::SDN::Zones; use PVE::Firewall; use PVE::QemuServer::Network; my $iface = shift; my $hotplug = 0; if ($iface eq '--hotplug') { $hotplug = 1; $iface = shift; } die "no interface specified\n" if !$iface; die "got strange interface name '$iface'\n" if $iface !~ m/^tap(\d+)i(\d+)$/; my $vmid = $1; my $netid = "net$2"; my $migratedfrom = $hotplug ? undef : $ENV{PVE_MIGRATED_FROM}; my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom); my $netconf = $conf->{$netid}; $netconf = $conf->{pending}->{$netid} if !$migratedfrom && defined($conf->{pending}->{$netid}); die "unable to get network config '$netid'\n" if !defined($netconf); my $net = PVE::QemuServer::Network::parse_net($netconf); die "unable to parse network config '$netid'\n" if !$net; PVE::Network::SDN::Vnets::add_dhcp_mapping($net->{bridge}, $net->{macaddr}, $vmid, $conf->{name}); PVE::Network::SDN::Zones::tap_create($iface, $net->{bridge}); PVE::QemuServer::Network::tap_plug( $iface, $net->{bridge}, $net->{tag}, $net->{firewall}, $net->{trunks}, $net->{rate}, ); exit 0; ================================================ FILE: src/usr/pve-bridge-hotplug ================================================ #!/bin/sh exec /usr/libexec/qemu-server/pve-bridge --hotplug "$@" ================================================ FILE: src/usr/pve-bridgedown ================================================ #!/usr/bin/perl use strict; use warnings; use PVE::Network; my $iface = shift; die "no interface specified\n" if !$iface; die "got strange interface name '$iface'\n" if $iface !~ m/^tap(\d+)i(\d+)$/; PVE::Network::tap_unplug($iface); exit 0; ================================================ FILE: src/usr/pve-dbus-vmstate@.service ================================================ [Unit] Description=PVE DBus VMState Helper (VM %i) Requires=dbus.socket After=dbus.socket PartOf=%i.scope [Service] Slice=qemu.slice Type=notify ExecStart=/usr/libexec/qemu-server/dbus-vmstate %i ================================================ FILE: src/usr/pve-q35-4.0.cfg ================================================ [device "ehci"] driver = "ich9-usb-ehci1" multifunction = "on" bus = "pcie.0" addr = "1d.7" [device "uhci-1"] driver = "ich9-usb-uhci1" multifunction = "on" bus = "pcie.0" addr = "1d.0" masterbus = "ehci.0" firstport = "0" [device "uhci-2"] driver = "ich9-usb-uhci2" multifunction = "on" bus = "pcie.0" addr = "1d.1" masterbus = "ehci.0" firstport = "2" [device "uhci-3"] driver = "ich9-usb-uhci3" multifunction = "on" bus = "pcie.0" addr = "1d.2" masterbus = "ehci.0" firstport = "4" [device "ehci-2"] driver = "ich9-usb-ehci2" multifunction = "on" bus = "pcie.0" addr = "1a.7" [device "uhci-4"] driver = "ich9-usb-uhci4" multifunction = "on" bus = "pcie.0" addr = "1a.0" masterbus = "ehci-2.0" firstport = "0" [device "uhci-5"] driver = "ich9-usb-uhci5" multifunction = "on" bus = "pcie.0" addr = "1a.1" masterbus = "ehci-2.0" firstport = "2" [device "uhci-6"] driver = "ich9-usb-uhci6" multifunction = "on" bus = "pcie.0" addr = "1a.2" masterbus = "ehci-2.0" firstport = "4" # FIXME: Remove this audio0 device at the next possible time # see: https://pve.proxmox.com/pipermail/pve-devel/2019-July/038417.html # https://pve.proxmox.com/pipermail/pve-devel/2019-July/038428.html [device "audio0"] driver = "ich9-intel-hda" bus = "pcie.0" addr = "1b.0" [device "ich9-pcie-port-1"] driver = "pcie-root-port" x-speed = "16" x-width = "32" multifunction = "on" bus = "pcie.0" addr = "1c.0" port = "1" chassis = "1" [device "ich9-pcie-port-2"] driver = "pcie-root-port" x-speed = "16" x-width = "32" multifunction = "on" bus = "pcie.0" addr = "1c.1" port = "2" chassis = "2" [device "ich9-pcie-port-3"] driver = "pcie-root-port" x-speed = "16" x-width = "32" multifunction = "on" bus = "pcie.0" addr = "1c.2" port = "3" chassis = "3" [device "ich9-pcie-port-4"] driver = "pcie-root-port" x-speed = "16" x-width = "32" multifunction = "on" bus = "pcie.0" addr = "1c.3" port = "4" chassis = "4" ## # Example PCIe switch with two downstream ports # #[device "pcie-switch-upstream-port-1"] # driver = "x3130-upstream" # bus = "ich9-pcie-port-4" # addr = "00.0" # #[device "pcie-switch-downstream-port-1-1"] # driver = "xio3130-downstream" # multifunction = "on" # bus = "pcie-switch-upstream-port-1" # addr = "00.0" # port = "1" # chassis = "5" # #[device "pcie-switch-downstream-port-1-2"] # driver = "xio3130-downstream" # multifunction = "on" # bus = "pcie-switch-upstream-port-1" # addr = "00.1" # port = "1" # chassis = "6" [device "pcidmi"] driver = "i82801b11-bridge" bus = "pcie.0" addr = "1e.0" [device "pci.0"] driver = "pci-bridge" bus = "pcidmi" addr = "1.0" chassis_nr = "1" [device "pci.1"] driver = "pci-bridge" bus = "pcidmi" addr = "2.0" chassis_nr = "2" [device "pci.2"] driver = "pci-bridge" bus = "pcidmi" addr = "3.0" chassis_nr = "3" [device "pci.3"] driver = "pci-bridge" bus = "pcidmi" addr = "4.0" chassis_nr = "4" ================================================ FILE: src/usr/pve-q35.cfg ================================================ [device "ehci"] driver = "ich9-usb-ehci1" multifunction = "on" bus = "pcie.0" addr = "1d.7" [device "uhci-1"] driver = "ich9-usb-uhci1" multifunction = "on" bus = "pcie.0" addr = "1d.0" masterbus = "ehci.0" firstport = "0" [device "uhci-2"] driver = "ich9-usb-uhci2" multifunction = "on" bus = "pcie.0" addr = "1d.1" masterbus = "ehci.0" firstport = "2" [device "uhci-3"] driver = "ich9-usb-uhci3" multifunction = "on" bus = "pcie.0" addr = "1d.2" masterbus = "ehci.0" firstport = "4" [device "ehci-2"] driver = "ich9-usb-ehci2" multifunction = "on" bus = "pcie.0" addr = "1a.7" [device "uhci-4"] driver = "ich9-usb-uhci4" multifunction = "on" bus = "pcie.0" addr = "1a.0" masterbus = "ehci-2.0" firstport = "0" [device "uhci-5"] driver = "ich9-usb-uhci5" multifunction = "on" bus = "pcie.0" addr = "1a.1" masterbus = "ehci-2.0" firstport = "2" [device "uhci-6"] driver = "ich9-usb-uhci6" multifunction = "on" bus = "pcie.0" addr = "1a.2" masterbus = "ehci-2.0" firstport = "4" [device "audio0"] driver = "ich9-intel-hda" bus = "pcie.0" addr = "1b.0" [device "ich9-pcie-port-1"] driver = "ioh3420" multifunction = "on" bus = "pcie.0" addr = "1c.0" port = "1" chassis = "1" [device "ich9-pcie-port-2"] driver = "ioh3420" multifunction = "on" bus = "pcie.0" addr = "1c.1" port = "2" chassis = "2" [device "ich9-pcie-port-3"] driver = "ioh3420" multifunction = "on" bus = "pcie.0" addr = "1c.2" port = "3" chassis = "3" [device "ich9-pcie-port-4"] driver = "ioh3420" multifunction = "on" bus = "pcie.0" addr = "1c.3" port = "4" chassis = "4" ## # Example PCIe switch with two downstream ports # #[device "pcie-switch-upstream-port-1"] # driver = "x3130-upstream" # bus = "ich9-pcie-port-4" # addr = "00.0" # #[device "pcie-switch-downstream-port-1-1"] # driver = "xio3130-downstream" # multifunction = "on" # bus = "pcie-switch-upstream-port-1" # addr = "00.0" # port = "1" # chassis = "5" # #[device "pcie-switch-downstream-port-1-2"] # driver = "xio3130-downstream" # multifunction = "on" # bus = "pcie-switch-upstream-port-1" # addr = "00.1" # port = "1" # chassis = "6" [device "pcidmi"] driver = "i82801b11-bridge" bus = "pcie.0" addr = "1e.0" [device "pci.0"] driver = "pci-bridge" bus = "pcidmi" addr = "1.0" chassis_nr = "1" [device "pci.1"] driver = "pci-bridge" bus = "pcidmi" addr = "2.0" chassis_nr = "2" [device "pci.2"] driver = "pci-bridge" bus = "pcidmi" addr = "3.0" chassis_nr = "3" [device "pci.3"] driver = "pci-bridge" bus = "pcidmi" addr = "4.0" chassis_nr = "4" ================================================ FILE: src/usr/pve-usb.cfg ================================================ [device "ehci"] driver = "ich9-usb-ehci1" addr = "1d.7" multifunction = "on" [device "uhci-1"] driver = "ich9-usb-uhci1" addr = "1d.0" multifunction = "on" masterbus = "ehci.0" firstport = "0" [device "uhci-2"] driver = "ich9-usb-uhci2" addr = "1d.1" multifunction = "on" masterbus = "ehci.0" firstport = "2" [device "uhci-3"] driver = "ich9-usb-uhci3" addr = "1d.2" multifunction = "on" masterbus = "ehci.0" firstport = "4"