@dannysauer: Thanks so much for showing me where to look. Just as a reminder, I have a two node Pacemaker/DRBD/KVM cluster, both built at the same time, starting with Leap 15.3… I found something VERY interesting… you’re gonna love this!
Both nodes have been upgraded to Leap 15.5, using the off-line method as recommended, from 15.3 to 15.4 to 15.5; this was done over time, upgrading other systems first to determine stability of each of the platforms before upgrading this cluster. Prior to 15.5, each node would successfully start and maintain the guests, as part of the natural fail-over process that I’ve configured. With the 15.5 upgrade, “Node 1”, let’s call it, still successfully mounted VM’s. Node 2, on the other hand, would no longer do that.
Looking at the /etc/apparmor.d/usr.sbin.libvirtd file you mentioned, I have rather drastic differences. Here is the content of the working node:
#include <tunables/global>
@{LIBVIRT}="libvirt"
profile libvirtd /usr/sbin/libvirtd flags=(attach_disconnected) {
#include <abstractions/base>
#include <abstractions/dbus>
capability kill,
capability net_admin,
capability net_raw,
capability setgid,
capability sys_admin,
capability sys_module,
capability sys_ptrace,
capability sys_pacct,
capability sys_nice,
capability sys_chroot,
capability setuid,
capability dac_override,
capability dac_read_search,
capability fowner,
capability chown,
capability setpcap,
capability mknod,
capability fsetid,
capability audit_write,
capability ipc_lock,
capability sys_rawio,
capability bpf,
capability perfmon,
# Needed for vfio
capability sys_resource,
mount options=(rw,rslave) -> /,
mount options=(rw, nosuid) -> /{var/,}run/libvirt/qemu/*.dev/,
umount /{var/,}run/libvirt/qemu/*.dev/,
umount /dev/,
# libvirt provides any mounts under /dev to qemu namespaces
mount options=(rw, move) /dev/ -> /{,var/}run/libvirt/qemu/*.dev/,
mount options=(rw, move) /dev/** -> /{,var/}run/libvirt/qemu/*{,/},
mount options=(rw, move) /{,var/}run/libvirt/qemu/*.dev/ -> /dev/,
mount options=(rw, move) /{,var/}run/libvirt/qemu/*{,/} -> /dev/**,
network inet stream,
network inet dgram,
network inet6 stream,
network inet6 dgram,
network netlink raw,
network packet dgram,
network packet raw,
# for --p2p migrations
unix (send, receive) type=stream addr=none peer=(label=unconfined addr=none),
ptrace (read,trace) peer=unconfined,
ptrace (read,trace) peer=@{profile_name},
ptrace (read,trace) peer=dnsmasq,
ptrace (read,trace) peer=/usr/sbin/dnsmasq,
ptrace (read,trace) peer=libvirt-*,
ptrace (read,trace) peer=swtpm,
signal (send) peer=dnsmasq,
signal (send) peer=/usr/sbin/dnsmasq,
signal (read, send) peer=libvirt-*,
signal (send) set=("kill", "term") peer=unconfined,
# For communication/control to qemu-bridge-helper
unix (send, receive) type=stream addr=none peer=(label=libvirtd//qemu_bridge_helper),
signal (send) set=("term") peer=libvirtd//qemu_bridge_helper,
# allow connect with openGraphicsFD, direction reversed in newer versions
unix (send, receive) type=stream addr=none peer=(label=libvirt-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*),
# unconfined also required if guests run without security module
unix (send, receive) type=stream addr=none peer=(label=unconfined),
# required if guests run unconfined seclabel type='none' but libvirtd is confined
signal (read, send) peer=unconfined,
# Very lenient profile for libvirtd since we want to first focus on confining
# the guests. Guests will have a very restricted profile.
/ r,
/** rwmkl,
/bin/* PUx,
/sbin/* PUx,
/usr/bin/* PUx,
/usr/sbin/virtlogd pix,
/usr/sbin/* PUx,
/{usr/,}lib/udev/scsi_id PUx,
/usr/{lib,lib64}/xen-common/bin/xen-toolstack PUx,
/usr/{lib,lib64,libexec}/xen/bin/* Ux,
/usr/{lib,libexec}/xen-*/bin/libxl-save-helper PUx,
/usr/{lib,libexec}/xen-*/bin/pygrub PUx,
/usr/{lib,lib64,lib/qemu,libexec}/vhost-user-gpu PUx,
/usr/{lib,lib64,lib/qemu,libexec}/virtiofsd PUx,
# Required by nwfilter_ebiptables_driver.c:ebiptablesWriteToTempFile() to
# read and run an ebtables script.
/var/lib/libvirt/virtd* ixr,
# force the use of virt-aa-helper
audit deny /{usr/,}sbin/apparmor_parser rwxl,
audit deny /etc/apparmor.d/libvirt/** wxl,
audit deny /sys/kernel/security/apparmor/features rwxl,
audit deny /sys/kernel/security/apparmor/matching rwxl,
audit deny /sys/kernel/security/apparmor/.* rwxl,
/sys/kernel/security/apparmor/profiles r,
/usr/lib64/libvirt/* PUxr,
/usr/lib64/libvirt/libvirt_parthelper ix,
/usr/lib64/libvirt/libvirt_iohelper ix,
/etc/libvirt/hooks/** rmix,
/etc/xen/scripts/** rmix,
# allow changing to our UUID-based named profiles
change_profile -> @{LIBVIRT}-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*,
/usr/{lib,lib64,lib/qemu,libexec}/qemu-bridge-helper Cx -> qemu_bridge_helper,
# child profile for bridge helper process
profile qemu_bridge_helper {
#include <abstractions/base>
capability setuid,
capability setgid,
capability setpcap,
capability net_admin,
network inet stream,
# For communication/control from libvirtd
unix (send, receive) type=stream addr=none peer=(label=libvirtd),
signal (receive) set=("term") peer=/usr/sbin/libvirtd,
signal (receive) set=("term") peer=libvirtd,
/dev/net/tun rw,
/etc/qemu/** r,
owner @{PROC}/*/status r,
/usr/{lib,lib64,lib/qemu,libexec}/qemu-bridge-helper rmix,
}
# Site-specific additions and overrides. See local/README for details.
include if exists <local/usr.sbin.libvirtd>
}
Notice that the mount and umount commands match what you pointed out, and yet this unit works! Very strange… The date stamp of this file is 5/28/24; the upgrade was performed 6/7/24, so the creation date matches that.
Now, the failing node looks like this:
# Last Modified: Mon Apr 5 15:03:58 2010
#include <tunables/global>
@{LIBVIRT}="libvirt"
profile libvirtd /usr/sbin/libvirtd flags=(attach_disconnected) {
#include <abstractions/base>
#include <abstractions/dbus>
capability kill,
capability net_admin,
capability net_raw,
capability setgid,
capability sys_admin,
capability sys_module,
capability sys_ptrace,
capability sys_pacct,
capability sys_nice,
capability sys_chroot,
capability setuid,
capability dac_override,
capability dac_read_search,
capability fowner,
capability chown,
capability setpcap,
capability mknod,
capability fsetid,
capability audit_write,
capability ipc_lock,
# Needed for vfio
capability sys_resource,
mount options=(rw,rslave) -> /,
mount options=(rw, nosuid) -> /{var/,}run/libvirt/qemu/*.dev/,
# libvirt provides any mounts under /dev to qemu namespaces
mount options=(rw, move) /dev/ -> /{,var/}run/libvirt/qemu/*.dev/,
mount options=(rw, move) /dev/** -> /{,var/}run/libvirt/qemu/*{,/},
mount options=(rw, move) /{,var/}run/libvirt/qemu/*.dev/ -> /dev/,
mount options=(rw, move) /{,var/}run/libvirt/qemu/*{,/} -> /dev/**,
network inet stream,
network inet dgram,
network inet6 stream,
network inet6 dgram,
network netlink raw,
network packet dgram,
network packet raw,
# for --p2p migrations
unix (send, receive) type=stream addr=none peer=(label=unconfined addr=none),
ptrace (read,trace) peer=unconfined,
ptrace (read,trace) peer=@{profile_name},
ptrace (read,trace) peer=dnsmasq,
ptrace (read,trace) peer=/usr/sbin/dnsmasq,
ptrace (read,trace) peer=libvirt-*,
signal (send) peer=dnsmasq,
signal (send) peer=/usr/sbin/dnsmasq,
signal (read, send) peer=libvirt-*,
signal (send) set=("kill", "term") peer=unconfined,
# For communication/control to qemu-bridge-helper
unix (send, receive) type=stream addr=none peer=(label=/usr/sbin/libvirtd//qemu_bridge_helper),
signal (send) set=("term") peer=/usr/sbin/libvirtd//qemu_bridge_helper,
# allow connect with openGraphicsFD, direction reversed in newer versions
unix (send, receive) type=stream addr=none peer=(label=libvirt-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*),
# unconfined also required if guests run without security module
unix (send, receive) type=stream addr=none peer=(label=unconfined),
# required if guests run unconfined seclabel type='none' but libvirtd is confined
signal (read, send) peer=unconfined,
# Very lenient profile for libvirtd since we want to first focus on confining
# the guests. Guests will have a very restricted profile.
/ r,
/** rwmkl,
/bin/* PUx,
/sbin/* PUx,
/usr/bin/* PUx,
/usr/sbin/virtlogd pix,
/usr/sbin/* PUx,
/{usr/,}lib/udev/scsi_id PUx,
/usr/{lib,lib64}/xen-common/bin/xen-toolstack PUx,
/usr/{lib,lib64}/xen/bin/* Ux,
/usr/lib/xen-*/bin/libxl-save-helper PUx,
# Required by nwfilter_ebiptables_driver.c:ebiptablesWriteToTempFile() to
# read and run an ebtables script.
/var/lib/libvirt/virtd* ixr,
# force the use of virt-aa-helper
audit deny /{usr/,}sbin/apparmor_parser rwxl,
audit deny /etc/apparmor.d/libvirt/** wxl,
audit deny /sys/kernel/security/apparmor/features rwxl,
audit deny /sys/kernel/security/apparmor/matching rwxl,
audit deny /sys/kernel/security/apparmor/.* rwxl,
/sys/kernel/security/apparmor/profiles r,
/usr/{lib,lib64}/libvirt/* PUxr,
/usr/{lib,lib64}/libvirt/libvirt_parthelper ix,
/usr/{lib,lib64}/libvirt/libvirt_iohelper ix,
/etc/libvirt/hooks/** rmix,
/etc/xen/scripts/** rmix,
# allow changing to our UUID-based named profiles
change_profile -> @{LIBVIRT}-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*,
/usr/{lib,lib64,lib/qemu,libexec}/qemu-bridge-helper Cx -> qemu_bridge_helper,
# child profile for bridge helper process
profile qemu_bridge_helper {
#include <abstractions/base>
capability setuid,
capability setgid,
capability setpcap,
capability net_admin,
network inet stream,
# For communication/control from libvirtd
unix (send, receive) type=stream addr=none peer=(label=/usr/sbin/libvirtd),
signal (receive) set=("term") peer=/usr/sbin/libvirtd,
signal (receive) set=("term") peer=libvirtd,
/dev/net/tun rw,
/etc/qemu/** r,
owner @{PROC}/*/status r,
/usr/{lib,lib64,lib/qemu,libexec}/qemu-bridge-helper rmix,
}
}
Notice that there are NO umount commands! There are other subtle differences as well, references to 32-bit mode vs. 64-bit mode, and some other components are missing that were introduced over time by the developers (my educated guess). The date stamp of this file is 10/29/2021. The creation date is 3/29/21, obviously from an older release of Leap. I plan to copy the file to the failing node to see what happens. Of course, I have to call into question what other files may be out of date that have produced this environment, and potentially rebuild this node from scratch. I will follow up shortly with my test results.
. It’s curious that it’s trying to move the mount instead of doing a bind/rbind, but I’m not a deep enough expert in what libvirt is doing there to say it’s wrong. 
