diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 60e6bcffeb6..fef5d4aa6de 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -85,7 +85,6 @@ import org.libvirt.DomainInfo; import org.libvirt.DomainInfo.DomainState; import org.libvirt.DomainInterfaceStats; import org.libvirt.DomainSnapshot; -import org.libvirt.Library; import org.libvirt.LibvirtException; import org.libvirt.MemoryStatistic; import org.libvirt.Network; @@ -3694,20 +3693,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } private void setupLibvirtEventListener() { - final Thread libvirtListenerThread = new Thread(() -> { - try { - Library.runEventLoop(); - } catch (LibvirtException e) { - s_logger.error("LibvirtException was thrown in event loop: ", e); - } catch (InterruptedException e) { - s_logger.error("Libvirt event loop was interrupted: ", e); - } - }); - try { - libvirtListenerThread.setDaemon(true); - libvirtListenerThread.start(); - Connect conn = LibvirtConnection.getConnection(); conn.addLifecycleListener(this::onDomainLifecycleChange); @@ -3727,7 +3713,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv * Checking for this helps us differentiate between events where cloudstack or admin stopped the VM vs guest * initiated, and avoid pushing extra updates for actions we are initiating without a need for extra tracking */ DomainEventDetail detail = domainEvent.getDetail(); - if (StoppedDetail.SHUTDOWN.equals(detail) || StoppedDetail.CRASHED.equals(detail)) { + if (StoppedDetail.SHUTDOWN.equals(detail) || StoppedDetail.CRASHED.equals(detail) || StoppedDetail.FAILED.equals(detail)) { s_logger.info("Triggering out of band status update due to completed self-shutdown or crash of VM"); _agentStatusUpdater.triggerUpdate(); } else { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java index 7563f964759..1f4ab23a43f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java @@ -23,6 +23,7 @@ import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; import org.apache.log4j.Logger; import org.libvirt.Connect; +import org.libvirt.Library; import org.libvirt.LibvirtException; import com.cloud.hypervisor.Hypervisor; @@ -34,6 +35,7 @@ public class LibvirtConnection { static private Connect s_connection; static private String s_hypervisorURI; + static private Thread libvirtEventThread; static public Connect getConnection() throws LibvirtException { return getConnection(s_hypervisorURI); @@ -45,6 +47,8 @@ public class LibvirtConnection { if (conn == null) { s_logger.info("No existing libvirtd connection found. Opening a new one"); + + setupEventListener(); conn = new Connect(hypervisorURI, false); s_logger.debug("Successfully connected to libvirt at: " + hypervisorURI); s_connections.put(hypervisorURI, conn); @@ -53,7 +57,15 @@ public class LibvirtConnection { conn.getVersion(); } catch (LibvirtException e) { s_logger.error("Connection with libvirtd is broken: " + e.getMessage()); + + try { + conn.close(); + } catch (LibvirtException closeEx) { + s_logger.debug("Ignoring error while trying to close broken connection:" + closeEx.getMessage()); + } + s_logger.debug("Opening a new libvirtd connection to: " + hypervisorURI); + setupEventListener(); conn = new Connect(hypervisorURI, false); s_connections.put(hypervisorURI, conn); } @@ -101,4 +113,33 @@ public class LibvirtConnection { return "qemu:///system"; } + + /** + * Set up Libvirt event handling and polling. This is not specific to a connection object instance, but needs + * to be done prior to creating connections. See the Libvirt documentation for virEventRegisterDefaultImpl and + * virEventRunDefaultImpl or the libvirt-java Library Javadoc for more information. + * @throws LibvirtException + */ + private static synchronized void setupEventListener() throws LibvirtException { + if (libvirtEventThread == null || !libvirtEventThread.isAlive()) { + // Registers a default event loop, must be called before connecting to hypervisor + Library.initEventLoop(); + libvirtEventThread = new Thread(() -> { + while (true) { + try { + // This blocking call contains a loop of its own that will process events until the event loop is stopped or exception is thrown. + Library.runEventLoop(); + } catch (LibvirtException e) { + s_logger.error("LibvirtException was thrown in event loop: ", e); + } catch (InterruptedException e) { + s_logger.error("Libvirt event loop was interrupted: ", e); + } + } + }); + + // Process events in separate thread. Failure to run event loop regularly will cause connections to close due to keepalive timeout. + libvirtEventThread.setDaemon(true); + libvirtEventThread.start(); + } + } }