CLOUDSTACK-5515: #cpu ,cpuspeed and ram is set to NULL in usage db(usage_vm_instance table) after vm stop and start Fixed populating usage event details in usage db on vm start/upgrade/dynamic_scale

engine/schema/src/com/cloud/event/dao/UsageEventDaoImpl.java

@@ -54,6 +54,10 @@ public class UsageEventDaoImpl extends GenericDaoBase<UsageEventVO, Long> implem
     private static final String COPY_ALL_EVENTS =
         "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size) "
             + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size FROM cloud.usage_event vmevt WHERE vmevt.id <= ?";
+    private static final String COPY_EVENT_DETAILS = "INSERT INTO cloud_usage.usage_event_details (id, usage_event_id, name, value) "
+            + "SELECT id, usage_event_id, name, value FROM cloud.usage_event_details vmevtDetails WHERE vmevtDetails.usage_event_id > ? and vmevtDetails.usage_event_id <= ? ";
+    private static final String COPY_ALL_EVENT_DETAILS = "INSERT INTO cloud_usage.usage_event_details (id, usage_event_id, name, value) "
+            + "SELECT id, usage_event_id, name, value FROM cloud.usage_event_details vmevtDetails WHERE vmevtDetails.usage_event_id <= ?";
     private static final String MAX_EVENT = "select max(id) from cloud.usage_event where created <= ?";
     @Inject
     protected UsageEventDetailsDao usageEventDetailsDao;
@@ -96,6 +100,7 @@ public class UsageEventDaoImpl extends GenericDaoBase<UsageEventVO, Long> implem
         long recentEventId = getMostRecentEventId();
         long maxEventId = getMaxEventId(endDate);
         TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB);
+        // Copy events from cloud db to usage db
         String sql = COPY_EVENTS;
         if (recentEventId == 0) {
             if (s_logger.isDebugEnabled()) {
@@ -115,12 +120,39 @@ public class UsageEventDaoImpl extends GenericDaoBase<UsageEventVO, Long> implem
             pstmt.setLong(i++, maxEventId);
             pstmt.executeUpdate();
             txn.commit();
-            return findRecentEvents(endDate);
         } catch (Exception ex) {
             txn.rollback();
             s_logger.error("error copying events from cloud db to usage db", ex);
             throw new CloudRuntimeException(ex.getMessage());
         }
+
+        // Copy event details from cloud db to usage db
+        sql = COPY_EVENT_DETAILS;
+        if (recentEventId == 0) {
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug("no recent event date, copying all event detailss");
+            }
+            sql = COPY_ALL_EVENT_DETAILS;
+        }
+
+        pstmt = null;
+        try {
+            txn.start();
+            pstmt = txn.prepareAutoCloseStatement(sql);
+            int i = 1;
+            if (recentEventId != 0) {
+                pstmt.setLong(i++, recentEventId);
+            }
+            pstmt.setLong(i++, maxEventId);
+            pstmt.executeUpdate();
+            txn.commit();
+        } catch (Exception ex) {
+            txn.rollback();
+            s_logger.error("error copying event details from cloud db to usage db", ex);
+            throw new CloudRuntimeException(ex.getMessage());
+        }
+
+        return findRecentEvents(endDate);
     }
 
     @DB

engine/schema/src/com/cloud/event/dao/UsageEventDetailsDao.java

@@ -23,7 +23,6 @@ import com.cloud.event.UsageEventDetailsVO;
 import com.cloud.utils.db.GenericDao;
 
 public interface UsageEventDetailsDao extends GenericDao<UsageEventDetailsVO, Long> {
-    Map<String, String> findDetails(long eventId);
 
     void persist(long eventId, Map<String, String> details);
 

engine/schema/src/com/cloud/event/dao/UsageEventDetailsDaoImpl.java

@@ -16,17 +16,11 @@
 // under the License.
 package com.cloud.event.dao;
 
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 import javax.ejb.Local;
 
-import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
@@ -41,8 +35,6 @@ import com.cloud.utils.db.TransactionLegacy;
 public class UsageEventDetailsDaoImpl extends GenericDaoBase<UsageEventDetailsVO, Long> implements UsageEventDetailsDao {
     public static final Logger s_logger = Logger.getLogger(UsageEventDetailsDaoImpl.class.getName());
 
-    private static final String EVENT_DETAILS_QUERY = "SELECT details.id, details.usage_event_id, details.name, details.value FROM `cloud`.`usage_event_details` details WHERE details.usage_event_id = ?";
-
     protected final SearchBuilder<UsageEventDetailsVO> EventDetailsSearch;
     protected final SearchBuilder<UsageEventDetailsVO> DetailSearch;
 
@@ -79,45 +71,6 @@ public class UsageEventDetailsDaoImpl extends GenericDaoBase<UsageEventDetailsVO
         return findOneBy(sc);
     }
 
-    @Override
-    public Map<String, String> findDetails(long eventId) {
-        Connection conn = null;
-        PreparedStatement pstmt = null;
-        ResultSet resultSet = null;
-        Map<String, String> details = new HashMap<String, String>();
-        try {
-            conn = TransactionLegacy.getStandaloneConnection();
-
-            pstmt = conn.prepareStatement(EVENT_DETAILS_QUERY);
-            pstmt.setLong(1, eventId);
-            resultSet = pstmt.executeQuery();
-
-            while (resultSet.next()) {
-                details.put(resultSet.getString(3), resultSet.getString(4));
-            }
-
-        } catch (SQLException e) {
-            throw new CloudRuntimeException("Error while executing SQL prepared statement", e);
-        } catch (Throwable e) {
-            throw new CloudRuntimeException("Caught: " + e);
-        } finally {
-            if (pstmt != null) {
-                try {
-                    pstmt.close();
-                } catch (SQLException e) {
-                }
-            }
-            if (conn != null) {
-                try {
-                    conn.close();
-                } catch (SQLException e) {
-                }
-            }
-        }
-
-        return details;
-    }
-
     @Override
     public void persist(long eventId, Map<String, String> details) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();

setup/db/db/schema-421to430.sql

@@ -646,6 +646,15 @@ CREATE TABLE `cloud`.`usage_event_details` (
   CONSTRAINT `fk_usage_event_details__usage_event_id` FOREIGN KEY `fk_usage_event_details__usage_event_id`(`usage_event_id`) REFERENCES `usage_event`(`id`) ON DELETE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
+CREATE TABLE `cloud_usage`.`usage_event_details` (
+  `id` bigint unsigned NOT NULL auto_increment,
+  `usage_event_id` bigint unsigned NOT NULL COMMENT 'usage event id',
+  `name` varchar(255) NOT NULL,
+  `value` varchar(1024) NOT NULL,
+  PRIMARY KEY (`id`),
+  CONSTRAINT `fk_usage_event_details__usage_event_id` FOREIGN KEY `fk_usage_event_details__usage_event_id`(`usage_event_id`) REFERENCES `usage_event`(`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
 CREATE TABLE `cloud`.`user_ip_address_details` (
   `id` bigint unsigned NOT NULL auto_increment,
   `user_ip_address_id` bigint unsigned NOT NULL COMMENT 'User ip address id',

usage/src/com/cloud/usage/UsageManagerImpl.java

@@ -44,6 +44,7 @@ import org.apache.cloudstack.usage.UsageTypes;
 import com.cloud.alert.AlertManager;
 import com.cloud.event.EventTypes;
 import com.cloud.event.UsageEventVO;
+import com.cloud.event.UsageEventDetailsVO;
 import com.cloud.event.dao.UsageEventDao;
 import com.cloud.event.dao.UsageEventDetailsDao;
 import com.cloud.usage.dao.UsageDao;
@@ -1067,7 +1068,7 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
                         usageInstance.setServiceOfferingId(soId);
                         usageInstance.setStartDate(event.getCreateDate());
                         usageInstance.setEndDate(null);
-                        _usageInstanceDao.persist(usageInstance);
+                        populateDynamicComputeOfferingDetailsAndPersist(usageInstance, event.getId());
                     }
                 }
 
@@ -1078,7 +1079,7 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
                 UsageVMInstanceVO usageInstanceNew =
                     new UsageVMInstanceVO(UsageTypes.RUNNING_VM, zoneId, event.getAccountId(), vmId, vmName, soId, templateId, hypervisorType, event.getCreateDate(),
                         null);
-                _usageInstanceDao.persist(usageInstanceNew);
+                populateDynamicComputeOfferingDetailsAndPersist(usageInstanceNew, event.getId());
             } catch (Exception ex) {
                 s_logger.error("Error saving usage instance for vm: " + vmId, ex);
             }
@@ -1105,38 +1106,11 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
             try {
                 Long templateId = event.getTemplateId();
                 String hypervisorType = event.getResourceType();
-                Long cpuCores = null;
-                Long memory = null;
-                Long cpuSpeed = null;
-
-                //populate the cpu, memory and cpuSpeed of the vm when created from a dynamic offering.
-                Map<String, String> usageDetails = _usageEventDetailsDao.findDetails(event.getId());
-
-                if (usageDetails != null && usageDetails.size() != 0) {
-                    if (usageDetails.get(UsageEventVO.DynamicParameters.cpuNumber.name()) != null) {
-                        cpuCores = Long.parseLong(usageDetails.get(UsageEventVO.DynamicParameters.cpuNumber.name()));
-                    }
-                    if (usageDetails.get(UsageEventVO.DynamicParameters.cpuSpeed.name()) != null) {
-                        cpuSpeed = Long.parseLong(usageDetails.get(UsageEventVO.DynamicParameters.cpuSpeed.name()));
-                    }
-                    if (usageDetails.get(UsageEventVO.DynamicParameters.memory.name()) != null) {
-                        memory = Long.parseLong(usageDetails.get(UsageEventVO.DynamicParameters.memory.name()));
-                    }
-                }
 
                 // add this VM to the usage helper table
                 UsageVMInstanceVO usageInstanceNew = new UsageVMInstanceVO(UsageTypes.ALLOCATED_VM, zoneId, event.getAccountId(), vmId, vmName,
                     soId, templateId, hypervisorType, event.getCreateDate(), null);
-                if (cpuCores != null) {
+                populateDynamicComputeOfferingDetailsAndPersist(usageInstanceNew, event.getId());
-                    usageInstanceNew.setCpuCores(cpuCores);
-                }
-                if (cpuSpeed != null) {
-                    usageInstanceNew.setCpuSpeed(cpuSpeed);
-                }
-                if (memory != null) {
-                    usageInstanceNew.setMemory(memory);
-                }
-                _usageInstanceDao.persist(usageInstanceNew);
             } catch (Exception ex) {
                 s_logger.error("Error saving usage instance for vm: " + vmId, ex);
             }
@@ -1176,7 +1150,7 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
             // add this VM to the usage helper table
             UsageVMInstanceVO usageInstanceNew =
                 new UsageVMInstanceVO(UsageTypes.ALLOCATED_VM, zoneId, event.getAccountId(), vmId, vmName, soId, templateId, hypervisorType, event.getCreateDate(), null);
-            _usageInstanceDao.persist(usageInstanceNew);
+            populateDynamicComputeOfferingDetailsAndPersist(usageInstanceNew, event.getId());
         } else if (EventTypes.EVENT_VM_DYNAMIC_SCALE.equals(event.getType())) {
             // Ending the running vm event
             SearchCriteria<UsageVMInstanceVO> sc = _usageInstanceDao.createSearchCriteria();
@@ -1211,7 +1185,7 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
                     usageInstance.setServiceOfferingId(soId);
                     usageInstance.setStartDate(event.getCreateDate());
                     usageInstance.setEndDate(null);
-                    _usageInstanceDao.persist(usageInstance);
+                    populateDynamicComputeOfferingDetailsAndPersist(usageInstance, event.getId());
                 }
             }
 
@@ -1221,10 +1195,30 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
             // add this VM to the usage helper table with new service offering Id
             UsageVMInstanceVO usageInstanceNew =
                 new UsageVMInstanceVO(UsageTypes.RUNNING_VM, zoneId, event.getAccountId(), vmId, vmName, soId, templateId, hypervisorType, event.getCreateDate(), null);
-            _usageInstanceDao.persist(usageInstanceNew);
+            populateDynamicComputeOfferingDetailsAndPersist(usageInstanceNew, event.getId());
         }
     }
 
+    private void populateDynamicComputeOfferingDetailsAndPersist(UsageVMInstanceVO usageInstance, Long eventId) {
+
+        //populate the cpu, memory and cpuSpeed of the vm when created from a dynamic offering.
+        UsageEventDetailsVO cpuNumber = _usageEventDetailsDao.findDetail(eventId, UsageEventVO.DynamicParameters.cpuNumber.name());
+        if (cpuNumber != null) {
+            usageInstance.setCpuCores(Long.parseLong(cpuNumber.getValue()));
+        }
+
+        UsageEventDetailsVO cpuSpeed = _usageEventDetailsDao.findDetail(eventId, UsageEventVO.DynamicParameters.cpuSpeed.name());
+        if (cpuSpeed != null) {
+            usageInstance.setCpuSpeed(Long.parseLong(cpuSpeed.getValue()));
+        }
+
+        UsageEventDetailsVO memory = _usageEventDetailsDao.findDetail(eventId, UsageEventVO.DynamicParameters.memory.name());
+        if (memory != null) {
+            usageInstance.setMemory(Long.parseLong(memory.getValue()));
+        }
+        _usageInstanceDao.persist(usageInstance);
+    }
+
     private void createNetworkHelperEntry(UserStatisticsVO userStat, UsageNetworkVO usageNetworkStats, long timestamp) {
         long currentAccountedBytesSent = 0L;
         long currentAccountedBytesReceived = 0L;