resolve sanity check last id file acces problems (#6825)

Co-authored-by: Suresh Kumar Anaparti <sureshkumar.anaparti@gmail.com>

debian/cloudstack-usage.postinst

@@ -41,6 +41,13 @@ case "$1" in
             ln -s /etc/cloudstack/management/key /etc/cloudstack/usage/key
         fi
 
+        # creating the last sanity checked id pointer
+        mkdir -p /usr/local/libexec
+        if [ ! -f "/usr/local/libexec/sanity-check-last-id" ]; then
+            echo 1 > /usr/local/libexec/sanity-check-last-id
+        fi
+        chown cloud:cloud /usr/local/libexec/sanity-check-last-id
+
         # Print help message
         if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then
             acs_version=$(dpkg -l |grep cloudstack-usage |head -n1 |awk '{print $3}')

packaging/centos7/cloud.spec

@@ -552,6 +552,12 @@ if [ ! -f "%{_sysconfdir}/%{name}/usage/key" ]; then
     ln -s %{_sysconfdir}/%{name}/management/key %{_sysconfdir}/%{name}/usage/key
 fi
 
+mkdir -p /usr/local/libexec
+if [ ! -f "/usr/local/libexec/sanity-check-last-id" ]; then
+    echo 1 > /usr/local/libexec/sanity-check-last-id
+fi
+chown cloud:cloud /usr/local/libexec/sanity-check-last-id
+
 %posttrans usage
 # Print help message
 if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then
@@ -689,6 +695,9 @@ pip3 install --upgrade urllib3
 %attr(0755,root,root) %{_bindir}/cloudstack-setup-baremetal
 
 %changelog
+* Fri Oct 14 2022 Daan Hoogland <daan.hoogland@gmail.com> 4.18.0
+- initialising sanity check pointer file
+
 * Thu Apr 30 2015 Rohit Yadav <bhaisaab@apache.org> 4.6.0
 - Remove awsapi package
 

packaging/centos8/cloud.spec

@@ -543,6 +543,12 @@ if [ ! -f "%{_sysconfdir}/%{name}/usage/key" ]; then
     ln -s %{_sysconfdir}/%{name}/management/key %{_sysconfdir}/%{name}/usage/key
 fi
 
+mkdir -p /usr/local/libexec
+if [ ! -f "/usr/local/libexec/sanity-check-last-id" ]; then
+    echo 1 > /usr/local/libexec/sanity-check-last-id
+fi
+chown cloud:cloud /usr/local/libexec/sanity-check-last-id
+
 %posttrans usage
 # Print help message
 if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then
@@ -677,6 +683,9 @@ pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
 %attr(0755,root,root) %{_bindir}/cloudstack-setup-baremetal
 
 %changelog
+* Fri Oct 14 2022 Daan Hoogland <daan.hoogland@gmail.com> 4.18.0
+- initialising sanity check pointer file
+
 * Thu Apr 30 2015 Rohit Yadav <bhaisaab@apache.org> 4.6.0
 - Remove awsapi package
 

packaging/suse15/cloud.spec

@@ -537,6 +537,12 @@ if [ ! -f "%{_sysconfdir}/%{name}/usage/key" ]; then
     ln -s %{_sysconfdir}/%{name}/management/key %{_sysconfdir}/%{name}/usage/key
 fi
 
+mkdir -p /usr/local/libexec
+if [ ! -f "/usr/local/libexec/sanity-check-last-id" ]; then
+    echo 1 > /usr/local/libexec/sanity-check-last-id
+fi
+chown cloud:cloud /usr/local/libexec/sanity-check-last-id
+
 %posttrans usage
 # Print help message
 if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then
@@ -671,6 +677,9 @@ pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
 %attr(0755,root,root) %{_bindir}/cloudstack-setup-baremetal
 
 %changelog
+* Fri Oct 14 2022 Daan Hoogland <daan.hoogland@gmail.com> 4.18.0
+- initialising sanity check pointer file
+
 * Tue Jun 29 2021 David Jumani <dj.davidjumani1994@gmail.com> 4.16.0
 - Adding SUSE 15 support
 

usage/src/main/java/com/cloud/usage/UsageManagerImpl.java

@@ -2156,6 +2156,7 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
     private class SanityCheck extends ManagedContextRunnable {
         @Override
         protected void runInContext() {
+            s_logger.info("running sanity check");
             UsageSanityChecker usc = new UsageSanityChecker();
             try {
                 String errors = usc.runSanityCheck();

usage/src/main/java/com/cloud/usage/UsageSanityChecker.java

@@ -51,7 +51,7 @@ public class UsageSanityChecker {
 
     protected void reset() {
         errors = new StringBuilder();
-        checkCases = new ArrayList<CheckCase>();
+        checkCases = new ArrayList<>();
     }
 
     protected boolean checkItemCountByPstmt() throws SQLException {
@@ -69,48 +69,51 @@ public class UsageSanityChecker {
         /*
          * Check for item usage records which are created after it is removed
          */
-        try (PreparedStatement pstmt = conn.prepareStatement(checkCase.sqlTemplate)) {
+        try (PreparedStatement pstmt = conn.prepareStatement(checkCase.getSqlTemplate())) {
-            if(checkCase.checkId) {
+            if (checkCase.isCheckId()) {
+                if (lastId > 0) {
                     pstmt.setInt(1, lastId);
+                }
+                if (maxId > lastId) {
                     pstmt.setInt(2, maxId);
                 }
-            try(ResultSet rs = pstmt.executeQuery();) {
-                if (rs.next() && (rs.getInt(1) > 0)) {
-                    errors.append(String.format("Error: Found %s %s\n", rs.getInt(1), checkCase.itemName));
-                    checkOk = false;
-                }
-            }catch (Exception e)
-            {
-                s_logger.error("checkItemCountByPstmt:Exception:"+e.getMessage());
-                throw new CloudRuntimeException("checkItemCountByPstmt:Exception:"+e.getMessage(),e);
             }
+            checkOk = isCheckOkForPstmt(checkCase, checkOk, pstmt);
         }
         catch (Exception e)
         {
-            s_logger.error("checkItemCountByPstmt:Exception:"+e.getMessage());
+            throwPreparedStatementExcecutionException("preparing statement", checkCase.getSqlTemplate(), e);
-            throw new CloudRuntimeException("checkItemCountByPstmt:Exception:"+e.getMessage(),e);
         }
         return checkOk;
     }
 
+    private boolean isCheckOkForPstmt(CheckCase checkCase, boolean checkOk, PreparedStatement pstmt) {
+        try (ResultSet rs = pstmt.executeQuery();) {
+            if (rs.next() && (rs.getInt(1) > 0)) {
+                errors.append(String.format("Error: Found %s %s%n", rs.getInt(1), checkCase.getItemName()));
+                checkOk = false;
+            }
+        } catch (Exception e)
+        {
+            throwPreparedStatementExcecutionException("check is failing", pstmt.toString(), e);
+        }
+        return checkOk;
+    }
+
+    private static void throwPreparedStatementExcecutionException(String msgPrefix, String stmt, Exception e) {
+        String msg = String.format("%s for prepared statement \"%s\" reason: %s", msgPrefix, stmt, e.getMessage());
+        s_logger.error(msg);
+        throw new CloudRuntimeException(msg, e);
+    }
+
     protected void checkMaxUsage() throws SQLException {
         int aggregationRange = DEFAULT_AGGREGATION_RANGE;
-        try (PreparedStatement pstmt = conn.prepareStatement(
+        String sql = "SELECT value FROM `cloud`.`configuration` WHERE name = 'usage.stats.job.aggregation.range'";
-                "SELECT value FROM `cloud`.`configuration` where name = 'usage.stats.job.aggregation.range'");)
+        try (PreparedStatement pstmt = conn.prepareStatement(sql);)
         {
-            try(ResultSet rs = pstmt.executeQuery();) {
+            aggregationRange = getAggregationRange(aggregationRange, pstmt);
-               if (rs.next()) {
-                    aggregationRange = rs.getInt(1);
-                } else {
-                    s_logger.debug("Failed to retrieve aggregation range. Using default : " + aggregationRange);
-                }
-            }catch (SQLException e) {
-                s_logger.error("checkMaxUsage:Exception:"+e.getMessage());
-                throw new CloudRuntimeException("checkMaxUsage:Exception:"+e.getMessage());
-            }
         } catch (SQLException e) {
-            s_logger.error("checkMaxUsage:Exception:"+e.getMessage());
+            throwPreparedStatementExcecutionException("preparing atatement", sql, e);
-            throw new CloudRuntimeException("checkMaxUsage:Exception:"+e.getMessage());
         }
         int aggregationHours = aggregationRange / 60;
 
@@ -120,6 +123,21 @@ public class UsageSanityChecker {
                 lastCheckId);
     }
 
+    private static int getAggregationRange(int aggregationRange, PreparedStatement pstmt) {
+        try (ResultSet rs = pstmt.executeQuery();) {
+           if (rs.next()) {
+                aggregationRange = rs.getInt(1);
+            } else {
+               if (s_logger.isDebugEnabled()) {
+                   s_logger.debug("Failed to retrieve aggregation range. Using default : " + aggregationRange);
+               }
+            }
+        } catch (SQLException e) {
+            throwPreparedStatementExcecutionException("retrieval aggregate value is failing", pstmt.toString(), e);
+        }
+        return aggregationRange;
+    }
+
     protected void checkVmUsage() {
         addCheckCase("select count(*) from cloud_usage.cloud_usage cu inner join cloud.vm_instance vm "
                 + "where vm.type = 'User' and cu.usage_type in (1 , 2) "
@@ -170,14 +188,21 @@ public class UsageSanityChecker {
     }
 
     protected void readLastCheckId(){
+        if (s_logger.isDebugEnabled()) {
+            s_logger.debug("reading last checked id for sanity check");
+        }
         try(BufferedReader reader = new BufferedReader(new FileReader(lastCheckFile));) {
             String lastIdText = null;
             lastId = -1;
-            if ((reader != null) && (lastIdText = reader.readLine()) != null) {
+            if ((lastIdText = reader.readLine()) != null) {
                 lastId = Integer.parseInt(lastIdText);
             }
         } catch (Exception e) {
-            s_logger.error("readLastCheckId:Exception:"+e.getMessage(),e);
+            String msg = String.format("error reading the LastCheckId reason: %s", e.getMessage());
+            s_logger.error(msg);
+            s_logger.debug(msg, e);
+        } finally {
+            s_logger.info(String.format("using %d as last checked id to start from in sanity check", lastId));
         }
     }
 
@@ -188,20 +213,23 @@ public class UsageSanityChecker {
             maxId = -1;
             if (rs.next() && (rs.getInt(1) > 0)) {
                 maxId = rs.getInt(1);
+                if (maxId > lastId) {
                     lastCheckId += " and cu.id <= ?";
                 }
+            }
         }catch (Exception e) {
             s_logger.error("readMaxId:"+e.getMessage(),e);
         }
     }
 
     protected void updateNewMaxId() {
+        s_logger.info(String.format("writing %d as the new last id checked", maxId));
         try (FileWriter fstream = new FileWriter(lastCheckFile);
              BufferedWriter out = new BufferedWriter(fstream);
         ){
             out.write("" + maxId);
         } catch (IOException e) {
-            s_logger.error("updateNewMaxId:Exception:"+e.getMessage());
+            s_logger.error(String.format("Exception writing the last checked id: %d reason: %s", maxId, e.getMessage()));
             // Error while writing last check id
         }
     }
@@ -238,13 +266,17 @@ public class UsageSanityChecker {
         return TransactionLegacy.getStandaloneConnection();
     }
 
-    public static void main(String args[]) {
+    /**
+     * usage something like: /usr/bin/java -Xmx2G -cp /usr/share/cloudstack-usage/*:/usr/share/cloudstack-usage/lib/*:/usr/share/cloudstack-mysql-ha/lib/*:/etc/cloudstack/usage:/usr/share/java/mysql-connector-java.jar:/usr/share/cloudstack-common com.cloud.usage.UsageSanityChecker
+     * @param args none
+     */
+    public static void main(String[] args) {
         UsageSanityChecker usc = new UsageSanityChecker();
         String sanityErrors;
         try {
             sanityErrors = usc.runSanityCheck();
             if (sanityErrors.length() > 0) {
-                s_logger.error(sanityErrors.toString());
+                s_logger.error(sanityErrors);
             }
         } catch (SQLException e) {
             e.printStackTrace();
@@ -266,19 +298,43 @@ public class UsageSanityChecker {
  * encapsulating what change for each specific case
  */
 class CheckCase {
-    public String sqlTemplate;
+    public String getSqlTemplate() {
-    public String itemName;
+        return this.sqlTemplate;
-    public boolean checkId = false;
+    }
+
+    public void setSqlTemplate(final String sqlTemplate) {
+        this.sqlTemplate = sqlTemplate;
+    }
+
+    public String getItemName() {
+        return this.itemName;
+    }
+
+    public void setItemName(final String itemName) {
+        this.itemName = itemName;
+    }
+
+    public boolean isCheckId() {
+        return this.checkId;
+    }
+
+    public void setCheckId(final boolean checkId) {
+        this.checkId = checkId;
+    }
+
+    private String sqlTemplate;
+    private String itemName;
+    private boolean checkId = false;
 
     public CheckCase(String sqlTemplate, String itemName, String lastCheckId) {
-        checkId = true;
+        setCheckId(true);
-        this.sqlTemplate = sqlTemplate + lastCheckId;
+        setSqlTemplate(sqlTemplate + lastCheckId);
-        this.itemName = itemName;
+        setItemName(itemName);
     }
 
     public CheckCase(String sqlTemplate, String itemName) {
         checkId = false;
-        this.sqlTemplate = sqlTemplate;
+        setSqlTemplate(sqlTemplate);
-        this.itemName = itemName;
+        setItemName(itemName);
     }
 }