adding failover protection case for when the mgmt svr crashes in the middle of executing something; along with custom response for cert

core/src/com/cloud/certificate/CertificateVO.java

@@ -39,6 +39,9 @@ public class CertificateVO {
     @Column(name="updated")
     private String updated;
 
+	@Column(name="mgmt_server_id")
+    private Long mgmtServerId;
+
     public CertificateVO() {}
 
     public Long getId() {
@@ -59,4 +62,13 @@ public class CertificateVO {
     public void setUpdated(String updated){
     	this.updated = updated;
     }
+
+
+    public Long getMgmtServerId() {
+		return mgmtServerId;
+	}
+
+	public void setMgmtServerId(Long mgmtServerId) {
+		this.mgmtServerId = mgmtServerId;
+	}
 }

core/src/com/cloud/certificate/dao/CertificateDao.java

@@ -22,5 +22,5 @@ import com.cloud.certificate.CertificateVO;
 import com.cloud.utils.db.GenericDao;
 
 public interface CertificateDao extends GenericDao<CertificateVO, Long> {
-	public Long persistCustomCertToDb(String certPath, CertificateVO cert);
+	public Long persistCustomCertToDb(String certPath, CertificateVO cert, Long managementServerId);
 }

core/src/com/cloud/certificate/dao/CertificateDaoImpl.java

@@ -24,7 +24,7 @@ public class CertificateDaoImpl extends GenericDaoBase<CertificateVO, Long>  imp
     }
     
 	@Override
-	public Long persistCustomCertToDb(String certPath, CertificateVO cert){		
+	public Long persistCustomCertToDb(String certPath, CertificateVO cert, Long managementServerId){		
 	    BufferedInputStream f = null;
 		String certStr = null;
 		try 
@@ -34,7 +34,8 @@ public class CertificateDaoImpl extends GenericDaoBase<CertificateVO, Long>  imp
 	        f.read(buffer);
 	        certStr = new String(buffer);
         	cert.setCertificate(certStr);
-        	cert.setUpdated("t");
+        	cert.setUpdated("Y");
+        	cert.setMgmtServerId(managementServerId);
         	update(cert.getId(),cert);
         	return cert.getId();
 	    } catch (FileNotFoundException e) {

server/src/com/cloud/api/response/CustomCertificateResponse.java

@@ -0,0 +1,36 @@
+/**
+ *  Copyright (C) 2010 Cloud.com, Inc.  All rights reserved.
+ * 
+ * This software is licensed under the GNU General Public License v3 or later.
+ * 
+ * It is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * 
+ */
+package com.cloud.api.response;
+
+import com.cloud.serializer.Param;
+import com.google.gson.annotations.SerializedName;
+
+public class CustomCertificateResponse extends BaseResponse {
+
+    @SerializedName("updatedconsoleproxyidlist") @Param(description="the list of the console proxy ids which were successfully updated")
+    private String updatedConsoleProxyIdList;
+
+	public String getUpdatedConsoleProxyIdList() {
+		return updatedConsoleProxyIdList;
+	}
+
+	public void setUpdatedConsoleProxyIdList(String updatedConsoleProxyIdList) {
+		this.updatedConsoleProxyIdList = updatedConsoleProxyIdList;
+	}
+
+}

server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java

@@ -268,6 +268,9 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, VirtualMach
     @Inject
     private VmManager _vmMgr;
     
+    @Inject
+    private ClusterManager _clMgr;
+
     private final ScheduledExecutorService _capacityScanScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("CP-Scan"));
     private final ExecutorService _requestHandlerScheduler = Executors.newCachedThreadPool(new NamedThreadFactory("Request-handler"));
 
@@ -2360,11 +2363,27 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, VirtualMach
         
         _capacityScanScheduler.scheduleAtFixedRate(getCapacityScanTask(), STARTUP_DELAY, _capacityScanInterval, TimeUnit.MILLISECONDS);
         
+		//cert job cleanup
+		cleanupCertTable(_clMgr.getId());
+
         if (s_logger.isInfoEnabled())
             s_logger.info("Console Proxy Manager is configured.");
         return true;
     }
 
+    private void cleanupCertTable(Long mServerId){
+    	CertificateVO cert = _certDao.listAll().get(0);//always 1 record in db
+    	Long mgmtSvrIdForCertJob = null;
+    	if(cert!=null){
+    		mgmtSvrIdForCertJob = cert.getMgmtServerId();
+    	}
+		if(mgmtSvrIdForCertJob!=null && mgmtSvrIdForCertJob.longValue() == (_clMgr.getId())){
+			CertificateVO lockedCert = _certDao.acquire(cert.getId());
+			lockedCert.setMgmtServerId(null);
+			_certDao.release(lockedCert.getId());
+		}
+    }
+    
     @Override
     public boolean destroyConsoleProxy(DestroyConsoleProxyCmd cmd) throws ServerApiException{
         Long proxyId = cmd.getId();
@@ -2448,7 +2467,7 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, VirtualMach
 		//get cert from db
 		CertificateVO cert = _certDao.listAll().get(0);
 		
-		if(cert.getUpdated().equals("t")){
+		if(cert.getUpdated().equalsIgnoreCase("Y")){
 			String certStr = cert.getCertificate(); 
 			long proxyVmId = (cmd).getProxyVmId();
 			ConsoleProxyVO consoleProxy = _consoleProxyDao.findById(proxyVmId);

server/src/com/cloud/server/ManagementServerImpl.java

@@ -5871,8 +5871,9 @@ public class ManagementServerImpl implements ManagementServer {
     	try 
     	{
     		CertificateVO cert = _certDao.listAll().get(0); //always 1 record in db
-    		
+    		if(cert.getMgmtServerId()!=null)
-    		if(cert.getUpdated().equals("t")){
+    			throw new ServerApiException(BaseCmd.CUSTOM_CERT_UPDATE_ERROR, "Another management server is in the process of custom cert updating");
+    		if(cert.getUpdated().equalsIgnoreCase("Y")){
 				 if(s_logger.isDebugEnabled())
 					 s_logger.debug("A custom certificate already exists in the DB, will replace it with the new one being uploaded");
 			}else{
@@ -5881,7 +5882,9 @@ public class ManagementServerImpl implements ManagementServer {
 			}
 			String certificatePath = cmd.getPath();
 			CertificateVO lockedCert = _certDao.acquire(cert.getId());
-			Long certVOId = _certDao.persistCustomCertToDb(certificatePath,lockedCert);//0 implies failure
+			//assigned mgmt server id to mark as processing under this ms
+			Long certVOId = _certDao.persistCustomCertToDb(certificatePath,lockedCert,this.getId());//0 implies failure
+			_certDao.release(lockedCert.getId());
 			
 			if (certVOId!=null && certVOId!=0) 
 			{
@@ -5928,7 +5931,9 @@ public class ManagementServerImpl implements ManagementServer {
 					}	
 				}
 
-				_certDao.release(lockedCert.getId());
+				CertificateVO lockedCertPostPatching = _certDao.acquire(cert.getId());
+				lockedCertPostPatching.setMgmtServerId(null);//release for other ms
+				_certDao.release(lockedCertPostPatching.getId());
 				return ("Updated:"+updatedCpIdList.size()+" out of:"+cpList.size()+" console proxies");
 			}
 			else

setup/db/create-schema.sql

@@ -135,10 +135,11 @@ CREATE TABLE `cloud`.`certificate` (
   `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
   `certificate` text COMMENT 'the actual custom certificate being stored in the db',
   `updated` varchar(1) COMMENT 'status of the certificate',
+  `mgmt_server_id` bigint unsigned DEFAULT NULL COMMENT 'management server instance id',
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
-INSERT INTO `cloud`.`certificate` (id,certificate,updated) VALUES ('1',null,'f');
+INSERT INTO `cloud`.`certificate` (id,certificate,updated) VALUES ('1',null,'N');
 
 CREATE TABLE `cloud`.`nics` (
   `id` bigint unsigned NOT NULL UNIQUE AUTO_INCREMENT COMMENT 'id',