Merge branch 'master' of ssh://git.cloud.com/var/lib/git/cloudstack-oss

agent/conf/log4j-cloud.xml.in

@@ -10,7 +10,7 @@
    <!-- A time/date based rolling appender -->
    <appender name="FILE" class="org.apache.log4j.rolling.RollingFileAppender">
       <param name="Append" value="true"/>
-      <param name="Threshold" value="DEBUG"/>
+      <param name="Threshold" value="INFO"/>
       <rollingPolicy class="org.apache.log4j.rolling.TimeBasedRollingPolicy">
         <param name="FileNamePattern" value="@AGENTLOG@.%d{yyyy-MM-dd}.gz"/>
         <param name="ActiveFileName" value="@AGENTLOG@"/>
@@ -38,7 +38,7 @@
    <!-- ================ -->
 
    <category name="com.cloud">
-     <priority value="DEBUG"/>
+     <priority value="INFO"/>
    </category>
    
    <category name="com.cloud.agent.metrics">

agent/distro/ubuntu/SYSCONFDIR/init.d/cloud-agent.in

@@ -1,7 +1,14 @@
 #!/bin/bash
+### BEGIN INIT INFO
+# Provides:          cloud agent
+# Required-Start:    $network 
+# Required-Stop:     $network 
+# Default-Start:     3 4 5
+# Default-Stop:      0 1 2 6
+# X-Interactive:     true
+# Short-Description: Start/stop apache2 web server
+### END INIT INFO
 
-# chkconfig: 35 99 10
-# description: Cloud Agent
 
 # WARNING: if this script is changed, then all other initscripts MUST BE changed to match it as well
 
@@ -23,6 +30,20 @@ unset OPTIONS
 DAEMONIZE=@BINDIR@/@PACKAGE@-daemonize
 PROG=@LIBEXECDIR@/agent-runner
 
+wait_for_network() {
+    i=1
+    while [ $i -lt 10 ]
+    do
+        if ip addr show cloudbr0 |grep -w inet > /dev/null 2>&1; then
+            break
+        else
+            continue
+        fi
+        sleep 1
+        let i=$i+1
+    done
+}
+
 start() {
         log_daemon_msg $"Starting $PROGNAME" "$SHORTNAME"
 	if [ -s "$PIDFILE" ] && kill -0 $(cat "$PIDFILE") >/dev/null 2>&1; then
@@ -38,6 +59,9 @@ start() {
 		exit 1
 	fi
  
+    #FIXME: wait for network
+    wait_for_network
+
 	if start-stop-daemon --start --quiet \
 		--pidfile "$PIDFILE" \
 		--exec "$DAEMONIZE" -- -n "$SHORTNAME" -p "$PIDFILE" -l "$LOGFILE" "$PROG" $OPTIONS

agent/src/com/cloud/agent/resource/computing/LibvirtComputingResource.java

@@ -1545,7 +1545,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
     		s_logger.warn("Failed to program network rules for vm " + cmd.getVmName());
     		return new SecurityIngressRuleAnswer(cmd, false, "programming network rules failed");
     	} else {
-    		s_logger.info("Programmed network rules for vm " + cmd.getVmName() + " guestIp=" + cmd.getGuestIp() + ", numrules=" + cmd.getRuleSet().length);
+    		s_logger.debug("Programmed network rules for vm " + cmd.getVmName() + " guestIp=" + cmd.getGuestIp() + ", numrules=" + cmd.getRuleSet().length);
     		return new SecurityIngressRuleAnswer(cmd);
     	}
     }
@@ -1857,7 +1857,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 		final OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
 		String result = cpuScript.execute(parser);
 		if (result != null) {
-			s_logger.info("Unable to get the host CPU state: " + result);
+			s_logger.debug("Unable to get the host CPU state: " + result);
 			return new Answer(cmd, false, result);
 		}
 		double cpuUtil = (100.0D - Double.parseDouble(parser.getLine()));
@@ -1869,7 +1869,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 		final OutputInterpreter.OneLineParser Memparser = new OutputInterpreter.OneLineParser();
 		result = memScript.execute(Memparser);
 		if (result != null) {
-			s_logger.info("Unable to get the host Mem state: " + result);
+			s_logger.debug("Unable to get the host Mem state: " + result);
 			return new Answer(cmd, false, result);
 		}
 		freeMem = Long.parseLong(Memparser.getLine());
@@ -1880,7 +1880,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 		final OutputInterpreter.OneLineParser totMemparser = new OutputInterpreter.OneLineParser();
 		result = totalMem.execute(totMemparser);
 		if (result != null) {
-			s_logger.info("Unable to get the host Mem state: " + result);
+			s_logger.debug("Unable to get the host Mem state: " + result);
 			return new Answer(cmd, false, result);
 		}
 		long totMem = Long.parseLong(totMemparser.getLine());
@@ -2824,7 +2824,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         long dom0ram = Math.min(ram/10, 768*1024*1024L);//save a maximum of 10% of system ram or 768M
         dom0ram = Math.max(dom0ram, _dom0MinMem);
         info.add(dom0ram);
-    	s_logger.info("cpus=" + cpus + ", speed=" + speed + ", ram=" + ram + ", dom0ram=" + dom0ram);
+    	s_logger.debug("cpus=" + cpus + ", speed=" + speed + ", ram=" + ram + ", dom0ram=" + dom0ram);
 
         return info;
     }

agent/src/com/cloud/agent/vmdata/JettyVmDataServer.java

@@ -21,6 +21,7 @@ package com.cloud.agent.vmdata;
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
@@ -38,6 +39,7 @@ import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.apache.commons.codec.binary.Base64;
 import org.apache.log4j.Logger;
 import org.mortbay.jetty.Connector;
 import org.mortbay.jetty.Handler;
@@ -294,11 +296,19 @@ public class JettyVmDataServer implements VmDataServer {
             try {
                 _fs.create(vmDataDir, item[1]);
                 String vmDataFile = vmDataDir + File.separator + item[1];
+                byte[] data;
                 if (item[2] != null) {
-                    BufferedWriter writer = new BufferedWriter(new FileWriter(vmDataFile));
-                    writer.write(item[2]);
+                    if (item[1].equals("user-data")) {
+                        data = Base64.decodeBase64(item[2]);
+                    } else {
+                        data = item[2].getBytes();
+                    }
+                    if (data != null && data.length > 0) {
+                        FileOutputStream writer = new FileOutputStream(vmDataFile);
+                        writer.write(data);
                         writer.close();
                     }
+                }
             } catch (IOException e) {
                 s_logger.warn("Failed to write vm data item " + item[1], e);
                 return new Answer(cmd, false, "Failed to write vm data item " + item[1]);

api/src/com/cloud/agent/api/storage/CreateEntityDownloadURLCommand.java

@@ -22,8 +22,16 @@ import com.cloud.agent.api.Command;
 
 public class CreateEntityDownloadURLCommand extends AbstractDownloadCommand {
 
+    public CreateEntityDownloadURLCommand(String parent, String installPath, String uuid) { // this constructor is for creating template download url
+        super();
+        this.parent = parent; // parent is required as not the template can be child of one of many parents
+        this.installPath = installPath;
+        this.extractLinkUUID = uuid;
+    }
+    
     public CreateEntityDownloadURLCommand(String installPath, String uuid) {
         super();
+        this.parent = parent;
         this.installPath = installPath;
         this.extractLinkUUID = uuid;
     }
@@ -32,6 +40,7 @@ public class CreateEntityDownloadURLCommand extends AbstractDownloadCommand {
     }
 
     private String installPath;
+    private String parent;
     private String extractLinkUUID;
     
     @Override
@@ -47,6 +56,14 @@ public class CreateEntityDownloadURLCommand extends AbstractDownloadCommand {
         this.installPath = installPath;
     }
     
+    public String getParent() {
+        return parent;
+    }
+
+    public void setParent(String parent) {
+        this.parent = parent;
+    }
+
 	public String getExtractLinkUUID() {
 		return extractLinkUUID;
 	}

api/src/com/cloud/agent/api/to/NicTO.java

@@ -27,7 +27,6 @@ public class NicTO extends NetworkTO {
     Integer networkRateMulticastMbps;
     boolean defaultNic;
 
-
     public NicTO() {
         super();
     }

api/src/com/cloud/api/BaseCmd.java

@@ -550,4 +550,7 @@ public abstract class BaseCmd {
 	            (accountType == Account.ACCOUNT_TYPE_DOMAIN_ADMIN) ||
 	            (accountType == Account.ACCOUNT_TYPE_READ_ONLY_ADMIN));
 	}
+    public static boolean isRootAdmin(short accountType) {
+        return ((accountType == Account.ACCOUNT_TYPE_ADMIN));
+    }
 }

api/src/com/cloud/api/ResponseGenerator.java

@@ -161,7 +161,7 @@ public interface ResponseGenerator {
     RemoteAccessVpnResponse createRemoteAccessVpnResponse(RemoteAccessVpn vpn);
 
     void createTemplateResponse(List<TemplateResponse> responses, Pair<Long, Long> templateZonePair, boolean isAdmin,
-            Account account);
+            Account account, boolean readyOnly);
 
     ListResponse<TemplateResponse> createTemplateResponse2(VirtualMachineTemplate template, Long zoneId);
 
@@ -187,7 +187,7 @@ public interface ResponseGenerator {
 
     EventResponse createEventResponse(Event event);
 
-    ListResponse<TemplateResponse> createIsoResponse(Set<Pair<Long,Long>> isoZonePairSet, boolean onlyReady, Account account, Boolean isBootable);
+    ListResponse<TemplateResponse> createIsoResponse(Set<Pair<Long,Long>> isoZonePairSet, boolean onlyReady, Account account, Boolean isBootable, boolean readyOnly);
 
     TemplateResponse createIsoResponse(VirtualMachineTemplate result);
 

api/src/com/cloud/api/commands/CreateNetworkCmd.java

@@ -18,6 +18,8 @@
 
 package com.cloud.api.commands;
 
+import java.util.List;
+
 import org.apache.log4j.Logger;
 
 import com.cloud.api.ApiConstants;
@@ -84,6 +86,9 @@ public class CreateNetworkCmd extends BaseCmd {
     @Parameter(name=ApiConstants.NETWORK_DOMAIN, type=CommandType.STRING, description="network domain")
     private String networkDomain;
     
+    @Parameter(name=ApiConstants.TAGS, type=CommandType.LIST, collectionType=CommandType.STRING, description="Tag the network")
+    private List<String> tags;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -91,6 +96,10 @@ public class CreateNetworkCmd extends BaseCmd {
         return networkOfferingId;
     }
     
+    public List<String> getTags() {
+        return tags;
+    }
+
     public Long getZoneId() {
         return zoneId;
     }

api/src/com/cloud/api/commands/ListIsosCmd.java

@@ -125,6 +125,17 @@ public class ListIsosCmd extends BaseListCmd {
         return zoneId;
     }
     
+    public boolean listInReadyState() {
+        Account account = UserContext.current().getCaller();
+        // It is account specific if account is admin type and domainId and accountName are not null
+        boolean isAccountSpecific = (account == null || isAdmin(account.getType())) && (getAccountName() != null) && (getDomainId() != null);
+        // Show only those that are downloaded.
+        TemplateFilter templateFilter = TemplateFilter.valueOf(getIsoFilter());
+        boolean onlyReady = (templateFilter == TemplateFilter.featured) || (templateFilter == TemplateFilter.selfexecutable) || (templateFilter == TemplateFilter.sharedexecutable)
+        || (templateFilter == TemplateFilter.executable && isAccountSpecific) || (templateFilter == TemplateFilter.community);
+        return onlyReady;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -160,7 +171,7 @@ public class ListIsosCmd extends BaseListCmd {
             isAdmin = true;            
         }
 
-       ListResponse<TemplateResponse> response = _responseGenerator.createIsoResponse(isoZonePairSet, isAdmin, account, bootable);
+       ListResponse<TemplateResponse> response = _responseGenerator.createIsoResponse(isoZonePairSet, isAdmin, account, bootable, listInReadyState());
         response.setResponseName(getCommandName());
         this.setResponseObject(response);
     }

api/src/com/cloud/api/commands/ListServiceOfferingsCmd.java

@@ -26,7 +26,6 @@ import com.cloud.api.ApiConstants;
 import com.cloud.api.BaseListCmd;
 import com.cloud.api.Implementation;
 import com.cloud.api.Parameter;
-import com.cloud.api.BaseCmd.CommandType;
 import com.cloud.api.response.ListResponse;
 import com.cloud.api.response.ServiceOfferingResponse;
 import com.cloud.offering.ServiceOffering;

api/src/com/cloud/api/commands/ListTemplatesCmd.java

@@ -106,6 +106,17 @@ public class ListTemplatesCmd extends BaseListCmd {
         return zoneId;
     }
     
+    public boolean listInReadyState() {
+        Account account = UserContext.current().getCaller();
+        // It is account specific if account is admin type and domainId and accountName are not null
+        boolean isAccountSpecific = (account == null || isAdmin(account.getType())) && (getAccountName() != null) && (getDomainId() != null);
+        // Show only those that are downloaded.
+        TemplateFilter templateFilter = TemplateFilter.valueOf(getTemplateFilter());
+        boolean onlyReady = (templateFilter == TemplateFilter.featured) || (templateFilter == TemplateFilter.selfexecutable) || (templateFilter == TemplateFilter.sharedexecutable)
+        || (templateFilter == TemplateFilter.executable && isAccountSpecific) || (templateFilter == TemplateFilter.community);
+        return onlyReady;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -133,7 +144,7 @@ public class ListTemplatesCmd extends BaseListCmd {
         List<TemplateResponse> templateResponses = new ArrayList<TemplateResponse>();
 
         for (Pair<Long, Long> template : templateZonePairSet) {
-            _responseGenerator.createTemplateResponse(templateResponses, template, isAdmin, account);
+            _responseGenerator.createTemplateResponse(templateResponses, template, isAdmin, account, listInReadyState());
         }
 
         response.setResponses(templateResponses);

api/src/com/cloud/api/commands/UpdateNetworkCmd.java

@@ -18,6 +18,8 @@
 
 package com.cloud.api.commands;
 
+import java.util.List;
+
 import org.apache.log4j.Logger;
 
 import com.cloud.api.ApiConstants;
@@ -50,6 +52,9 @@ public class UpdateNetworkCmd extends BaseCmd {
     @Parameter(name=ApiConstants.DISPLAY_TEXT, type=CommandType.STRING, description="the new display text for the network")
     private String displayText;
     
+    @Parameter(name=ApiConstants.TAGS, type=CommandType.LIST, collectionType=CommandType.STRING, description="tags for the network")
+    private List<String> tags;
+    
   
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
@@ -67,6 +72,10 @@ public class UpdateNetworkCmd extends BaseCmd {
         return displayText;
     }
     
+    public List<String> getTags() {
+        return tags;
+    }
+    
     
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
@@ -90,7 +99,7 @@ public class UpdateNetworkCmd extends BaseCmd {
     @Override
     public void execute() throws InsufficientCapacityException, ConcurrentOperationException{
         
-        Network result = _networkService.updateNetwork(getId(), getNetworkName(), getDisplayText(), UserContext.current().getCaller());
+        Network result = _networkService.updateNetwork(getId(), getNetworkName(), getDisplayText(), tags, UserContext.current().getCaller());
         if (result != null) {
             NetworkResponse response = _responseGenerator.createNetworkResponse(result);
             response.setResponseName(getCommandName());

api/src/com/cloud/api/response/DomainRouterResponse.java

@@ -113,6 +113,12 @@ public class DomainRouterResponse extends BaseResponse {
     @SerializedName(ApiConstants.DOMAIN) @Param(description="the domain associated with the router")
     private String domainName;
     
+    @SerializedName("serviceofferingid") @Param(description="the ID of the service offering of the virtual machine")
+    private Long serviceOfferingId;
+
+    @SerializedName("serviceofferingname") @Param(description="the name of the service offering of the virtual machine")
+    private String serviceOfferingName;
+    
 
     
     
@@ -353,4 +359,19 @@ public class DomainRouterResponse extends BaseResponse {
         this.linkLocalNetworkId = linkLocalNetworkId;
     }
     
+    public Long getServiceOfferingId() {
+        return serviceOfferingId;
+    }
+
+    public void setServiceOfferingId(Long serviceOfferingId) {
+        this.serviceOfferingId = serviceOfferingId;
+    }
+
+    public String getServiceOfferingName() {
+        return serviceOfferingName;
+    }
+
+    public void setServiceOfferingName(String serviceOfferingName) {
+        this.serviceOfferingName = serviceOfferingName;
+    }
 }

api/src/com/cloud/api/response/TemplateResponse.java

@@ -19,6 +19,7 @@ package com.cloud.api.response;
 
 import java.util.Date;
 
+import com.cloud.api.ApiConstants;
 import com.cloud.serializer.Param;
 import com.cloud.storage.Storage.ImageFormat;
 import com.google.gson.annotations.SerializedName;
@@ -113,6 +114,12 @@ public class TemplateResponse extends BaseResponse {
     @SerializedName("sourcetemplateid") @Param(description="the template ID of the parent template if present")
     private Long sourcetemplateId;    
     
+    @SerializedName(ApiConstants.HOST_ID) @Param(description="the ID of the secondary storage host for the template")
+    private Long hostId;
+
+    @SerializedName("hostname") @Param(description="the name of the secondary storage host for the template")
+    private String hostName;
+    
     public Long getObjectId() {
     	return getId();
     }
@@ -348,4 +355,20 @@ public class TemplateResponse extends BaseResponse {
     public void setSourceTemplateId(Long sourcetemplateId) {
         this.sourcetemplateId = sourcetemplateId;
     }    
+    
+    public Long getHostId() {
+        return hostId;
+    }
+
+    public void setHostId(Long hostId) {
+        this.hostId = hostId;
+    }
+
+    public String getHostName() {
+        return hostName;
+    }
+
+    public void setHostName(String hostName) {
+        this.hostName = hostName;
+    }
 }

api/src/com/cloud/host/Status.java

@@ -33,7 +33,7 @@ public enum Status {
     Maintenance(false, false, false),
     Alert(true, true, true),
     Removed(true, false, true),
-    Rebalance(false, false, false);
+    Rebalancing(false, false, false);
     
     private final boolean updateManagementServer;
     private final boolean checkManagementServer;
@@ -137,7 +137,7 @@ public enum Status {
         s_fsm.addTransition(Status.Up, Event.Ping, Status.Up);
         s_fsm.addTransition(Status.Up, Event.AgentConnected, Status.Connecting);
         s_fsm.addTransition(Status.Up, Event.ManagementServerDown, Status.Disconnected);
-        s_fsm.addTransition(Status.Up, Event.StartAgentRebalance, Status.Rebalance);
+        s_fsm.addTransition(Status.Up, Event.StartAgentRebalance, Status.Rebalancing);
         s_fsm.addTransition(Status.Updating, Event.PingTimeout, Status.Alert);
         s_fsm.addTransition(Status.Updating, Event.Ping, Status.Updating);
         s_fsm.addTransition(Status.Updating, Event.AgentConnected, Status.Connecting);
@@ -183,8 +183,8 @@ public enum Status {
         s_fsm.addTransition(Status.Alert, Event.Ping, Status.Up);
         s_fsm.addTransition(Status.Alert, Event.Remove, Status.Removed);
         s_fsm.addTransition(Status.Alert, Event.ManagementServerDown, Status.Alert);
-        s_fsm.addTransition(Status.Rebalance, Event.RebalanceFailed, Status.Alert);
-        s_fsm.addTransition(Status.Rebalance, Event.RebalanceCompleted, Status.Connecting);
+        s_fsm.addTransition(Status.Rebalancing, Event.RebalanceFailed, Status.Alert);
+        s_fsm.addTransition(Status.Rebalancing, Event.RebalanceCompleted, Status.Connecting);
     }
 
     public static void main(String[] args) {

api/src/com/cloud/network/NetworkService.java

@@ -81,7 +81,7 @@ public interface NetworkService {
 
     Long getDedicatedNetworkDomain(long networkId);
 
-    Network updateNetwork(long networkId, String name, String displayText, Account caller);
+    Network updateNetwork(long networkId, String name, String displayText, List<String> tags, Account caller);
 
     Integer getNetworkRate(long networkId, Long vmId);
 

api/src/com/cloud/vm/VirtualMachine.java

@@ -216,12 +216,12 @@ public interface VirtualMachine extends RunningOn, ControlledEntity, StateObject
     /**
      * @return pod id.
      */
-    public Long getPodId();
+    public Long getPodIdToDeployIn();
     
     /**
      * @return data center id.
      */
-    public long getDataCenterId();
+    public long getDataCenterIdToDeployIn();
     
     /**
      * @return id of the host it was assigned last time.
@@ -230,8 +230,6 @@ public interface VirtualMachine extends RunningOn, ControlledEntity, StateObject
     @Override
     public Long getHostId();
     
-    public void setLastHostId(Long lastHostId);
-    
     /**
      * @return should HA be enabled for this machine?
      */

client/WEB-INF/classes/resources/messages.properties

@@ -210,6 +210,7 @@ label.add.primary.storage=Add Primary Storage
 label.add.secondary.storage=Add Secondary Storage
 label.add.security.group=Add Security Group
 label.add.service.offering=Add Service Offering
+label.add.system.service.offering=Add System Service Offering
 label.add.template=Add Template
 label.add.user=Add User
 label.add.vlan=Add VLAN
@@ -429,6 +430,7 @@ label.menu.physical.resources=Physical Resources
 label.menu.running.instances=Running Instances
 label.menu.security.groups=Security Groups
 label.menu.service.offerings=Service Offerings
+label.menu.system.service.offerings=System Service Offerings
 label.menu.snapshots=Snapshots
 label.menu.stopped.instances=Stopped Instances
 label.menu.storage=Storage
@@ -541,6 +543,7 @@ label.security.groups=Security Groups
 label.sent=Sent
 label.server=Server
 label.service.offering=Service Offering
+label.system.service.offering=System Service Offering
 label.session.expired=Session Expired
 label.shared=Shared
 label.size=Size
@@ -774,7 +777,8 @@ message.zone.step.2.desc=Please enter the following info to add a new zone
 message.zone.step.3.desc=Please enter the following info to add a new pod 
 message.apply.snapshot.policy=You have successfully updated your current snapshot policy.
 message.disable.snapshot.policy=You have successfully disabled your current snapshot policy.
-message.action.change.service.warning=Your instance must be stopped before attempting to change its current service offering.
+message.action.change.service.warning.for.instance=Your instance must be stopped before attempting to change its current service offering.
+message.action.change.service.warning.for.router=Your router must be stopped before attempting to change its current service offering.
 message.action.reset.password.warning=Your instance must be stopped before attempting to change its current password.
 message.action.reset.password.off=Your instance currently does not support this feature.
 

client/tomcatconf/components.xml.in

@@ -93,7 +93,7 @@
         <adapters key="com.cloud.acl.SecurityChecker">
             <adapter name="DomainChecker" class="com.cloud.acl.DomainChecker"/>
         </adapters>
-        <adapters key="com.cloud.cluster.agentlb">
+        <adapters key="com.cloud.cluster.agentlb.AgentLoadBalancerPlanner">
             <adapter name="ClusterBasedAgentLbPlanner" class="com.cloud.cluster.agentlb.ClusterBasedAgentLoadBalancerPlanner"/>
         </adapters>
 

cloud.spec

@@ -221,7 +221,6 @@ Requires: jpackage-utils
 Requires: %{name}-daemonize
 Requires: /sbin/service
 Requires: /sbin/chkconfig
-Requires: jnetpcap
 Group:     System Environment/Libraries
 
 %package baremetal-agent
@@ -304,6 +303,7 @@ The Cloud.com command line tools contain a few Python modules that can call clou
 %package premium-agent
 Summary: Cloud.com premium agent
 Requires: cloud-agent
+Requires: jnetpcap
 Group:     System Environment/Libraries
 %description premium-agent
 The Cloud.com premium agent

core/src/com/cloud/agent/transport/Request.java

@@ -198,8 +198,8 @@ public class Request {
         _agentId = agentId;
     }
 
-    public void setVia(long agentId) {
-        _via = agentId;
+    public void setVia(long viaId) {
+        _via = viaId;
     }
 
     public boolean executeInSequence() {
@@ -238,8 +238,8 @@ public class Request {
         buffer.putLong(_seq);
         buffer.putInt(contentSize);
         buffer.putLong(_mgmtId);
-        buffer.putLong(_via);
         buffer.putLong(_agentId);
+        buffer.putLong(_via);
         buffer.flip();
 
         return buffer;
@@ -302,6 +302,11 @@ public class Request {
         }
     }
     
+    @Override
+    public String toString() {
+        return log("", true, Level.DEBUG);
+    }
+
     protected String log(String msg, boolean logContent, Level level) {
         StringBuilder content = new StringBuilder();
         if (logContent) {
@@ -373,8 +378,8 @@ public class Request {
         final long seq = buff.getLong();
         final int size = buff.getInt();
         final long mgmtId = buff.getLong();
-        final long via = buff.getLong();
         final long agentId = buff.getLong();
+        final long via = buff.getLong();
 
         byte[] command = null;
         int offset = 0;
@@ -421,6 +426,10 @@ public class Request {
     }
 
     public static long getAgentId(final byte[] bytes) {
+        return NumbersUtil.bytesToLong(bytes, 28);
+    }
+
+    public static long getViaAgentId(final byte[] bytes) {
         return NumbersUtil.bytesToLong(bytes, 24);
     }
     

core/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java

@@ -32,12 +32,14 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Queue;
+import java.util.Random;
 import java.util.Set;
 import java.util.UUID;
 
@@ -120,8 +122,8 @@ import com.cloud.agent.api.StartupStorageCommand;
 import com.cloud.agent.api.StopAnswer;
 import com.cloud.agent.api.StopCommand;
 import com.cloud.agent.api.StoragePoolInfo;
-import com.cloud.agent.api.UpgradeSnapshotCommand;
 import com.cloud.agent.api.UpdateHostPasswordCommand;
+import com.cloud.agent.api.UpgradeSnapshotCommand;
 import com.cloud.agent.api.VmStatsEntry;
 import com.cloud.agent.api.check.CheckSshAnswer;
 import com.cloud.agent.api.check.CheckSshCommand;
@@ -182,7 +184,6 @@ import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.Volume;
 import com.cloud.storage.VolumeVO;
-import com.cloud.storage.resource.StoragePoolResource;
 import com.cloud.storage.template.TemplateInfo;
 import com.cloud.template.VirtualMachineTemplate.BootloaderType;
 import com.cloud.utils.NumbersUtil;
@@ -222,14 +223,21 @@ import com.xensource.xenapi.VMGuestMetrics;
 import com.xensource.xenapi.XenAPIObject;
 
 /**
- * Encapsulates the interface to the XenServer API.
+ * CitrixResourceBase encapsulates the calls to the XenServer Xapi process
+ * to perform the required functionalities for CloudStack.
+ * 
+ * ==============>  READ THIS  <==============
+ * Because the XenServer objects can expire when the session expires, we cannot
+ * keep any of the actual XenServer objects in this class.  The only
+ * thing that is constant is the UUID of the XenServer objects but not the
+ * objects themselves!  This is very important before you do any changes in
+ * this code here.
  * 
  */
 @Local(value = ServerResource.class)
 public abstract class CitrixResourceBase implements ServerResource, HypervisorResource {
     private static final Logger s_logger = Logger.getLogger(CitrixResourceBase.class);
     protected static final XenServerConnectionPool _connPool = XenServerConnectionPool.getInstance();
-    protected static final int MB = 1024 * 1024;
     protected String _name;
     protected String _username;
     protected Queue<String> _password=new LinkedList<String>();
@@ -247,38 +255,43 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
     protected String _guestNetworkName;
     protected int _wait;
     protected String _instance; //instance name (default is usually "VM")
+    static final Random _rand = new Random(System.currentTimeMillis());
 
     protected IAgentControl _agentControl;
 
     final int _maxWeight = 256;
 
-    protected final XenServerHost _host = new XenServerHost();
+    protected final XsHost _host = new XsHost();
 
     // Guest and Host Performance Statistics
-    protected boolean _collectHostStats = false;
     protected String _consolidationFunction = "AVERAGE";
     protected int _pollingIntervalInSeconds = 60;
 
     protected boolean _canBridgeFirewall = false;
     protected boolean _isOvs = false;
     protected List<VIF> _tmpDom0Vif = new ArrayList<VIF>();
-    protected HashMap<StoragePoolType, StoragePoolResource> _pools = new HashMap<StoragePoolType, StoragePoolResource>(5);
+    protected String _localGateway;
 
     public enum SRType {
         NFS, LVM, ISCSI, ISO, LVMOISCSI, LVMOHBA;
         
+        String _str;
+        
+        private SRType() {
+            _str = super.toString().toLowerCase();
+        }
+
         @Override
         public String toString() {
-            return super.toString().toLowerCase();
+            return _str;
         }
 
         public boolean equals(String type) {
-            return super.toString().equalsIgnoreCase(type);
+            return _str.equalsIgnoreCase(type);
         }
     }
 
     protected static HashMap<Types.VmPowerState, State> s_statesTable;
-    protected String _localGateway;
     static {
         s_statesTable = new HashMap<Types.VmPowerState, State>();
         s_statesTable.put(Types.VmPowerState.HALTED, State.Stopped);
@@ -320,25 +333,6 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
     public void disconnected() {
     }
 
-    protected Pair<VM, VM.Record> getVmByNameLabel(Connection conn, Host host, String nameLabel, boolean getRecord) throws XmlRpcException, XenAPIException {
-        Set<VM> vms = host.getResidentVMs(conn);
-        for (VM vm : vms) {
-            VM.Record rec = null;
-            String name = null;
-            if (getRecord) {
-                rec = vm.getRecord(conn);
-                name = rec.nameLabel;
-            } else {
-                name = vm.getNameLabel(conn);
-            }
-            if (name.equals(nameLabel)) {
-                return new Pair<VM, VM.Record>(vm, rec);
-            }
-        }
-
-        return null;
-    }
-
     protected boolean pingdomr(Connection conn, String host, String port) {
         String status;
         status = callHostPlugin(conn, "vmops", "pingdomr", "host", host, "port", port);
@@ -351,7 +345,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
 
     }
     
-    protected boolean pingxenserver() {
+    protected boolean pingXenServer() {
         Session slaveSession = null;
         Connection slaveConn = null;
         try {
@@ -499,20 +493,25 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
         }
     }
     
-    Pair<Network, String> getNativeNetworkForTraffic(Connection conn, TrafficType type) throws XenAPIException, XmlRpcException {
+    protected XsLocalNetwork getNativeNetworkForTraffic(Connection conn, TrafficType type, String tag) throws XenAPIException, XmlRpcException {
+        if (tag != null) {
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug("Looking for network named " + tag);
+            }
+            return getNetworkByName(conn, tag);
+        }
+        
         if (type == TrafficType.Guest) {
-            return new Pair<Network, String>(Network.getByUuid(conn, _host.guestNetwork), _host.guestPif);
+            return new XsLocalNetwork(Network.getByUuid(conn, _host.guestNetwork), null, PIF.getByUuid(conn, _host.guestPif), null);
         } else if (type == TrafficType.Control) {
             setupLinkLocalNetwork(conn);
-            return new Pair<Network, String>(Network.getByUuid(conn, _host.linkLocalNetwork), null);
+            return new XsLocalNetwork(Network.getByUuid(conn, _host.linkLocalNetwork));
         } else if (type == TrafficType.Management) {
-            return new Pair<Network, String>(Network.getByUuid(conn, _host.privateNetwork), _host.privatePif);
+            return new XsLocalNetwork(Network.getByUuid(conn, _host.privateNetwork), null, PIF.getByUuid(conn, _host.privatePif), null);
         } else if (type == TrafficType.Public) {
-            return new Pair<Network, String>(Network.getByUuid(conn, _host.publicNetwork), _host.publicPif);
+            return new XsLocalNetwork(Network.getByUuid(conn, _host.publicNetwork), null, PIF.getByUuid(conn, _host.publicPif), null);
         } else if (type == TrafficType.Storage) {
-            return new Pair<Network, String>(Network.getByUuid(conn, _host.storageNetwork1), _host.storagePif1);
-        } else if (type == TrafficType.Vpn) {
-            return new Pair<Network, String>(Network.getByUuid(conn, _host.publicNetwork), _host.publicPif);
+            return new XsLocalNetwork(Network.getByUuid(conn, _host.storageNetwork1), null, PIF.getByUuid(conn, _host.storagePif1), null);
         }
         
         throw new CloudRuntimeException("Unsupported network type: " + type);
@@ -572,8 +571,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
 					vswitchNw = networks.iterator().next();
 				}
 
-				enableXenServerNetwork(conn, vswitchNw, "vswitch",
-						"vswicth network");
+				enableXenServerNetwork(conn, vswitchNw, "vswitch", "vswicth network");
 				_host.vswitchNetwork = vswitchNw;
 			}
 			return _host.vswitchNetwork;
@@ -609,16 +607,17 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
 
     
     protected Network getNetwork(Connection conn, NicTO nic) throws XenAPIException, XmlRpcException {
-        Pair<Network, String> network = getNativeNetworkForTraffic(conn, nic.getType());
+        String[] tags = nic.getTags();
+        XsLocalNetwork network = getNativeNetworkForTraffic(conn, nic.getType(), tags != null && tags.length > 0 ? tags[0] : null);
         if (nic.getBroadcastUri() != null && nic.getBroadcastUri().toString().contains("untagged")) {
-            return network.first();
+            return network.getNetwork();
         } else if (nic.getBroadcastType() == BroadcastDomainType.Vlan) {
             URI broadcastUri = nic.getBroadcastUri();
             assert broadcastUri.getScheme().equals(BroadcastDomainType.Vlan.scheme());
             long vlan = Long.parseLong(broadcastUri.getHost());
-            return enableVlanNetwork(conn, vlan, network.first(), network.second());
+            return enableVlanNetwork(conn, vlan, network);
         } else if (nic.getBroadcastType() == BroadcastDomainType.Native || nic.getBroadcastType() == BroadcastDomainType.LinkLocal) {
-            return network.first();
+            return network.getNetwork();
         } else if (nic.getBroadcastType() == BroadcastDomainType.Vswitch) {
             String broadcastUri = nic.getBroadcastUri().toString();
             String header = broadcastUri.substring(Networks.BroadcastDomainType.Vswitch.scheme().length() + "://".length());
@@ -1580,7 +1579,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
     }
 
     protected CheckHealthAnswer execute(CheckHealthCommand cmd) {
-        boolean result = pingxenserver();
+        boolean result = pingXenServer();
         return new CheckHealthAnswer(cmd, result);
     }
 
@@ -1799,9 +1798,10 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
             }
             
             vmStatsAnswer.setCPUUtilization(vmStatsAnswer.getCPUUtilization()*100);
-            if(s_logger.isDebugEnabled())
+            if(s_logger.isDebugEnabled()) {
                 s_logger.debug("Vm cpu utilization " + vmStatsAnswer.getCPUUtilization());
             }
+        }
 
         return vmResponseMap;
     }
@@ -2880,8 +2880,9 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
                 "op", "download", "hostname", swift.getHostName(), "account", swift.getAccount(),
                 "username", swift.getUserName(), "token", swift.getToken(), "rfilename", rfilename,
                 "lfilename", lfilename);
-            if( result != null && result.equals("true")) 
+            if( result != null && result.equals("true")) {
                 return true;
+            }
         } catch (Exception e) {
             s_logger.warn("swift download failed due to " + e.toString());
         }
@@ -2895,8 +2896,9 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
                 "op", "upload", "hostname", swift.getHostName(), "account", swift.getAccount(),
                 "username", swift.getUserName(), "token", swift.getToken(), "rfilename", rfilename,
                 "lfilename", lfilename);
-            if( result != null && result.equals("true")) 
+            if( result != null && result.equals("true")) {
                 return true;
+            }
         } catch (Exception e) {
             s_logger.warn("swift download failed due to " + e.toString());
         }
@@ -2909,8 +2911,9 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
             result = callHostPlugin(conn, "swift", "swift",
                 "op", "delete", "hostname", swift.getHostName(), "account", swift.getAccount(),
                 "username", swift.getUserName(), "token", swift.getToken(), "rfilename", rfilename);
-            if( result != null && result.equals("true")) 
+            if( result != null && result.equals("true")) {
                 return true;
+            }
         } catch (Exception e) {
             s_logger.warn("swift download failed due to " + e.toString());
         }
@@ -3267,7 +3270,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
         return true;
     }
 
-    protected Nic getManageMentNetwork(Connection conn) throws XmlRpcException, XenAPIException {       
+    protected XsLocalNetwork getManagementNetwork(Connection conn) throws XmlRpcException, XenAPIException {
         PIF mgmtPif = null;
         PIF.Record mgmtPifRec = null;
         Host host = Host.getByUuid(conn, _host.uuid);
@@ -3303,27 +3306,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
         }
         Network nk =  mgmtPifRec.network;
         Network.Record nkRec = nk.getRecord(conn);
-        return new Nic(nk, nkRec, mgmtPif, mgmtPifRec);
-    }
-    
-    
-    protected Nic getLocalNetwork(Connection conn, String name) throws XmlRpcException, XenAPIException {
-        Set<Network> networks = Network.getByNameLabel(conn, name);
-        for (Network network : networks) {
-            Network.Record nr = network.getRecord(conn);
-            for (PIF pif : nr.PIFs) {
-                PIF.Record pr = pif.getRecord(conn);
-                if (_host.uuid.equals(pr.host.getUuid(conn))) {
-                    if (s_logger.isDebugEnabled()) {
-                        s_logger.debug("Found a network called " + name + " on host=" + _host.ip + ";  Network="
-                                + nr.uuid + "; pif=" + pr.uuid);
-                    }
-                    return new Nic(network, nr, pif, pr);
-                }
-            }
-        }
-
-        return null;
+        return new XsLocalNetwork(nk, nkRec, mgmtPif, mgmtPifRec);
     }
     
     
@@ -3385,144 +3368,185 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
         return getVDIbyUuid(conn, volumePath);
     }
     
-    protected Network getNetworkByName(Connection conn, String name) throws BadServerResponse, XenAPIException, XmlRpcException {
+    /**
+     * getNetworkByName() retrieves what the server thinks is the actual
+     * network used by the XenServer host.  This method should always be
+     * used to talk to retrieve a network by the name.  The reason is
+     * because of the problems in using the name label as the way to find
+     * the Network.
+     * 
+     * To see how we are working around these problems, take a look at
+     * enableVlanNetwork().  The following description assumes you have looked
+     * at the description on that method.
+     * 
+     * In order to understand this, we have to see what type of networks are
+     * within a XenServer that's under CloudStack control.
+     * 
+     *   - Native Networks: these are networks that are untagged on the
+     *     XenServer and are used to crate VLAN networks on.  These are
+     *     created by the user and is assumed to be one per cluster.
+     *   - VLAN Networks: these are dynamically created by CloudStack and can
+     *     have problems with duplicated names.
+     *   - LinkLocal Networks: these are dynamically created by CloudStack and
+     *     can also have problems with duplicated names but these don't have
+     *     actual PIFs.
+     * 
+     *  In order to speed to retrieval of a network, we do the following:
+     *    - We retrieve by the name.  If only one network is retrieved, we
+     *      assume we retrieved the right network.
+     *    - If more than one network is retrieved, we check to see which one
+     *      has the pif for the local host and use that.
+     *    - If a pif is not found, then we look at the tags and find the
+     *      one with the lowest timestamp. (See enableVlanNetwork())
+     * 
+     * @param conn Xapi connection
+     * @param name name of the network
+     * @return XsNic an object that contains network, network record, pif, and pif record.
+     * @throws XenAPIException
+     * @throws XmlRpcException
+     * 
+     * @see enableVlanNetwork
+     */
+    protected XsLocalNetwork getNetworkByName(Connection conn, String name) throws XenAPIException, XmlRpcException {
         Set<Network> networks = Network.getByNameLabel(conn, name);
-        if (networks.size() > 0) {
-            assert networks.size() == 1 : "How did we find more than one network with this name label" + name + "?  Strange....";
-            return networks.iterator().next(); // Found it.
+        if (networks.size() == 1) {
+            return new XsLocalNetwork(networks.iterator().next(), null, null, null);
         }
         
+        if (networks.size() == 0) {
             return null;
         }
         
-    protected synchronized Network getNetworkByName(Connection conn, String name, boolean lookForPif) throws XenAPIException, XmlRpcException {
-        Network found = null;
-        Set<Network> networks = Network.getByNameLabel(conn, name);
-        if (networks.size() == 1) {
-            found = networks.iterator().next();
-        } else if (networks.size() > 1) {
         if (s_logger.isDebugEnabled()) {
             s_logger.debug("Found more than one network with the name " + name);
         }
+        Network earliestNetwork = null;
+        Network.Record earliestNetworkRecord = null;
+        long earliestTimestamp = Long.MAX_VALUE;
+        int earliestRandom = Integer.MAX_VALUE;
         for (Network network : networks) {
-                if (!lookForPif) {
-                    found = network;
-                    break;
+            XsLocalNetwork nic = new XsLocalNetwork(network);
+            
+            if (nic.getPif(conn) != null) {
+                return nic;
             }
 
-                Network.Record netr = network.getRecord(conn);
-                s_logger.debug("Checking network " + netr.uuid);
-                if (netr.PIFs.size() == 0) {
-                    if (s_logger.isDebugEnabled()) {
-                        s_logger.debug("Network " + netr.uuid + " has no pifs so skipping that.");
-                    }
-                } else {
-                    for (PIF pif : netr.PIFs) {
-                        PIF.Record pifr = pif.getRecord(conn);
-                        if (_host.uuid.equals(pifr.host.getUuid(conn))) {
-                            if (s_logger.isDebugEnabled()) {
-                                s_logger.debug("Network " + netr.uuid + " has a pif " + pifr.uuid + " for our host ");
-                            }
-                            found = network;
-                            break;
+            Network.Record record = network.getRecord(conn);
+            if (record.tags != null) {
+                for (String tag : record.tags) {
+                    Pair<Long, Integer> stamp = parseTimestamp(tag);
+                    if (stamp == null) {
+                        continue;
                     }
+                    
+                    if (stamp.first() < earliestTimestamp || (stamp.first() == earliestTimestamp && stamp.second() < earliestRandom)) {
+                        earliestTimestamp = stamp.first();
+                        earliestRandom = stamp.second();
+                        earliestNetwork = network;
+                        earliestNetworkRecord = record;
                     }
                 }
             }
         }
 
-        return found;
+        return earliestNetwork != null ? new XsLocalNetwork(earliestNetwork, earliestNetworkRecord, null, null) : null;
     }
     
-    protected Network enableVlanNetwork(Connection conn, long tag, String pifUuid) throws XenAPIException, XmlRpcException {
-        // In XenServer, vlan is added by
-        // 1. creating a network.
-        // 2. creating a vlan associating network with the pif.
-        // We always create
-        // 1. a network with VLAN[vlan id in decimal]
-        // 2. a vlan associating the network created with the pif to private
-        // network.
+    protected String generateTimeStamp() {
+        return new StringBuilder("CsCreateTime-").append(System.currentTimeMillis()).append("-").append(_rand.nextInt()).toString();
+    }
+    
+    protected Pair<Long, Integer> parseTimestamp(String timeStampStr) {
+        String[] tokens = timeStampStr.split("-");
+        assert(tokens.length == 3) : "It's our timestamp but it doesn't fit our format: " + timeStampStr;
+        if (!tokens[0].equals("CsCreateTime-")) {
+            return null;
+        }
+        return new Pair<Long, Integer>(Long.parseLong(tokens[1]), Integer.parseInt(tokens[2]));
+    }
+    
+    /**
+     * enableVlanNetwork creates a Network object, Vlan object, and thereby
+     * a tagged PIF object in Xapi.
+     * 
+     * In XenServer, VLAN is added by
+     *   - Create a network, which is unique cluster wide.
+     *   - Find the PIF that you want to create the VLAN on.
+     *   - Create a VLAN using the network and the PIF.  As a result of this
+     *     operation, a tagged PIF object is also created.
+     * 
+     * Here is a list of problems with clustered Xapi implementation that
+     * we are trying to circumvent.
+     *   - There can be multiple Networks with the same name-label so searching
+     *     using name-label is not unique.
+     *   - There are no other ways to search for Networks other than listing
+     *     all of them which is not efficient in our implementation because
+     *     we can have over 4000 VLAN networks.
+     *   - In a clustered situation, it's possible for both hosts to detect
+     *     that the Network is missing and both creates it.  This causes a
+     *     lot of problems as one host may be using one Network and another
+     *     may be using a different network for their VMs.  This causes
+     *     problems in migration because the VMs are logically attached
+     *     to different networks in Xapi's database but in reality, they
+     *     are attached to the same network.
+     * 
+     * To work around these problems, we do the following.
+     * 
+     *   - When creating the VLAN network, we name it as VLAN-UUID of the
+     *     Network it is created on-VLAN Tag.  Because VLAN tags is unique with
+     *     one particular network, this is a unique name-label to quickly
+     *     retrieve the the VLAN network with when we need it again.
+     *   - When we create the VLAN network, we add a timestamp and a random
+     *     number as a tag into the network.  Then instead of creating
+     *     VLAN on that network, we actually retrieve the Network again
+     *     and this time uses the VLAN network with lowest timestamp or
+     *     lowest random number as the VLAN network.  This allows VLAN creation
+     *     to happen on multiple hosts concurrently but even if two VLAN
+     *     networks were created with the same name, only one of them is used.
+     * 
+     * One cavaet about this approach is that it relies on the timestamp to
+     * be relatively accurate among different hosts.
+     * 
+     * @param conn Xapi Connection
+     * @param tag VLAN tag
+     * @param network network on this host to create the VLAN on.
+     * @return VLAN Network created.
+     * @throws XenAPIException
+     * @throws XmlRpcException
+     */
+    protected Network enableVlanNetwork(Connection conn, long tag, XsLocalNetwork network) throws XenAPIException, XmlRpcException {
         Network vlanNetwork = null;
-        String name = "VLAN" + Long.toString(tag);
-
-        synchronized (name.intern()) {
-            vlanNetwork = getNetworkByName(conn, name);
-            if (vlanNetwork == null) { // Can't find it, then create it.
+        String oldName = "VLAN" + Long.toString(tag);
+        String newName = "VLAN-" + network.getNetworkRecord(conn).uuid + "-" + tag;
+        XsLocalNetwork vlanNic = getNetworkByName(conn, newName);
+        if (vlanNic == null) {
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug("Couldn't find vlan network with the new name so trying old name: " + oldName);
+            }
+            vlanNic = getNetworkByName(conn, oldName);
+            if (vlanNic != null) {
+                s_logger.info("Renaming VLAN with old name " + oldName + " to " + newName);
+                vlanNic.getNetwork().setNameLabel(conn, newName);
+            }
+        }
+        if (vlanNic == null) { // Can't find it, then create it.
             if (s_logger.isDebugEnabled()) {
                 s_logger.debug("Creating VLAN network for " + tag + " on host " + _host.ip);
             }
             Network.Record nwr = new Network.Record();
-                nwr.nameLabel = name;
-                nwr.bridge = name;
+            nwr.nameLabel = newName;
+            nwr.tags = new HashSet<String>();
+            nwr.tags.add(generateTimeStamp());
             vlanNetwork = Network.create(conn, nwr);
+            vlanNic = getNetworkByName(conn, newName);
         }
 
-            PIF nPif = PIF.getByUuid(conn, pifUuid);
-            PIF.Record nPifr = nPif.getRecord(conn);
+        PIF nPif = network.getPif(conn);
+        PIF.Record nPifr = network.getPifRecord(conn);
         
-            Network.Record vlanNetworkr = vlanNetwork.getRecord(conn);
-            if (vlanNetworkr.PIFs != null) {
-                for (PIF pif : vlanNetworkr.PIFs) {
-                    PIF.Record pifr = pif.getRecord(conn);
-                    if(pifr.host.equals(nPifr.host)) {
-                        if (pifr.device.equals(nPifr.device) ) {
-                            pif.plug(conn);
+        vlanNetwork = vlanNic.getNetwork();
+        if (vlanNic.getPif(conn) != null) {
             return vlanNetwork;
-                        } else {
-                            throw new CloudRuntimeException("Creating VLAN " + tag + " on " + nPifr.device + " failed due to this VLAN is already created on " + pifr.device);                	
-                        }
-                        
-                    }
-                }
-            }
-
-            if (s_logger.isDebugEnabled()) {
-                s_logger.debug("Creating VLAN " + tag + " on host " + _host.ip + " on device " + nPifr.device);
-            }
-            VLAN vlan = VLAN.create(conn, nPif, tag, vlanNetwork);
-            PIF untaggedPif = vlan.getUntaggedPIF(conn);
-            if (!untaggedPif.getCurrentlyAttached(conn)) {
-                untaggedPif.plug(conn);
-            }
-        }
-
-        return vlanNetwork;
-    }
-
-    protected Network enableVlanNetwork(Connection conn, long tag, Network network, String pifUuid) throws XenAPIException, XmlRpcException {
-        // In XenServer, vlan is added by
-        // 1. creating a network.
-        // 2. creating a vlan associating network with the pif.
-        // We always create
-        // 1. a network with VLAN[vlan id in decimal]
-        // 2. a vlan associating the network created with the pif to private
-        // network.
-
-        Network vlanNetwork = null;
-        String name = "VLAN" + Long.toString(tag);
-
-        vlanNetwork = getNetworkByName(conn, name, true);
-        if (vlanNetwork == null) { // Can't find it, then create it.
-            if (s_logger.isDebugEnabled()) {
-                s_logger.debug("Creating VLAN network for " + tag + " on host " + _host.ip);
-            }
-            Network.Record nwr = new Network.Record();
-            nwr.nameLabel = name;
-            nwr.bridge = name;
-            vlanNetwork = Network.create(conn, nwr);
-        }
-
-        PIF nPif = PIF.getByUuid(conn, pifUuid);
-        PIF.Record nPifr = nPif.getRecord(conn);
-
-        Network.Record vlanNetworkr = vlanNetwork.getRecord(conn);
-        if (vlanNetworkr.PIFs != null) {
-            for (PIF pif : vlanNetworkr.PIFs) {
-                PIF.Record pifr = pif.getRecord(conn);
-                if (pifr.device.equals(nPifr.device) && pifr.host.equals(nPifr.host)) {
-                    return vlanNetwork;
-                }
-            }
         }
 
         if (s_logger.isDebugEnabled()) {
@@ -3614,9 +3638,9 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
     public PingCommand getCurrentStatus(long id) {
         try {
             Connection conn = getConnection();
-            if (!pingxenserver()) {
+            if (!pingXenServer()) {
                 Thread.sleep(1000);
-                if (!pingxenserver()) {
+                if (!pingXenServer()) {
                     s_logger.warn(" can not ping xenserver " + _host.uuid);
                     return null;
                 }
@@ -3677,18 +3701,18 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
                 _host.speed = hc.getSpeed(conn).intValue();
                 break;
             }
-            Nic privateNic = getManageMentNetwork(conn);
-            _privateNetworkName = privateNic.nr.nameLabel;           
-            _host.privatePif = privateNic.pr.uuid;
-            _host.privateNetwork = privateNic.nr.uuid;
+            XsLocalNetwork privateNic = getManagementNetwork(conn);
+            _privateNetworkName = privateNic.getNetworkRecord(conn).nameLabel;
+            _host.privatePif = privateNic.getPifRecord(conn).uuid;
+            _host.privateNetwork = privateNic.getNetworkRecord(conn).uuid;
             
             _canBridgeFirewall = can_bridge_firewall(conn);
             
             _host.systemvmisouuid = null;
             
-            Nic guestNic = null;
+            XsLocalNetwork guestNic = null;
             if (_guestNetworkName != null && !_guestNetworkName.equals(_privateNetworkName)) {
-            	guestNic = getLocalNetwork(conn, _guestNetworkName);
+            	guestNic = getNetworkByName(conn, _guestNetworkName);
             	if (guestNic == null) {
             		s_logger.warn("Unable to find guest network " + _guestNetworkName);
             		throw new IllegalArgumentException("Unable to find guest network " + _guestNetworkName + " for host " + _host.ip);
@@ -3697,12 +3721,12 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
             	guestNic = privateNic;
             	_guestNetworkName = _privateNetworkName;
             }
-            _host.guestNetwork = guestNic.nr.uuid;
-            _host.guestPif = guestNic.pr.uuid;
+            _host.guestNetwork = guestNic.getNetworkRecord(conn).uuid;
+            _host.guestPif = guestNic.getPifRecord(conn).uuid;
 
-            Nic publicNic = null;
+            XsLocalNetwork publicNic = null;
             if (_publicNetworkName != null && !_publicNetworkName.equals(_guestNetworkName)) {
-                publicNic = getLocalNetwork(conn, _publicNetworkName);
+                publicNic = getNetworkByName(conn, _publicNetworkName);
                 if (publicNic == null) {
                     s_logger.warn("Unable to find public network " + _publicNetworkName + " for host " + _host.ip);
                     throw new IllegalArgumentException("Unable to find public network " + _publicNetworkName + " for host " + _host.ip);
@@ -3711,30 +3735,30 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
                 publicNic = guestNic;
                 _publicNetworkName = _guestNetworkName;
             }
-            _host.publicPif = publicNic.pr.uuid;
-            _host.publicNetwork = publicNic.nr.uuid;
+            _host.publicPif = publicNic.getPifRecord(conn).uuid;
+            _host.publicNetwork = publicNic.getNetworkRecord(conn).uuid;
 
-            Nic storageNic1 = null;
+            XsLocalNetwork storageNic1 = null;
             if (_storageNetworkName1 != null && !_storageNetworkName1.equals(_guestNetworkName)) {
-                storageNic1 = getLocalNetwork(conn, _storageNetworkName1);
+                storageNic1 = getNetworkByName(conn, _storageNetworkName1);
             }
             if (storageNic1 == null) {
                 storageNic1 = guestNic;
                 _storageNetworkName1 = _guestNetworkName;
             }
-            _host.storageNetwork1 = storageNic1.nr.uuid;
-            _host.storagePif1 = storageNic1.pr.uuid;
+            _host.storageNetwork1 = storageNic1.getNetworkRecord(conn).uuid;
+            _host.storagePif1 = storageNic1.getPifRecord(conn).uuid;
 
-            Nic storageNic2 = null;
+            XsLocalNetwork storageNic2 = null;
             if (_storageNetworkName2 != null && !_storageNetworkName2.equals(_guestNetworkName)) {
-                storageNic2 = getLocalNetwork(conn, _storageNetworkName2);
+                storageNic2 = getNetworkByName(conn, _storageNetworkName2);
             }
             if (storageNic2 == null) {
                 storageNic2 = guestNic;
                 _storageNetworkName2 = _guestNetworkName;
             }
-            _host.storageNetwork2 = storageNic2.nr.uuid;
-            _host.storagePif2 = storageNic2.pr.uuid;
+            _host.storageNetwork2 = storageNic2.getNetworkRecord(conn).uuid;
+            _host.storagePif2 = storageNic2.getPifRecord(conn).uuid;
                        
             s_logger.info("Private Network is " + _privateNetworkName + " for host " + _host.ip);
             s_logger.info("Guest Network is " + _guestNetworkName + " for host " + _host.ip);
@@ -5013,6 +5037,10 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
             }
         }
         try {
+            Set<VDI> snapshots = vdi.getSnapshots(conn);
+            for( VDI snapshot: snapshots ) {
+                snapshot.destroy(conn);
+            }           
             vdi.destroy(conn);
         } catch (Exception e) {
             String msg = "VDI destroy for " + volumeUUID + " failed due to " + e.toString();
@@ -5680,17 +5708,12 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
         if (uri != null) {
             String secondaryStorageMountPath = uri.getHost() + ":" + uri.getPath();
      
-            if (secondaryStorageMountPath == null) {
-                details = "Couldn't delete snapshot because the URL passed: " + secondaryStoragePoolURL
-                        + " is invalid.";
-            } else {
             details = deleteSnapshotBackup(conn, dcId, accountId, volumeId, secondaryStorageMountPath, backupUUID);
             success = (details != null && details.equals("1"));
             if (success) {
                 s_logger.debug("Successfully deleted snapshot backup " + backupUUID);
             }
         }
-        }
         return new DeleteSnapshotBackupAnswer(cmd, true, details);
     }
 
@@ -5738,16 +5761,12 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
         if (uri != null) {
             String secondaryStorageMountPath = uri.getHost() + ":" + uri.getPath();
 
-            if (secondaryStorageMountPath == null) {
-                details = "Couldn't delete snapshotsDir because the URL passed: " + secondaryStoragePoolURL + " is invalid.";
-            } else {
             details = deleteSnapshotsDir(conn, dcId, accountId, volumeId, secondaryStorageMountPath);
             success = (details != null && details.equals("1"));
             if (success) {
                 s_logger.debug("Successfully deleted snapshotsDir for volume: " + volumeId);
             }
         }
-        }
 
         return new Answer(cmd, success, details);
     }
@@ -6092,24 +6111,71 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
 
     
 
-    protected class Nic {
-        public Network n;
-        public Network.Record nr;
-        public PIF p;
-        public PIF.Record pr;
+    /**
+     * XsNic represents a network and the host's specific PIF.
+     */
+    protected class XsLocalNetwork {
+        private Network _n;
+        private Network.Record _nr;
+        private PIF _p;
+        private PIF.Record _pr;
 
-        public Nic(Network n, Network.Record nr, PIF p, PIF.Record pr) {
-            this.n = n;
-            this.nr = nr;
-            this.p = p;
-            this.pr = pr;
+        public XsLocalNetwork(Network n) {
+            this(n, null, null, null);
+        }
+        
+        public XsLocalNetwork(Network n, Network.Record nr, PIF p, PIF.Record pr) {
+            _n = n;
+            _nr = nr;
+            _p = p;
+            _pr = pr;
+        }
+        
+        public Network getNetwork() {
+            return _n;
+        }
+        
+        public Network.Record getNetworkRecord(Connection conn) throws XenAPIException, XmlRpcException {
+            if (_nr == null) {
+                _nr = _n.getRecord(conn);
+            }
+            
+            return _nr;
+        }
+        
+        public PIF getPif(Connection conn) throws XenAPIException, XmlRpcException {
+            if (_p == null) {
+                Network.Record nr = getNetworkRecord(conn);
+                for (PIF pif : nr.PIFs) {
+                    PIF.Record pr = pif.getRecord(conn);
+                    if (_host.uuid.equals(pr.host.getUuid(conn))) {
+                        if (s_logger.isDebugEnabled()) {
+                            s_logger.debug("Found a network called " + nr.nameLabel + " on host=" + _host.ip + ";  Network=" + nr.uuid + "; pif=" + pr.uuid);
+                        }
+                        _p = pif;
+                        _pr = pr;
+                        break;
+                    }
+                }
+            }
+            return _p;
+        }
+        
+        public PIF.Record getPifRecord(Connection conn) throws XenAPIException, XmlRpcException {
+            if (_pr == null) {
+                PIF p = getPif(conn);
+                if (_pr == null) {
+                    _pr = p.getRecord(conn);
+                }
+            }
+            return _pr;
         }
     }
 
     // A list of UUIDs that are gathered from the XenServer when
     // the resource first connects to XenServer. These UUIDs do
     // not change over time.
-    protected class XenServerHost {
+    protected class XsHost {
         public String systemvmisouuid;
         public String uuid;
         public String ip;

core/src/com/cloud/hypervisor/xen/resource/XenServer56Resource.java

@@ -416,7 +416,7 @@ public class XenServer56Resource extends CitrixResourceBase {
 
     @Override
     public StartupCommand[] initialize() {
-        pingxenserver();
+        pingXenServer();
         StartupCommand[] cmds = super.initialize();
         Connection conn = getConnection();       
         if (!setIptables(conn)) {

core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java

@@ -539,8 +539,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
             _params.put(StorageLayer.InstanceConfigKey, _storage);
             _dlMgr = new DownloadManagerImpl();
             _dlMgr.configure("DownloadManager", _params);
-            //_upldMgr = new UploadManagerImpl();
-            //_upldMgr.configure("UploadManager", params);
+            _upldMgr = new UploadManagerImpl();
+            _upldMgr.configure("UploadManager", params);
         } catch (ConfigurationException e) {
             s_logger.warn("Caught problem while configuring DownloadManager", e);
             return false;

core/src/com/cloud/storage/template/DownloadManagerImpl.java

@@ -608,6 +608,7 @@ public class DownloadManagerImpl implements DownloadManager {
 
     private List<String> listTemplates(String rootdir) {
         List<String> result = new ArrayList<String>();
+        
         Script script = new Script(listTmpltScr, s_logger);
         script.add("-r", rootdir);
         ZfsPathParser zpp = new ZfsPathParser(rootdir);
@@ -621,6 +622,11 @@ public class DownloadManagerImpl implements DownloadManager {
     public Map<String, TemplateInfo> gatherTemplateInfo(String rootDir) {
         Map<String, TemplateInfo> result = new HashMap<String, TemplateInfo>();
         String templateDir = rootDir + File.separator + _templateDir;
+        
+        if (! _storage.exists(templateDir)) {
+            _storage.mkdirs(templateDir);
+        }
+        
         List<String> publicTmplts = listTemplates(templateDir);
         for (String tmplt : publicTmplts) {
             String path = tmplt.substring(0, tmplt.lastIndexOf(File.separator));

core/src/com/cloud/storage/template/UploadManagerImpl.java

@@ -373,11 +373,10 @@ public class UploadManagerImpl implements UploadManager {
     	}
 
         
-        // Create a symbolic link from the actual directory to the template location. The entity would be directly visible under /var/www/html/userdata 
-        cmd.getInstallPath();
+        // Create a symbolic link from the actual directory to the template location. The entity would be directly visible under /var/www/html/userdata/cmd.getInstallPath();
         command = new Script("/bin/bash", s_logger);
         command.add("-c");
-        command.add("ln -sf " + extractMountPoint + File.separator + cmd.getInstallPath() + " " + extractDir + uuid);
+        command.add("ln -sf /mnt/SecStorage/" + cmd.getParent() + File.separator + cmd.getInstallPath() + " " + extractDir + uuid);
         result = command.execute();
         if (result != null) {
             String errorString = "Error in linking  err=" + result; 

core/src/com/cloud/vm/DomainRouterVO.java

@@ -90,8 +90,8 @@ public class DomainRouterVO extends VMInstanceVO implements VirtualRouter {
     }
 
     @Override
-    public long getDataCenterId() {
-        return dataCenterId;
+    public long getDataCenterIdToDeployIn() {
+        return dataCenterIdToDeployIn;
     }
     
     public String getPublicNetmask() {
@@ -123,6 +123,10 @@ public class DomainRouterVO extends VMInstanceVO implements VirtualRouter {
 		this.role = role;
 	}
 
+	@Override
+	public long getServiceOfferingId() {
+	    return serviceOfferingId;
+	}
     public void setServiceOfferingId(long serviceOfferingId) {
         this.serviceOfferingId = serviceOfferingId;
     }

core/src/com/cloud/vm/VMInstanceVO.java

@@ -93,15 +93,15 @@ public class VMInstanceVO implements VirtualMachine, FiniteStateObject<State, Vi
     protected Long lastHostId;
 
     @Column(name="pod_id", updatable=true, nullable=false)
-    protected Long podId;
+    protected Long podIdToDeployIn;
 
     @Column(name="private_mac_address", updatable=true, nullable=true)
     protected String privateMacAddress;
 
     @Column(name="data_center_id", updatable=true, nullable=false)
-    protected long dataCenterId;
+    protected long dataCenterIdToDeployIn;
 
-    @Column(name="type", updatable=false, nullable=false, length=32)
+    @Column(name="vm_type", updatable=false, nullable=false, length=32)
     @Enumerated(value=EnumType.STRING)
     protected Type type;
 
@@ -231,8 +231,8 @@ public class VMInstanceVO implements VirtualMachine, FiniteStateObject<State, Vi
 	}
 	
 	@Override
-    public long getDataCenterId() {
-	    return dataCenterId;
+    public long getDataCenterIdToDeployIn() {
+	    return dataCenterIdToDeployIn;
 	}
 	
 	@Override
@@ -335,7 +335,6 @@ public class VMInstanceVO implements VirtualMachine, FiniteStateObject<State, Vi
 		return lastHostId;
 	}
 	
-	@Override
     public void setLastHostId(Long lastHostId) {
 		this.lastHostId = lastHostId;
 	}
@@ -360,12 +359,12 @@ public class VMInstanceVO implements VirtualMachine, FiniteStateObject<State, Vi
     }
 
     @Override
-    public Long getPodId() {
-        return podId;
+    public Long getPodIdToDeployIn() {
+        return podIdToDeployIn;
     }
     
     public void setPodId(long podId) {
-        this.podId = podId;
+        this.podIdToDeployIn = podId;
     }
 
     public void setPrivateMacAddress(String privateMacAddress) {
@@ -373,7 +372,7 @@ public class VMInstanceVO implements VirtualMachine, FiniteStateObject<State, Vi
     }
 
     public void setDataCenterId(long dataCenterId) {
-        this.dataCenterId = dataCenterId;
+        this.dataCenterIdToDeployIn = dataCenterId;
     }
     
     public boolean isRemoved() {

core/test/com/cloud/agent/transport/RequestTest.java

@@ -42,8 +42,13 @@ public class RequestTest extends TestCase {
     private static final Logger s_logger = Logger.getLogger(RequestTest.class);
 
     public void testSerDeser() {
+        s_logger.info("Testing serializing and deserializing works as expected");
+        
+        s_logger.info("UpdateHostPasswordCommand should have two parameters that doesn't show in logging");
         UpdateHostPasswordCommand cmd1 = new UpdateHostPasswordCommand("abc", "def");
+        s_logger.info("SecStorageFirewallCfgCommand has a context map that shouldn't show up in debug level");
         SecStorageFirewallCfgCommand cmd2 = new SecStorageFirewallCfgCommand();
+        s_logger.info("GetHostStatsCommand should not show up at all in debug level");
         GetHostStatsCommand cmd3 = new GetHostStatsCommand("hostguid", "hostname", 101);
         cmd2.addPortConfig("abc", "24", true, "eth0");
         cmd2.addPortConfig("127.0.0.1", "44", false, "eth1");
@@ -113,6 +118,7 @@ public class RequestTest extends TestCase {
     }
 
     public void testDownload() {
+        s_logger.info("Testing Download answer");
         VMTemplateVO template = new VMTemplateVO(1, "templatename", ImageFormat.QCOW2, true, true, true, TemplateType.USER, "url", true, 32, 1, "chksum", "displayText", true, 30, true,
                 HypervisorType.KVM);
         DownloadCommand cmd = new DownloadCommand("secUrl", template, 30000000l);
@@ -128,6 +134,7 @@ public class RequestTest extends TestCase {
     }
 
     public void testLogging() {
+        s_logger.info("Testing Logging");
         GetHostStatsCommand cmd3 = new GetHostStatsCommand("hostguid", "hostname", 101);
         Request sreq = new Request(2, 3, new Command[] { cmd3 }, true, true);
         sreq.setSequence(1);

deps/.classpath

@@ -51,6 +51,6 @@
 	<classpathentry exported="true" kind="lib" path="vmware-vim25.jar"/>
 	<classpathentry exported="true" kind="lib" path="jetty-6.1.26.jar"/>
 	<classpathentry exported="true" kind="lib" path="jetty-util-6.1.26.jar"/>
-
+	<classpathentry exported="true" kind="lib" path="cloud-manageontap.jar"/>
 	<classpathentry kind="output" path="bin"/>
 </classpath>

python/lib/cloudutils/globalEnv.py

@@ -9,7 +9,7 @@ class globalEnv:
         #debug
         self.debug = False
         #management server IP
-        self.mgtSvr = None
+        self.mgtSvr = "myagent.cloud.com"
         #zone id or zone name
         self.zone = None
         #pod id or pod name

python/lib/cloudutils/syscfg.py

@@ -72,7 +72,11 @@ class sysConfigAgent(sysConfig):
             return True
 
         if os.geteuid() != 0:
-            raise CloudInternalException("Need to execute with root permission")
+            raise CloudInternalException("Need to execute with root permission\n")
+        
+        hostname = bash("hostname -f")
+        if not hostname.isSuccess():
+            raise CloudInternalException("Checking hostname ... [Failed]\nPlease edit /etc/hosts, add a Fully Qualified Domain Name as your hostname\n")
 
         kvmEnabled = self.svo.isKVMEnabled()
         if not kvmEnabled:
@@ -125,7 +129,7 @@ class sysConfigServer(sysConfig):
             raise CloudInternalException("Need to execute with root permission")
         hostname = bash("hostname -f")
         if not hostname.isSuccess():
-            raise CloudInternalException("Checking hostname ... [Failed]\nNeed to have a Fully Qualified Domain Name as your hostname")
+            raise CloudInternalException("Checking hostname ... [Failed]\nPlease edit /etc/hosts, add a Fully Qualified Domain Name as your hostname\n")
         return True
         
 class sysConfigServerRedhat(sysConfigServer):

scripts/vm/hypervisor/xenserver/copy_vhd_from_secondarystorage.sh

@@ -111,6 +111,12 @@ if [ $type == "nfs" ]; then
   uuid=$(uuidgen -r)
   desvhd=/var/run/sr-mount/$sruuid/$uuid
   copyvhd $desvhd $vhdfile 0 $type
+  $VHDUTIL set -n $desvhd -f "hidden" -v "0" > /dev/null
+  if [ $? -ne 0 ]; then
+    echo "21#failed to set hidden to 0  $desvhd"
+    cleanup
+    exit 0
+  fi
   mv /var/run/sr-mount/$sruuid/$uuid /var/run/sr-mount/$sruuid/${uuid}.vhd
   xe sr-scan uuid=$sruuid
   if [ $? -ne 0 ]; then
@@ -140,6 +146,12 @@ elif [ $type == "lvmoiscsi" -o $type == "lvm" -o $type == "lvmohba" ]; then
     exit 0
   fi
   copyvhd $desvhd $vhdfile $lvsize $type
+  $VHDUTIL set -n $desvhd -f "hidden" -v "0" > /dev/null
+  if [ $? -ne 0 ]; then
+    echo "22#failed to set hidden to 0  $desvhd"
+    cleanup
+    exit 0
+  fi
   xe sr-scan uuid=$sruuid
   if [ $? -ne 0 ]; then
     echo "14#failed to scan sr $sruuid"

scripts/vm/hypervisor/xenserver/xcpserver/patch

@@ -4,8 +4,7 @@
 # [Name of file]=[source path],[file permission],[destination path]
 # [destination path] is required.
 # If [file permission] is missing, 755 is assumed.
-# If [source path] is missing, it looks in the same
-# directory as the patch file.
+# If [source path] is missing, it looks in the same directory as the patch file.
 # If [source path] starts with '/', then it is absolute path.
 # If [source path] starts with '~', then it is path relative to management server home directory.
 # If [source path] does not start with '/' or '~', then it is relative path to the location of the patch file. 

server/src/com/cloud/acl/DomainChecker.java

@@ -82,7 +82,7 @@ public class DomainChecker extends AdapterBase implements SecurityChecker {
             // validate that the template is usable by the account
             if (!template.isPublicTemplate()) {
                 Account owner = _accountDao.findById(template.getAccountId());
-                if (BaseCmd.isAdmin(owner.getType()) || (owner.getId() == caller.getId())) {
+                if (BaseCmd.isRootAdmin(caller.getType()) || (owner.getId() == caller.getId())) {
                     return true;
                 }
                 

server/src/com/cloud/agent/manager/AgentAttache.java

@@ -17,11 +17,13 @@
  */
 package com.cloud.agent.manager;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
@@ -238,8 +240,25 @@ public abstract class AgentAttache {
         return _requests.size();
     }
     
-    public int getListenersSize() {
-        return _waitForList.size();
+    public int getNonRecurringListenersSize() {
+        List<Listener> nonRecurringListenersList = new ArrayList<Listener>();
+        if (_waitForList.isEmpty()) {
+            return 0;
+        } else {
+            final Set<Map.Entry<Long, Listener>> entries = _waitForList.entrySet();
+            final Iterator<Map.Entry<Long, Listener>> it = entries.iterator();
+            while (it.hasNext()) {
+                final Map.Entry<Long, Listener> entry = it.next();
+                final Listener monitor = entry.getValue();
+                if (!monitor.isRecurring()) {
+                    //TODO - remove this debug statement later
+                    s_logger.debug("Listener is " + entry.getValue() + " waiting on " + entry.getKey());
+                    nonRecurringListenersList.add(monitor);
+                }
+           }
+        }
+
+        return nonRecurringListenersList.size();
     }
 
     public boolean processAnswers(final long seq, final Response resp) {

server/src/com/cloud/agent/manager/AgentManagerImpl.java

@@ -82,6 +82,7 @@ import com.cloud.capacity.CapacityVO;
 import com.cloud.capacity.dao.CapacityDao;
 import com.cloud.cluster.ManagementServerNode;
 import com.cloud.cluster.StackMaid;
+import com.cloud.configuration.Config;
 import com.cloud.configuration.dao.ConfigurationDao;
 import com.cloud.dc.ClusterDetailsDao;
 import com.cloud.dc.ClusterDetailsVO;
@@ -275,6 +276,9 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, Manager {
 
         s_logger.info("Ping Timeout is " + _pingTimeout);
         
+        value = configs.get(Config.DirectAgentLoadSize.key());
+        int threads = NumbersUtil.parseInt(value, 16);
+
         _instance = configs.get("instance.name");
         if (_instance == null) {
             _instance = "DEFAULT";
@@ -287,7 +291,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, Manager {
         _monitor = new AgentMonitor(_nodeId, _hostDao, _vmDao, _dcDao, _podDao, this, _alertMgr, _pingTimeout);
         registerForHostEvents(_monitor, true, true, false);
 
-        _executor = new ThreadPoolExecutor(16, 100, 60l, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("AgentTaskPool"));
+        _executor = new ThreadPoolExecutor(threads, threads, 60l, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("AgentTaskPool"));
 
         _connection = new NioServer("AgentManager", _port, workers + 10, this);
 
@@ -489,9 +493,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, Manager {
     @Override
     public HostVO getSSAgent(HostVO ssHost) {
         if( ssHost.getType() == Host.Type.LocalSecondaryStorage ) {
-            if( ssHost.getStatus() == Status.Up ) {
             return  ssHost;
-            }
         } else if ( ssHost.getType() == Host.Type.SecondaryStorage) {
             Long dcId = ssHost.getDataCenterId();
             List<HostVO> ssAHosts = _hostDao.listBy(Host.Type.SecondaryStorageVM, dcId);
@@ -2155,7 +2157,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, Manager {
 
             Response response = null;
             if (attache == null) {
-                s_logger.debug("Processing sequence " + request.getSequence() + ": Processing " + request.toString());
+                request.logD("Processing the first command ");
                 if (!(cmd instanceof StartupCommand)) {
                     s_logger.warn("Throwing away a request because it came through as the first command on a connect: " + request.toString());
                     return;

server/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java

@@ -129,7 +129,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
 
         // schedule transfer scan executor - if agent LB is enabled
         if (_clusterMgr.isAgentRebalanceEnabled()) {
-            s_transferExecutor.scheduleAtFixedRate(getTransferScanTask(), ClusteredAgentRebalanceService.DEFAULT_TRANSFER_CHECK_INTERVAL, ClusteredAgentRebalanceService.DEFAULT_TRANSFER_CHECK_INTERVAL,
+            s_transferExecutor.scheduleAtFixedRate(getTransferScanTask(), 60000, ClusteredAgentRebalanceService.DEFAULT_TRANSFER_CHECK_INTERVAL,
                     TimeUnit.MILLISECONDS);
         }
 
@@ -524,7 +524,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
             }
         }
         if (agent == null) {
-            throw new AgentUnavailableException("Host is not in the right state", hostId);
+            throw new AgentUnavailableException("Host is not in the right state: " + host.getStatus() , hostId);
         }
 
         return agent;
@@ -698,45 +698,48 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
     @Override
     public void startRebalanceAgents() {
         
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Management server " + _nodeId + " was asked to do agent rebalancing; checking how many hosts can be taken from this server");
-        }
-        
         List<ManagementServerHostVO> allMS = _mshostDao.listBy(ManagementServerHost.State.Up, ManagementServerHost.State.Starting);
-        List<HostVO> allManagedAgents = _hostDao.listManagedAgents();
+        List<HostVO> allManagedAgents = _hostDao.listManagedRoutingAgents();
 
-        long avLoad = 0L;
+        int avLoad = 0;
 
         if (!allManagedAgents.isEmpty() && !allMS.isEmpty()) {
             avLoad = allManagedAgents.size() / allMS.size();
         } else {
             if (s_logger.isDebugEnabled()) {
-                s_logger.debug("Management server " + _nodeId + " found no hosts to rebalance. Current number of active management server nodes in the system is " + allMS.size() + "; number of managed agents is " + allMS.size());
+                s_logger.debug("There are no hosts to rebalance in the system. Current number of active management server nodes in the system is " + allMS.size() + "; number of managed agents is " + allManagedAgents.size());
             }
             return;
         }
         
+        if (avLoad == 0L) {
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug("As calculated average load is less than 1, rounding it to 1");
+            }
+            avLoad = 1;
+        }
+
         for (ManagementServerHostVO node : allMS) {
             if (node.getMsid() != _nodeId) {
                 
                 List<HostVO> hostsToRebalance = new ArrayList<HostVO>();
                 for (AgentLoadBalancerPlanner lbPlanner : _lbPlanners) {
                     hostsToRebalance = lbPlanner.getHostsToRebalance(node.getMsid(), avLoad);
-                    if (!hostsToRebalance.isEmpty()) {
+                    if (hostsToRebalance != null && !hostsToRebalance.isEmpty()) {
                         break;
                     } else {
-                        s_logger.debug("Agent load balancer planner " + lbPlanner.getName() + " found no hosts to be rebalanced from management server " + _nodeId);
+                        s_logger.debug("Agent load balancer planner " + lbPlanner.getName() + " found no hosts to be rebalanced from management server " + node.getMsid());
                     }
                 }
 
-                if (!hostsToRebalance.isEmpty()) {
+                if (hostsToRebalance != null && !hostsToRebalance.isEmpty()) {
                     for (HostVO host : hostsToRebalance) {
                         long hostId = host.getId();
                         s_logger.debug("Asking management server " + node.getMsid() + " to give away host id=" + hostId);
                         boolean result = true;
-                        HostTransferMapVO transfer = _hostTransferDao.startAgentTransfering(hostId, _nodeId, node.getMsid());
+                        HostTransferMapVO transfer = _hostTransferDao.startAgentTransfering(hostId, node.getMsid(), _nodeId);
                         try {
-                            Answer[] answer = sendRebalanceCommand(hostId, _nodeId, Event.RequestAgentRebalance);
+                            Answer[] answer = sendRebalanceCommand(hostId, node.getMsid(), Event.RequestAgentRebalance);
                             if (answer == null) {
                                 s_logger.warn("Failed to get host id=" + hostId + " from management server " + node.getMsid());
                                 result = false;
@@ -756,7 +759,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                         }
                     }
                 } else {
-                    s_logger.debug("Management server " + _nodeId + " found no hosts to rebalance.");
+                    s_logger.debug("Found no hosts to rebalance from the management server " + node.getMsid());
                 }
             }
         }
@@ -797,10 +800,11 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                         s_logger.debug("Found " + _agentToTransferIds.size() + " agents to transfer");
                         for (Long hostId : _agentToTransferIds) {
                             AgentAttache attache = findAttache(hostId);
-                            if (attache.getQueueSize() == 0 && attache.getListenersSize() == 0) {
-                                boolean result = true;
+                            if (attache.getQueueSize() == 0 && attache.getNonRecurringListenersSize() == 0) {
+                                boolean result = false;
                                 _agentToTransferIds.remove(hostId);
                                 try {
+                                    _hostTransferDao.startAgentTransfer(hostId);
                                     result = rebalanceHost(hostId);
                                 } finally {
                                     if (result) {
@@ -810,15 +814,15 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                                     }
                                 }
                             } else {
-                                // if we timed out waiting for the host to reconnect, remove host from rebalance list and mark it as failed to rebalance
+                                // if we timed out waiting for the host to reconnect, remove host from rebalance list and delete from op_host_transfer DB
                                 // no need to do anything with the real attache
                                 Date cutTime = DateUtil.currentGMTTime();
-                                if (!(_hostTransferDao.isActive(hostId, new Date(cutTime.getTime() - rebalanceTimeOut)))) {
+                                if (_hostTransferDao.isNotActive(hostId, new Date(cutTime.getTime() - rebalanceTimeOut))) {
                                     s_logger.debug("Timed out waiting for the host id=" + hostId + " to be ready to transfer, failing rebalance for this host");
                                     _agentToTransferIds.remove(hostId);
-                                    HostTransferMapVO transferMap = _hostTransferDao.findById(hostId);
-                                    transferMap.setState(HostTransferState.TransferFailed);
-                                    _hostTransferDao.update(hostId, transferMap);
+                                    _hostTransferDao.completeAgentTransfer(hostId);
+                                } else {
+                                    s_logger.debug("Agent " + hostId + " can't be transfered yet as its request queue size is " + attache.getQueueSize() + " and listener queue size is " + attache.getNonRecurringListenersSize());
                                 }
                             }
                         }
@@ -859,7 +863,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                 _agents.put(hostId, attache);
                 if (host != null && host.getRemoved() == null) {
                     host.setManagementServerId(null);
-                    s_logger.debug("Updating host id=" + hostId + " with the status " + Status.Rebalance);
+                    s_logger.debug("Updating host id=" + hostId + " with the status " + Status.Rebalancing);
                     _hostDao.updateStatus(host, Event.StartAgentRebalance, _nodeId);
                 }
                 
@@ -877,6 +881,9 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                     s_logger.debug("Got host id=" + hostId + " from management server " + map.getFutureOwner());
                 } 
                 
+            } else {
+                s_logger.warn("Unable to find agent " + hostId + " on management server " + _nodeId);
+                result = false;
             }
         } else if (map.getFutureOwner() == _nodeId) {
             try {
@@ -887,61 +894,57 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                 loadDirectlyConnectedHost(host);
             } catch (Exception ex) {
                 s_logger.warn("Unable to load directly connected host " + host.getId() + " as a part of rebalance due to exception: ", ex);
+                result = false;
             }
         }
         
         return result;
-    
     }
     
     private boolean finishRebalance(final long hostId, Event event) {
         HostTransferMapVO map = _hostTransferDao.findById(hostId);
-        AgentAttache attache = findAttache(hostId);
-        
-        if (attache == null) {
-            s_logger.debug("Unable to find attache for the host id=" + hostId + ", assuming that the agent disconnected already");
-            HostTransferState state = (event == Event.RebalanceCompleted) ? HostTransferState.TransferCompleted : HostTransferState.TransferFailed;
-            map.setState(state);
-            _hostTransferDao.update(hostId, map); 
-            return true;
-        }
         
         if (map.getInitialOwner() != _nodeId) {
             s_logger.warn("Why finish rebalance called not by initial host owner???");
             return false;
         }  
 
+        boolean success = (event == Event.RebalanceCompleted) ? true : false;
         if (s_logger.isDebugEnabled()) { 
-            s_logger.debug("Finishing rebalancing for the host id=" + hostId);
+            s_logger.debug("Finishing rebalancing for the agent " + hostId + " with result " + success);
         }
         
-        if (event == Event.RebalanceFailed) {
-            ((ClusteredDirectAgentAttache) attache).setTransferMode(false);
-            s_logger.debug("Rebalance failed for the host id=" + hostId);
-            map.setState(HostTransferState.TransferFailed);
-            _hostTransferDao.update(hostId, map);
-        } else if (event == Event.RebalanceCompleted) {
-
-            //1) Get all the requests remove transfer attache
+        AgentAttache attache = findAttache(hostId);
+        if (attache == null) {
+            s_logger.debug("Unable to find attache for the host id=" + hostId + ", assuming that the agent disconnected already");
+            return true;
+        } else if (success) {
+            s_logger.debug("Management server " + _nodeId + " is completing agent " + hostId + " rebalance");
+            //1) Get all the requests before removing transfer attache
             LinkedList<Request> requests = ((ClusteredDirectAgentAttache) attache).getRequests();
-            removeAgent(attache, Status.Rebalance);
+            removeAgent(attache, Status.Rebalancing);
             
             //2) Create forward attache
-            createAttache(hostId);
-            
+            try {
+                getAttache(hostId);
                 //3) forward all the requests to the management server which owns the host now
                 if (!requests.isEmpty()) {
+                    s_logger.debug("Forwarding requests held in transfer attache " + hostId + " from the management server " + _nodeId + " to " + map.getFutureOwner());
                     for (Request request : requests) {
                         routeToPeer(Long.toString(map.getFutureOwner()), request.getBytes());
                     }
                 }
-            
-            map.setState(HostTransferState.TransferCompleted);
-            _hostTransferDao.update(hostId, map); 
-            
-            return true;
-            
+            } catch (AgentUnavailableException ex) {
+                s_logger.warn("Not creating forward attache as agent is not available", ex);
+                //TODO - - have to handle the case when requests can't be forwarded due to lack of forward attache
             }
+            
+        } else {
+            ((ClusteredDirectAgentAttache) attache).setTransferMode(false);
+            //TODO - have to handle the case when agent fails to rebalance 1) Either connect it back 2) Or disconnect it
+        }
+        
+        _hostTransferDao.completeAgentTransfer(hostId);
         return true;
     }
 }

server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java

@@ -73,7 +73,7 @@ public class ConsoleProxyAlertAdapter implements AlertAdapter {
 			_alertMgr.sendAlert(
 				AlertManager.ALERT_TYPE_CONSOLE_PROXY,
 				args.getZoneId(),
-				proxy.getPodId(),
+				proxy.getPodIdToDeployIn(),
 				"Console proxy up in zone: " + dc.getName() + ", proxy: " + proxy.getHostName() + ", public IP: " + proxy.getPublicIpAddress() 
 				 	+ ", private IP: " + (proxy.getPrivateIpAddress() == null ? "N/A" : proxy.getPrivateIpAddress()),
 			 	"Console proxy up (zone " + dc.getName() + ")" 	
@@ -89,7 +89,7 @@ public class ConsoleProxyAlertAdapter implements AlertAdapter {
 			_alertMgr.sendAlert(
 				AlertManager.ALERT_TYPE_CONSOLE_PROXY,
 				args.getZoneId(),
-				proxy.getPodId(),
+				proxy.getPodIdToDeployIn(),
 				"Console proxy down in zone: " + dc.getName() + ", proxy: " + proxy.getHostName() + ", public IP: " + proxy.getPublicIpAddress() 
 				 	+ ", private IP: " + (proxy.getPrivateIpAddress() == null ? "N/A" : proxy.getPrivateIpAddress()),
 			 	"Console proxy down (zone " + dc.getName() + ")" 	
@@ -105,7 +105,7 @@ public class ConsoleProxyAlertAdapter implements AlertAdapter {
 			_alertMgr.sendAlert(
 				AlertManager.ALERT_TYPE_CONSOLE_PROXY,
 				args.getZoneId(),
-				proxy.getPodId(),
+				proxy.getPodIdToDeployIn(),
 				"Console proxy rebooted in zone: " + dc.getName() + ", proxy: " + proxy.getHostName() + ", public IP: " + proxy.getPublicIpAddress() 
 				 	+ ", private IP: " + (proxy.getPrivateIpAddress() == null ? "N/A" : proxy.getPrivateIpAddress()),
 			 	"Console proxy rebooted (zone " + dc.getName() + ")" 	
@@ -121,7 +121,7 @@ public class ConsoleProxyAlertAdapter implements AlertAdapter {
 			_alertMgr.sendAlert(
 				AlertManager.ALERT_TYPE_CONSOLE_PROXY,
 				args.getZoneId(),
-				proxy.getPodId(),
+				proxy.getPodIdToDeployIn(),
 				"Console proxy creation failure. zone: " + dc.getName() + ", proxy: " + proxy.getHostName() + ", public IP: " + proxy.getPublicIpAddress() 
 				 	+ ", private IP: " + (proxy.getPrivateIpAddress() == null ? "N/A" : proxy.getPrivateIpAddress()) 
 				 	+ ", error details: " + args.getMessage(),
@@ -138,7 +138,7 @@ public class ConsoleProxyAlertAdapter implements AlertAdapter {
 			_alertMgr.sendAlert(
 				AlertManager.ALERT_TYPE_CONSOLE_PROXY,
 				args.getZoneId(),
-				proxy.getPodId(),
+				proxy.getPodIdToDeployIn(),
 				"Console proxy startup failure. zone: " + dc.getName() + ", proxy: " + proxy.getHostName() + ", public IP: " + proxy.getPublicIpAddress() 
 				 	+ ", private IP: " + (proxy.getPrivateIpAddress() == null ? "N/A" : proxy.getPrivateIpAddress()) 
 				 	+ ", error details: " + args.getMessage(),
@@ -155,7 +155,7 @@ public class ConsoleProxyAlertAdapter implements AlertAdapter {
 			_alertMgr.sendAlert(
 				AlertManager.ALERT_TYPE_CONSOLE_PROXY,
 				args.getZoneId(),
-				proxy.getPodId(),
+				proxy.getPodIdToDeployIn(),
 				"Failed to open console proxy firewall port. zone: " + dc.getName() + ", proxy: " + proxy.getHostName() 
 					+ ", public IP: " + proxy.getPublicIpAddress() 
 					+ ", private IP: " + (proxy.getPrivateIpAddress() == null ? "N/A" : proxy.getPrivateIpAddress()),
@@ -172,7 +172,7 @@ public class ConsoleProxyAlertAdapter implements AlertAdapter {
 			_alertMgr.sendAlert(
 				AlertManager.ALERT_TYPE_STORAGE_MISC,
 				args.getZoneId(),
-				proxy.getPodId(),
+				proxy.getPodIdToDeployIn(),
 				"Console proxy storage issue. zone: " + dc.getName() + ", message: " + args.getMessage(),
 				"Console proxy alert (zone " + dc.getName() + ")"
 			);

server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java

@@ -73,7 +73,7 @@ public class SecondaryStorageVmAlertAdapter implements AlertAdapter {
 			_alertMgr.sendAlert(
 				AlertManager.ALERT_TYPE_SSVM,
 				args.getZoneId(),
-				secStorageVm.getPodId(),
+				secStorageVm.getPodIdToDeployIn(),
 				"Secondary Storage Vm up in zone: " + dc.getName() + ", secStorageVm: " + secStorageVm.getHostName() + ", public IP: " + secStorageVm.getPublicIpAddress() 
 				 	+ ", private IP: " + (secStorageVm.getPrivateIpAddress() == null ? "N/A" : secStorageVm.getPrivateIpAddress()),
 			 	"Secondary Storage Vm up (zone " + dc.getName() + ")" 	
@@ -89,7 +89,7 @@ public class SecondaryStorageVmAlertAdapter implements AlertAdapter {
 			_alertMgr.sendAlert(
 				AlertManager.ALERT_TYPE_SSVM,
 				args.getZoneId(),
-				secStorageVm.getPodId(),
+				secStorageVm.getPodIdToDeployIn(),
 				"Secondary Storage Vm down in zone: " + dc.getName() + ", secStorageVm: " + secStorageVm.getHostName() + ", public IP: " + secStorageVm.getPublicIpAddress() 
 				 	+ ", private IP: " + (secStorageVm.getPrivateIpAddress() == null ? "N/A" : secStorageVm.getPrivateIpAddress()),
 			 	"Secondary Storage Vm down (zone " + dc.getName() + ")" 	
@@ -105,7 +105,7 @@ public class SecondaryStorageVmAlertAdapter implements AlertAdapter {
 			_alertMgr.sendAlert(
 				AlertManager.ALERT_TYPE_SSVM,
 				args.getZoneId(),
-				secStorageVm.getPodId(),
+				secStorageVm.getPodIdToDeployIn(),
 				"Secondary Storage Vm rebooted in zone: " + dc.getName() + ", secStorageVm: " + secStorageVm.getHostName() + ", public IP: " + secStorageVm.getPublicIpAddress() 
 				 	+ ", private IP: " + (secStorageVm.getPrivateIpAddress() == null ? "N/A" : secStorageVm.getPrivateIpAddress()),
 			 	"Secondary Storage Vm rebooted (zone " + dc.getName() + ")" 	
@@ -121,7 +121,7 @@ public class SecondaryStorageVmAlertAdapter implements AlertAdapter {
 			_alertMgr.sendAlert(
 				AlertManager.ALERT_TYPE_SSVM,
 				args.getZoneId(),
-				secStorageVm.getPodId(),
+				secStorageVm.getPodIdToDeployIn(),
 				"Secondary Storage Vm creation failure. zone: " + dc.getName() + ", secStorageVm: " + secStorageVm.getHostName() + ", public IP: " + secStorageVm.getPublicIpAddress() 
 				 	+ ", private IP: " + (secStorageVm.getPrivateIpAddress() == null ? "N/A" : secStorageVm.getPrivateIpAddress()) 
 				 	+ ", error details: " + args.getMessage(),
@@ -138,7 +138,7 @@ public class SecondaryStorageVmAlertAdapter implements AlertAdapter {
 			_alertMgr.sendAlert(
 				AlertManager.ALERT_TYPE_SSVM,
 				args.getZoneId(),
-				secStorageVm.getPodId(),
+				secStorageVm.getPodIdToDeployIn(),
 				"Secondary Storage Vm startup failure. zone: " + dc.getName() + ", secStorageVm: " + secStorageVm.getHostName() + ", public IP: " + secStorageVm.getPublicIpAddress() 
 				 	+ ", private IP: " + (secStorageVm.getPrivateIpAddress() == null ? "N/A" : secStorageVm.getPrivateIpAddress()) 
 				 	+ ", error details: " + args.getMessage(),
@@ -155,7 +155,7 @@ public class SecondaryStorageVmAlertAdapter implements AlertAdapter {
 			_alertMgr.sendAlert(
 				AlertManager.ALERT_TYPE_SSVM,
 				args.getZoneId(),
-				secStorageVm.getPodId(),
+				secStorageVm.getPodIdToDeployIn(),
 				"Failed to open secondary storage vm firewall port. zone: " + dc.getName() + ", secStorageVm: " + secStorageVm.getHostName() 
 					+ ", public IP: " + secStorageVm.getPublicIpAddress() 
 					+ ", private IP: " + (secStorageVm.getPrivateIpAddress() == null ? "N/A" : secStorageVm.getPrivateIpAddress()),
@@ -172,7 +172,7 @@ public class SecondaryStorageVmAlertAdapter implements AlertAdapter {
 			_alertMgr.sendAlert(
 				AlertManager.ALERT_TYPE_STORAGE_MISC,
 				args.getZoneId(),
-				secStorageVm.getPodId(),
+				secStorageVm.getPodIdToDeployIn(),
 				"Secondary Storage Vm storage issue. zone: " + dc.getName() + ", message: " + args.getMessage(),
 				"Secondary Storage Vm alert (zone " + dc.getName() + ")"
 			);

server/src/com/cloud/api/ApiDBUtils.java

@@ -478,18 +478,13 @@ public class ApiDBUtils {
         return _volumeDao.getHypervisorType(volumeId);
     }
 
-    public static List<VMTemplateHostVO> listTemplateHostBy(long templateId, Long zoneId) {
+    public static List<VMTemplateHostVO> listTemplateHostBy(long templateId, Long zoneId, boolean readyOnly) {
         if (zoneId != null) {
             VMTemplateVO vmTemplate = findTemplateById(templateId);
             if (vmTemplate.getHypervisorType() == HypervisorType.BareMetal) {
                 return _templateHostDao.listByTemplateId(templateId);
             } else {
-                HostVO secondaryStorageHost = _storageMgr.getSecondaryStorageHost(zoneId);
-                if (secondaryStorageHost == null) {
-                    return new ArrayList<VMTemplateHostVO>();
-                } else {
-                    return _templateHostDao.listByHostTemplate(secondaryStorageHost.getId(), templateId);
-                }
+                return _templateHostDao.listByZoneTemplate(zoneId, templateId, readyOnly);
             }
         } else {
             return _templateHostDao.listByOnlyTemplateId(templateId);

server/src/com/cloud/api/ApiResponseHelper.java

@@ -17,6 +17,7 @@
  */
 package com.cloud.api;
 
+import java.net.URLEncoder;
 import java.text.DecimalFormat;
 import java.util.ArrayList;
 import java.util.Date;
@@ -219,7 +220,7 @@ public class ApiResponseHelper implements ResponseGenerator {
         boolean accountIsAdmin = (account.getType() == Account.ACCOUNT_TYPE_ADMIN);
         AccountResponse accountResponse = new AccountResponse();
         accountResponse.setId(account.getId());
-        accountResponse.setName(account.getAccountName());
+        accountResponse.setName(encodeParam(account.getAccountName()));
         accountResponse.setAccountType(account.getType());
         accountResponse.setDomainId(account.getDomainId());
         accountResponse.setDomainName(ApiDBUtils.findDomainById(account.getDomainId()).getName());
@@ -316,20 +317,20 @@ public class ApiResponseHelper implements ResponseGenerator {
         List<UserResponse> userResponseList = new ArrayList<UserResponse>();
         for (UserVO user : usersForAccount) {
             UserResponse userResponse = new UserResponse();
-            userResponse.setAccountName(account.getAccountName());
+            userResponse.setAccountName(encodeParam(account.getAccountName()));
             userResponse.setAccountType(account.getType());
             userResponse.setApiKey(user.getApiKey());
             userResponse.setCreated(user.getCreated());
             userResponse.setDomainId(account.getDomainId());
             userResponse.setDomainName(ApiDBUtils.findDomainById(account.getDomainId()).getName());
-            userResponse.setEmail(user.getEmail());
+            userResponse.setEmail(encodeParam(user.getEmail()));
             userResponse.setFirstname(user.getFirstname());
             userResponse.setId(user.getId());
             userResponse.setSecretKey(user.getSecretKey());
             userResponse.setLastname(user.getLastname());
             userResponse.setState(user.getState().toString());
             userResponse.setTimezone(user.getTimezone());
-            userResponse.setUsername(user.getUsername());
+            userResponse.setUsername(encodeParam(user.getUsername()));
 
             userResponseList.add(userResponse);
         }
@@ -1058,9 +1059,9 @@ public class ApiResponseHelper implements ResponseGenerator {
             }
 
             // Data Center Info
-            DataCenter zone = dataCenters.get(userVm.getDataCenterId());
+            DataCenter zone = dataCenters.get(userVm.getDataCenterIdToDeployIn());
             if (zone == null) {
-                zone = ApiDBUtils.findZoneById(userVm.getDataCenterId());
+                zone = ApiDBUtils.findZoneById(userVm.getDataCenterIdToDeployIn());
                 dataCenters.put(zone.getId(), zone);
             }
 
@@ -1223,11 +1224,13 @@ public class ApiResponseHelper implements ResponseGenerator {
 
     @Override
     public DomainRouterResponse createDomainRouterResponse(VirtualRouter router) {
+    	Map<Long, ServiceOffering> serviceOfferings = new HashMap<Long, ServiceOffering>();
+    	
         DomainRouterResponse routerResponse = new DomainRouterResponse();
         routerResponse.setId(router.getId());
-        routerResponse.setZoneId(router.getDataCenterId());
+        routerResponse.setZoneId(router.getDataCenterIdToDeployIn());
         routerResponse.setName(router.getHostName());
-        routerResponse.setPodId(router.getPodId());
+        routerResponse.setPodId(router.getPodIdToDeployIn());
         routerResponse.setTemplateId(router.getTemplateId());
         routerResponse.setCreated(router.getCreated());
         routerResponse.setState(router.getState());
@@ -1237,6 +1240,16 @@ public class ApiResponseHelper implements ResponseGenerator {
             routerResponse.setHostName(ApiDBUtils.findHostById(router.getHostId()).getName());
         }
           
+        // Service Offering Info
+        ServiceOffering offering = serviceOfferings.get(router.getServiceOfferingId());
+
+        if (offering == null) {
+            offering = ApiDBUtils.findServiceOfferingById(router.getServiceOfferingId());
+            serviceOfferings.put(offering.getId(), offering);
+        }
+        routerResponse.setServiceOfferingId(offering.getId());
+        routerResponse.setServiceOfferingName(offering.getName());
+              
         Account accountTemp = ApiDBUtils.findAccountById(router.getAccountId());
         if (accountTemp != null) {
             routerResponse.setAccountName(accountTemp.getAccountName());
@@ -1268,7 +1281,7 @@ public class ApiResponseHelper implements ResponseGenerator {
                 }
             }
         }
-        DataCenter zone = ApiDBUtils.findZoneById(router.getDataCenterId());
+        DataCenter zone = ApiDBUtils.findZoneById(router.getDataCenterIdToDeployIn());
         if (zone != null) {
             routerResponse.setZoneName(zone.getName());
             routerResponse.setDns1(zone.getDns1());
@@ -1287,10 +1300,10 @@ public class ApiResponseHelper implements ResponseGenerator {
 
             vmResponse.setId(vm.getId());
             vmResponse.setSystemVmType(vm.getType().toString().toLowerCase());
-            vmResponse.setZoneId(vm.getDataCenterId());
+            vmResponse.setZoneId(vm.getDataCenterIdToDeployIn());
 
             vmResponse.setName(vm.getHostName());
-            vmResponse.setPodId(vm.getPodId());
+            vmResponse.setPodId(vm.getPodIdToDeployIn());
             vmResponse.setTemplateId(vm.getTemplateId());
             vmResponse.setCreated(vm.getCreated());
 
@@ -1309,7 +1322,7 @@ public class ApiResponseHelper implements ResponseGenerator {
                 vmResponse.setActiveViewerSessions(proxy.getActiveSession());
             }
 
-            DataCenter zone = ApiDBUtils.findZoneById(vm.getDataCenterId());
+            DataCenter zone = ApiDBUtils.findZoneById(vm.getDataCenterIdToDeployIn());
             if (zone != null) {
                 vmResponse.setZoneName(zone.getName());
                 vmResponse.setDns1(zone.getDns1());
@@ -1328,6 +1341,7 @@ public class ApiResponseHelper implements ResponseGenerator {
                         vmResponse.setPublicIp(singleNicProfile.getIp4Address());
                         vmResponse.setPublicMacAddress(singleNicProfile.getMacAddress());
                         vmResponse.setPublicNetmask(singleNicProfile.getNetmask());
+                        vmResponse.setGateway(singleNicProfile.getGateway());
                     } else if (network.getTrafficType() == TrafficType.Management) {
                         vmResponse.setPrivateIp(singleNicProfile.getIp4Address());
                         vmResponse.setPrivateMacAddress(singleNicProfile.getMacAddress());
@@ -1437,12 +1451,21 @@ public class ApiResponseHelper implements ResponseGenerator {
     }
 
     @Override
-    public void createTemplateResponse(List<TemplateResponse> responses, Pair<Long, Long> templateZonePair, boolean isAdmin, Account account) {
-        List<VMTemplateHostVO> templateHostRefsForTemplate = ApiDBUtils.listTemplateHostBy(templateZonePair.first(), templateZonePair.second());
+    public void createTemplateResponse(List<TemplateResponse> responses, Pair<Long, Long> templateZonePair, boolean isAdmin, Account account, boolean readyOnly) {
+        List<VMTemplateHostVO> templateHostRefsForTemplate = ApiDBUtils.listTemplateHostBy(templateZonePair.first(), templateZonePair.second(), readyOnly);
         VMTemplateVO template = ApiDBUtils.findTemplateById(templateZonePair.first());
 
         for (VMTemplateHostVO templateHostRef : templateHostRefsForTemplate) {
-
+            if (readyOnly) {
+                if (templateHostRef.getDownloadState() != Status.DOWNLOADED) {
+                    continue;
+                }
+                for (TemplateResponse res : responses) {
+                    if (res.getId() == templateHostRef.getTemplateId()) {
+                        continue;
+                    }
+                }
+            }
             TemplateResponse templateResponse = new TemplateResponse();
             templateResponse.setId(template.getId());
             templateResponse.setName(template.getName());
@@ -1479,6 +1502,9 @@ public class ApiResponseHelper implements ResponseGenerator {
             }
 
             HostVO host = ApiDBUtils.findHostById(templateHostRef.getHostId());
+            templateResponse.setHostId(host.getId());
+            templateResponse.setHostName(host.getName());
+            
             DataCenterVO datacenter = ApiDBUtils.findZoneById(host.getDataCenterId());
             
             // Add the zone ID
@@ -1974,7 +2000,7 @@ public class ApiResponseHelper implements ResponseGenerator {
     }
 
     @Override
-    public ListResponse<TemplateResponse> createIsoResponse(Set<Pair<Long, Long>> isoZonePairSet, boolean isAdmin, Account account, Boolean isBootable) {
+    public ListResponse<TemplateResponse> createIsoResponse(Set<Pair<Long, Long>> isoZonePairSet, boolean isAdmin, Account account, Boolean isBootable, boolean readyOnly) {
 
         ListResponse<TemplateResponse> response = new ListResponse<TemplateResponse>();
         List<TemplateResponse> isoResponses = new ArrayList<TemplateResponse>();
@@ -2010,7 +2036,7 @@ public class ApiResponseHelper implements ResponseGenerator {
                 }
             }
 
-            List<VMTemplateHostVO> isoHosts = ApiDBUtils.listTemplateHostBy(iso.getId(), isoZonePair.second());
+            List<VMTemplateHostVO> isoHosts = ApiDBUtils.listTemplateHostBy(iso.getId(), isoZonePair.second(), readyOnly);
             for (VMTemplateHostVO isoHost : isoHosts) {
                 TemplateResponse isoResponse = new TemplateResponse();
                 isoResponse.setId(iso.getId());
@@ -2501,4 +2527,13 @@ public class ApiResponseHelper implements ResponseGenerator {
             return sg.getId();
         }
     }
+    
+    private String encodeParam(String value) {
+		try {
+			return URLEncoder.encode(value, "UTF-8").replaceAll("\\+", "%20");
+		} catch (Exception e) {
+			s_logger.warn("Unable to encode: " + value);
+		}
+		return value;
+	}
 }

server/src/com/cloud/async/AsyncJobManagerImpl.java

@@ -146,18 +146,20 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
 
     @Override @DB
     public void completeAsyncJob(long jobId, int jobStatus, int resultCode, Object resultObject) {
-    	if(s_logger.isDebugEnabled())
+    	if(s_logger.isDebugEnabled()) {
             s_logger.debug("Complete async job-" + jobId + ", jobStatus: " + jobStatus +
     			", resultCode: " + resultCode + ", result: " + resultObject);
+        }
     	
     	Transaction txt = Transaction.currentTxn();
     	try {
     		txt.start();
     		AsyncJobVO job = _jobDao.findById(jobId);
     		if(job == null) {
-    	    	if(s_logger.isDebugEnabled())
+    	    	if(s_logger.isDebugEnabled()) {
                     s_logger.debug("job-" + jobId + " no longer exists, we just log completion info here. " + jobStatus +
     	    			", resultCode: " + resultCode + ", result: " + resultObject);
+                }
     			
     			txt.rollback();
     			return;
@@ -186,25 +188,28 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
 
     @Override @DB
     public void updateAsyncJobStatus(long jobId, int processStatus, Object resultObject) {
-    	if(s_logger.isDebugEnabled())
+    	if(s_logger.isDebugEnabled()) {
             s_logger.debug("Update async-job progress, job-" + jobId + ", processStatus: " + processStatus +
     			", result: " + resultObject);
+        }
     	
     	Transaction txt = Transaction.currentTxn();
     	try {
     		txt.start();
     		AsyncJobVO job = _jobDao.findById(jobId);
     		if(job == null) {
-    	    	if(s_logger.isDebugEnabled())
+    	    	if(s_logger.isDebugEnabled()) {
                     s_logger.debug("job-" + jobId + " no longer exists, we just log progress info here. progress status: " + processStatus);
+                }
     			
     			txt.rollback();
     			return;
     		}
     		
     		job.setProcessStatus(processStatus);
-    		if(resultObject != null)
+    		if(resultObject != null) {
                 job.setResult(ApiSerializerHelper.toSerializedStringOld(resultObject));
+            }
     		job.setLastUpdated(DateUtil.currentGMTTime());
     		_jobDao.update(jobId, job);
     		txt.commit();
@@ -216,9 +221,10 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
 
     @Override @DB
     public void updateAsyncJobAttachment(long jobId, String instanceType, Long instanceId) {
-    	if(s_logger.isDebugEnabled())
+    	if(s_logger.isDebugEnabled()) {
             s_logger.debug("Update async-job attachment, job-" + jobId + ", instanceType: " + instanceType +
     			", instanceId: " + instanceId);
+        }
     	
     	Transaction txt = Transaction.currentTxn();
     	try {
@@ -248,8 +254,9 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
             return;
         }
     	
-        if(s_logger.isDebugEnabled())
+        if(s_logger.isDebugEnabled()) {
             s_logger.debug("Sync job-" + job.getId() + " execution on object " + syncObjType + "." + syncObjId);
+        }
 
     	SyncQueueVO queue = null;
 
@@ -259,8 +266,9 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
 
     	for(int i = 0; i < 5; i++) {
     		queue = _queueMgr.queue(syncObjType, syncObjId, "AsyncJob", job.getId());
-    		if(queue != null)
+    		if(queue != null) {
                 break;
+            }
 
     		try {
 				Thread.sleep(1000 + random.nextInt(5000));
@@ -302,8 +310,9 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
 
     @Override @DB
     public AsyncJobResult queryAsyncJobResult(long jobId) {
-    	if(s_logger.isTraceEnabled())
+    	if(s_logger.isTraceEnabled()) {
             s_logger.trace("Query async-job status, job-" + jobId);
+        }
     	
     	Transaction txt = Transaction.currentTxn();
     	AsyncJobResult jobResult = new AsyncJobResult(jobId);
@@ -321,15 +330,17 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
     			if(job.getStatus() == AsyncJobResult.STATUS_SUCCEEDED ||
     				job.getStatus() == AsyncJobResult.STATUS_FAILED) {
     				
-    		    	if(s_logger.isDebugEnabled())
+    		    	if(s_logger.isDebugEnabled()) {
                         s_logger.debug("Async job-" + jobId + " completed");
+                    }
     			} else {
     				job.setLastPolled(DateUtil.currentGMTTime());
     				_jobDao.update(jobId, job);
     			}
     		} else {
-    	    	if(s_logger.isDebugEnabled())
+    	    	if(s_logger.isDebugEnabled()) {
                     s_logger.debug("Async job-" + jobId + " does not exist, invalid job id?");
+                }
     			
     			jobResult.setJobStatus(AsyncJobResult.STATUS_FAILED);
     			jobResult.setResult("job-" + jobId + " does not exist");
@@ -343,8 +354,9 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
     		txt.rollback();
     	}
     	
-    	if(s_logger.isTraceEnabled())
+    	if(s_logger.isTraceEnabled()) {
             s_logger.trace("Job status: " + jobResult.toString());
+        }
     	
     	return jobResult;
     }
@@ -366,6 +378,7 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
         return new Runnable() {
             @Override
             public void run() {
+                try {
                     long jobId = 0;
                     
                     try {
@@ -423,8 +436,9 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
                             checkQueue(job.getSyncSource().getQueueId());
                         }
     
-                    if (s_logger.isDebugEnabled())
+                        if (s_logger.isDebugEnabled()) {
                             s_logger.debug("Done executing " + job.getCmd() + " for job-" + jobId);
+                        }
                         
                     } catch(Throwable e) {
                         if (e instanceof AsyncCommandQueued) {
@@ -475,6 +489,12 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
                         txn.close();
                         NDC.pop();
                     }
+                } catch (Throwable th) {
+                    try {
+                        s_logger.error("Caught: " + th);
+                    } catch (Throwable th2) {
+                    }
+                }
             }
         };
     }
@@ -493,8 +513,9 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
             _jobDao.update(job.getId(), job);
             scheduleExecution(job);
         } else {
-            if(s_logger.isDebugEnabled())
+            if(s_logger.isDebugEnabled()) {
                 s_logger.debug("Unable to find related job for queue item: " + item.toString());
+            }
 
             _queueMgr.purgeItem(item.getId());
         }
@@ -503,10 +524,11 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
     @Override
     public void releaseSyncSource(AsyncJobExecutor executor) {
     	if(executor.getSyncSource() != null) {
-    		if(s_logger.isDebugEnabled())
+    		if(s_logger.isDebugEnabled()) {
                 s_logger.debug("Release sync source for job-" + executor.getJob().getId() + " sync source: "
 					+ executor.getSyncSource().getContentType() + "-"
 					+ executor.getSyncSource().getContentId());
+            }
     		
 			_queueMgr.purgeItem(executor.getSyncSource().getId());
 			checkQueue(executor.getSyncSource().getQueueId());
@@ -518,8 +540,9 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
     		try {
 	        	SyncQueueItemVO item = _queueMgr.dequeueFromOne(queueId, getMsid());
 		    	if(item != null) {
-		    		if(s_logger.isDebugEnabled())
+		    		if(s_logger.isDebugEnabled()) {
                         s_logger.debug("Executing sync queue item: " + item.toString());
+                    }
 		    		
 		    		executeQueueItem(item, false);
 		    	} else {
@@ -540,8 +563,9 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
 					List<SyncQueueItemVO> l = _queueMgr.dequeueFromAny(getMsid(), MAX_ONETIME_SCHEDULE_SIZE);
 					if(l != null && l.size() > 0) {
 						for(SyncQueueItemVO item: l) {
-							if(s_logger.isDebugEnabled())
+							if(s_logger.isDebugEnabled()) {
                                 s_logger.debug("Execute sync-queue item: " + item.toString());
+                            }
 							executeQueueItem(item, false);
 						}
 					}
@@ -592,8 +616,9 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
 				    List<SyncQueueItemVO> blockItems = _queueMgr.getBlockedQueueItems(_jobCancelThresholdSeconds*1000, false);
 				    if(blockItems != null && blockItems.size() > 0) {
 				        for(SyncQueueItemVO item : blockItems) {
-				            if(item.getContentType().equalsIgnoreCase("AsyncJob"))
+				            if(item.getContentType().equalsIgnoreCase("AsyncJob")) {
                                 completeAsyncJob(item.getContentId(), 2, 0, getResetResultMessage("Job is cancelled as it has been blocking others for too long"));
+                            }
 				        
 				            // purge the item and resume queue processing
 				            _queueMgr.purgeItem(item.getId());
@@ -611,8 +636,9 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
 	}
 	
 	private long getMsid() {
-		if(_clusterMgr != null)
+		if(_clusterMgr != null) {
             return _clusterMgr.getManagementNodeId();
+        }
 		
 		return MacAddress.getMacAddress().toLong();
 	}
@@ -620,8 +646,9 @@ public class AsyncJobManagerImpl implements AsyncJobManager, ClusterManagerListe
 	private void cleanupPendingJobs(List<SyncQueueItemVO> l) {
 		if(l != null && l.size() > 0) {
 			for(SyncQueueItemVO item: l) {
-				if(s_logger.isInfoEnabled())
+				if(s_logger.isInfoEnabled()) {
                     s_logger.info("Discard left-over queue item: " + item.toString());
+                }
 				
 				String contentType = item.getContentType();
 				if(contentType != null && contentType.equals("AsyncJob")) {

server/src/com/cloud/cluster/ClusterManagerImpl.java

@@ -574,16 +574,13 @@ public class ClusterManagerImpl implements ClusterManager {
                     Connection conn = getHeartbeatConnection();
                     _mshostDao.update(conn, _mshostId, getCurrentRunId(), DateUtil.currentGMTTime());
 
-                    // for cluster in Starting state check if there are any agents being transfered
                     if (_state == State.Starting) {
                         synchronized (stateLock) {
-                            if (isClusterReadyToStart()) {
                             _mshostDao.update(conn, _mshostId, getCurrentRunId(), State.Up, DateUtil.currentGMTTime());
                             _state = State.Up;
                             stateLock.notifyAll();
                         }
                     }
-                    }
 
                     if (s_logger.isTraceEnabled()) {
                         s_logger.trace("Cluster manager peer-scan, id:" + _mshostId);
@@ -618,45 +615,6 @@ public class ClusterManagerImpl implements ClusterManager {
                     s_logger.error("Problem with the cluster heartbeat!", e);
                 }
             }
-            
-            private boolean isClusterReadyToStart() {
-                if (!_agentLBEnabled) {
-                    return true;
-                }
-                boolean isReady = false;
-                int transferCount = _hostTransferDao.listHostsJoiningCluster(_msId).size();
-                if (transferCount == 0) {
-                    //Check how many servers got transfered successfully
-                    List<HostTransferMapVO> rebalancedHosts = _hostTransferDao.listBy(_msId, HostTransferState.TransferCompleted);
-                    
-                    if (!rebalancedHosts.isEmpty() && s_logger.isDebugEnabled()) {
-                        s_logger.debug(rebalancedHosts.size() + " hosts joined the cluster " + _msId + " as a result of rebalance process");
-
-                    }
-                    
-                    for (HostTransferMapVO host : rebalancedHosts) {
-                        _hostTransferDao.remove(host.getId());
-                    }     
-                    
-                    //Check how many servers failed to transfer
-                    List<HostTransferMapVO> failedToRebalanceHosts = _hostTransferDao.listBy(_msId, HostTransferState.TransferFailed);
-                    if (!failedToRebalanceHosts.isEmpty() && s_logger.isDebugEnabled()) {
-                        s_logger.debug(failedToRebalanceHosts.size() + " hosts failed to join the cluster " + _msId + " as a result of rebalance process");
-                    }
-                    
-                    for (HostTransferMapVO host : failedToRebalanceHosts) {
-                        _hostTransferDao.remove(host.getId());
-                    }
-                    
-                    s_logger.debug("There are no hosts currently joining cluser msid=" + _msId + ", so management server is ready to start");
-                    isReady = true;
-                } else if (s_logger.isDebugEnabled()) {
-                    //TODO : change to trace mode later
-                    s_logger.debug("There are " + transferCount + " agents currently joinging the cluster " + _msId);
-                }
-                
-                return isReady;
-            }
         };  
     }
 
@@ -952,16 +910,16 @@ public class ClusterManagerImpl implements ClusterManager {
                 s_logger.info("Management server (host id : " + _mshostId + ") is being started at " + _clusterNodeIP + ":" + _currentServiceAdapter.getServicePort());
             }
             
-            // use seperate thread for heartbeat updates
-            _heartbeatScheduler.scheduleAtFixedRate(getHeartbeatTask(), heartbeatInterval, heartbeatInterval, TimeUnit.MILLISECONDS);
-            _notificationExecutor.submit(getNotificationTask());
-            
-            // Do agent rebalancing
+            // Initiate agent rebalancing
             if (_agentLBEnabled) {
                 s_logger.debug("Management server " + _msId + " is asking other peers to rebalance their agents");
                 _rebalanceService.startRebalanceAgents();
             }
                     
+            // use seperate thread for heartbeat updates
+            _heartbeatScheduler.scheduleAtFixedRate(getHeartbeatTask(), heartbeatInterval, heartbeatInterval, TimeUnit.MILLISECONDS);
+            _notificationExecutor.submit(getNotificationTask());
+
              //wait here for heartbeat task to update the host state
              try {
                  synchronized (stateLock) {

server/src/com/cloud/cluster/agentlb/AgentLoadBalancerPlanner.java

@@ -26,6 +26,6 @@ import com.cloud.utils.component.Adapter;
 
 public interface AgentLoadBalancerPlanner extends Adapter{
     
-    List<HostVO> getHostsToRebalance(long msId, long avLoad);
+    List<HostVO> getHostsToRebalance(long msId, int avLoad);
 
 }

server/src/com/cloud/cluster/agentlb/ClusterBasedAgentLoadBalancerPlanner.java

@@ -65,11 +65,11 @@ public class ClusterBasedAgentLoadBalancerPlanner implements AgentLoadBalancerPl
     }
     
     @Override
-    public List<HostVO> getHostsToRebalance(long msId, long avLoad) {
+    public List<HostVO> getHostsToRebalance(long msId, int avLoad) {
         List<HostVO> allHosts = _hostDao.listByManagementServer(msId);
  
         if (allHosts.size() <= avLoad) {
-            s_logger.debug("Agent load for management server " + msId + " doesn't exceed av load " + avLoad + "; so it doesn't participate in agent rebalancing process");
+            s_logger.debug("Agent load = " + allHosts.size() + " for management server " + msId + " doesn't exceed average system agent load = " + avLoad + "; so it doesn't participate in agent rebalancing process");
             return null;
         }
         
@@ -96,9 +96,9 @@ public class ClusterBasedAgentLoadBalancerPlanner implements AgentLoadBalancerPl
         
         hostToClusterMap = sortByClusterSize(hostToClusterMap);
         
-        long hostsToGive = allHosts.size() - avLoad;
-        long hostsLeftToGive = hostsToGive;
-        long hostsLeft = directHosts.size();
+        int hostsToGive = allHosts.size() - avLoad;
+        int hostsLeftToGive = hostsToGive;
+        int hostsLeft = directHosts.size();
         List<HostVO> hostsToReturn = new ArrayList<HostVO>();
         int count = 0;
         

server/src/com/cloud/cluster/agentlb/HostTransferMapVO.java

@@ -35,7 +35,7 @@ import com.cloud.utils.db.GenericDao;
 public class HostTransferMapVO {
 
     public enum HostTransferState {
-        TransferRequested, TransferStarted, TransferCompleted, TransferFailed;
+        TransferRequested, TransferStarted;
     }
 
     @Id

server/src/com/cloud/cluster/agentlb/dao/HostTransferMapDao.java

@@ -29,13 +29,15 @@ public interface HostTransferMapDao extends GenericDao<HostTransferMapVO, Long>
 
     List<HostTransferMapVO> listHostsLeavingCluster(long clusterId);
 
-    List<HostTransferMapVO> listHostsJoiningCluster(long clusterId);
+    List<HostTransferMapVO> listHostsJoiningCluster(long futureOwnerId);
 
     HostTransferMapVO startAgentTransfering(long hostId, long currentOwner, long futureOwner);
 
-    boolean completeAgentTransfering(long hostId, boolean success);
+    boolean completeAgentTransfer(long hostId);
     
     List<HostTransferMapVO> listBy(long futureOwnerId, HostTransferState state);
     
-    boolean isActive(long hostId, Date cutTime);
+    boolean isNotActive(long hostId, Date cutTime);
+    
+    boolean startAgentTransfer(long hostId);
 }

server/src/com/cloud/cluster/agentlb/dao/HostTransferMapDaoImpl.java

@@ -51,7 +51,7 @@ public class HostTransferMapDaoImpl extends GenericDaoBase<HostTransferMapVO, Lo
         
         IntermediateStateSearch = createSearchBuilder();
         IntermediateStateSearch.and("futureOwner", IntermediateStateSearch.entity().getFutureOwner(), SearchCriteria.Op.EQ);
-        IntermediateStateSearch.and("state", IntermediateStateSearch.entity().getState(), SearchCriteria.Op.NOTIN);
+        IntermediateStateSearch.and("state", IntermediateStateSearch.entity().getState(), SearchCriteria.Op.IN);
         IntermediateStateSearch.done();
         
         InactiveSearch = createSearchBuilder();
@@ -72,10 +72,10 @@ public class HostTransferMapDaoImpl extends GenericDaoBase<HostTransferMapVO, Lo
     }
 
     @Override
-    public List<HostTransferMapVO> listHostsJoiningCluster(long clusterId) {
+    public List<HostTransferMapVO> listHostsJoiningCluster(long futureOwnerId) {
         SearchCriteria<HostTransferMapVO> sc = IntermediateStateSearch.create();
-        sc.setParameters("futureOwner", clusterId);
-        sc.setParameters("state", HostTransferState.TransferRequested, HostTransferState.TransferStarted);
+        sc.setParameters("futureOwner", futureOwnerId);
+        sc.setParameters("state", HostTransferState.TransferRequested);
         return listBy(sc);
     }
     
@@ -88,14 +88,8 @@ public class HostTransferMapDaoImpl extends GenericDaoBase<HostTransferMapVO, Lo
     }
 
     @Override
-    public boolean completeAgentTransfering(long hostId, boolean success) {
-        HostTransferMapVO transfer = findById(hostId);
-        if (success) {
-            transfer.setState(HostTransferState.TransferCompleted);
-        } else {
-            transfer.setState(HostTransferState.TransferFailed);
-        }
-        return update(hostId, transfer);
+    public boolean completeAgentTransfer(long hostId) {
+        return remove(hostId);
     }
     
     @Override
@@ -108,18 +102,24 @@ public class HostTransferMapDaoImpl extends GenericDaoBase<HostTransferMapVO, Lo
     }
     
     @Override
-    public boolean isActive(long hostId, Date cutTime) {
+    public boolean isNotActive(long hostId, Date cutTime) {
         SearchCriteria<HostTransferMapVO> sc = InactiveSearch.create();
         sc.setParameters("id", hostId);
         sc.setParameters("state", HostTransferState.TransferRequested);
         sc.setParameters("created", cutTime);
         
-
         if (listBy(sc).isEmpty()) {
-            return true;
-        } else {
             return false;
+        } else {
+            return true;
         }
     }
     
+    @Override
+    public boolean startAgentTransfer(long hostId) {
+        HostTransferMapVO transfer = findById(hostId);
+        transfer.setState(HostTransferState.TransferStarted);
+        return update(hostId, transfer);
+    }
+    
 }

server/src/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java

@@ -328,7 +328,7 @@ public class ManagementServerHostDaoImpl extends GenericDaoBase<ManagementServer
 	public List<ManagementServerHostVO> listBy(ManagementServerHost.State...states) {
 	    SearchCriteria<ManagementServerHostVO> sc = StateSearch.create();
 
-        sc.setParameters("status", (Object[]) states);
+        sc.setParameters("state", (Object[]) states);
         
         return listBy(sc);
 	}

server/src/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -130,6 +130,7 @@ import com.cloud.utils.db.Transaction;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.net.NetUtils;
 import com.cloud.vm.VMInstanceVO;
+import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.dao.ConsoleProxyDao;
 import com.cloud.vm.dao.DomainRouterDao;
 import com.cloud.vm.dao.SecondaryStorageVmDao;
@@ -937,10 +938,14 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura
                 selectSql += " AND taken IS NOT NULL";
             }
 
-            if (tableName.equals("host_pod_ref")) {
+            if (tableName.equals("host_pod_ref") || tableName.equals("host") || tableName.equals("volumes")) {
                 selectSql += " AND removed is NULL";
             }
             
+            if (tableName.equals("vm_instance")) {
+                selectSql += " AND state != '" + VirtualMachine.State.Expunging.toString() + "'";
+            }
+
             Transaction txn = Transaction.currentTxn();
             try {
                 PreparedStatement stmt = txn.prepareAutoCloseStatement(selectSql);
@@ -1060,11 +1065,10 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura
         List<NetworkVO> networks = _networkDao.listByZoneIncludingRemoved(zoneId);
         if (networks != null && !networks.isEmpty()) {
             for (NetworkVO network : networks) {
-                _networkDao.expunge(network.getId());
+                _networkDao.remove(network.getId());
             }
         }
-
-        success = _zoneDao.expunge(zoneId);
+        success = _zoneDao.remove(zoneId);
 
         txn.commit();
 
@@ -1367,7 +1371,7 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura
                     }
                 }
                 userNetwork.setBroadcastDomainType(broadcastDomainType);
-                _networkMgr.setupNetwork(systemAccount, offering, userNetwork, plan, null, null, true, isNetworkDefault, false, null);
+                _networkMgr.setupNetwork(systemAccount, offering, userNetwork, plan, null, null, true, isNetworkDefault, false, null, null);
             }
         }
     }

server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java

@@ -571,7 +571,7 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx
 
         Map<String, Object> context = new HashMap<String, Object>();
         context.put("dc", dc);
-        HostPodVO pod = _podDao.findById(proxy.getPodId());
+        HostPodVO pod = _podDao.findById(proxy.getPodIdToDeployIn());
         context.put("pod", pod);
         context.put("proxyVmId", proxy.getId());
 
@@ -1116,7 +1116,7 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx
                 }
 
                 SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT, this,
-                        new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_REBOOTED, proxy.getDataCenterId(), proxy.getId(), proxy, null));
+                        new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_REBOOTED, proxy.getDataCenterIdToDeployIn(), proxy.getId(), proxy, null));
 
                 return true;
             } else {
@@ -1325,7 +1325,7 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx
         if (externalDhcp) {
             buf.append(" bootproto=dhcp");
         }
-        DataCenterVO dc = _dcDao.findById(profile.getVirtualMachine().getDataCenterId());
+        DataCenterVO dc = _dcDao.findById(profile.getVirtualMachine().getDataCenterIdToDeployIn());
         buf.append(" internaldns1=").append(dc.getInternalDns1());
         if (dc.getInternalDns2() != null) {
             buf.append(" internaldns2=").append(dc.getInternalDns2());

server/src/com/cloud/consoleproxy/StaticConsoleProxyManager.java

@@ -40,7 +40,7 @@ public class StaticConsoleProxyManager extends AgentBasedConsoleProxyManager imp
     @Override
     protected HostVO findHost(VMInstanceVO vm) {
         
-        List<HostVO> hosts = _hostDao.listBy(Type.ConsoleProxy, vm.getDataCenterId());
+        List<HostVO> hosts = _hostDao.listBy(Type.ConsoleProxy, vm.getDataCenterIdToDeployIn());
         
         return hosts.isEmpty() ? null : hosts.get(0);
     }

server/src/com/cloud/dc/DataCenterVO.java

@@ -18,6 +18,7 @@
 
 package com.cloud.dc;
 
+import java.util.Date;
 import java.util.Map;
 
 import javax.persistence.Column;
@@ -34,6 +35,7 @@ import javax.persistence.Transient;
 import com.cloud.network.Network.Provider;
 import com.cloud.org.Grouping;
 import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.db.GenericDao;
 
 @Entity
 @Table(name="data_center")
@@ -112,6 +114,9 @@ public class DataCenterVO implements DataCenter {
 	@Column(name="zone_token")
 	private String zoneToken;    
 	
+    @Column(name=GenericDao.REMOVED_COLUMN)
+    private Date removed;
+    
     // This is a delayed load value.  If the value is null,
     // then this field has not been loaded yet.
     // Call the dao to load it.
@@ -398,4 +403,8 @@ public class DataCenterVO implements DataCenter {
  	public void setZoneToken(String zoneToken) {
  		this.zoneToken = zoneToken;
  	}
+ 	
+ 	public Date getRemoved() {
+ 	    return removed;
+ 	}
 }

server/src/com/cloud/dc/dao/DataCenterDaoImpl.java

@@ -32,6 +32,7 @@ import com.cloud.dc.DataCenterIpAddressVO;
 import com.cloud.dc.DataCenterLinkLocalIpAddressVO;
 import com.cloud.dc.DataCenterVO;
 import com.cloud.dc.DataCenterVnetVO;
+import com.cloud.dc.HostPodVO;
 import com.cloud.dc.PodVlanVO;
 import com.cloud.org.Grouping;
 import com.cloud.utils.NumbersUtil;
@@ -355,4 +356,18 @@ public class DataCenterDaoImpl extends GenericDaoBase<DataCenterVO, Long> implem
         }
         return result;
     }
+    
+    @Override
+    public boolean remove(Long id) {
+        Transaction txn = Transaction.currentTxn();
+        txn.start();
+        DataCenterVO zone = createForUpdate();
+        zone.setName(null);
+        
+        update(id, zone);
+
+        boolean result = super.remove(id);
+        txn.commit();
+        return result;
+    }
 }

server/src/com/cloud/deploy/BareMetalPlanner.java

@@ -87,7 +87,7 @@ public class BareMetalPlanner implements DeploymentPlanner {
 			}
 		}
 		
-		List<ClusterVO> clusters = _clusterDao.listByDcHyType(vm.getDataCenterId(), HypervisorType.BareMetal.toString());
+		List<ClusterVO> clusters = _clusterDao.listByDcHyType(vm.getDataCenterIdToDeployIn(), HypervisorType.BareMetal.toString());
 		if (clusters.size() != 1) {
 			throw new CloudRuntimeException("Invaild baremetal cluster number " + clusters.size());
 		}

server/src/com/cloud/deploy/FirstFitPlanner.java

@@ -110,7 +110,7 @@ public class FirstFitPlanner extends PlannerBase implements DeploymentPlanner {
  	    String _allocationAlgorithm = _configDao.getValue(Config.VmAllocationAlgorithm.key());
 		VirtualMachine vm = vmProfile.getVirtualMachine();
 		ServiceOffering offering = vmProfile.getServiceOffering();
-		DataCenter dc = _dcDao.findById(vm.getDataCenterId());
+		DataCenter dc = _dcDao.findById(vm.getDataCenterIdToDeployIn());
 		int cpu_requested = offering.getCpu() * offering.getSpeed();
 		long ram_requested = offering.getRamSize() * 1024L * 1024L;
 		

server/src/com/cloud/ha/HighAvailabilityManagerImpl.java

@@ -273,8 +273,8 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager, Clu
             }
 
             if (!(_forceHA || vm.isHaEnabled())) {
-                String hostDesc = "id:" + vm.getHostId() + ", availability zone id:" + vm.getDataCenterId() + ", pod id:" + vm.getPodId();
-                _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodId(), "VM (name: " + vm.getHostName() + ", id: " + vm.getId() + ") stopped unexpectedly on host " + hostDesc,
+                String hostDesc = "id:" + vm.getHostId() + ", availability zone id:" + vm.getDataCenterIdToDeployIn() + ", pod id:" + vm.getPodIdToDeployIn();
+                _alertMgr.sendAlert(alertType, vm.getDataCenterIdToDeployIn(), vm.getPodIdToDeployIn(), "VM (name: " + vm.getHostName() + ", id: " + vm.getId() + ") stopped unexpectedly on host " + hostDesc,
                         "Virtual Machine " + vm.getHostName() + " (id: " + vm.getId() + ") running on host [" + vm.getHostId() + "] stopped unexpectedly.");
 
                 if (s_logger.isDebugEnabled()) {
@@ -404,7 +404,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager, Clu
 
                 if (!fenced) {
                     s_logger.debug("We were unable to fence off the VM " + vm);
-                    _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodId(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
+                    _alertMgr.sendAlert(alertType, vm.getDataCenterIdToDeployIn(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
                             "Insufficient capacity to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
                     return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
                 }
@@ -462,19 +462,19 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager, Clu
             }
         } catch (final InsufficientCapacityException e) {
             s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
-            _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodId(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
+            _alertMgr.sendAlert(alertType, vm.getDataCenterIdToDeployIn(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
                     "Insufficient capacity to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
         } catch (final ResourceUnavailableException e) {
             s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
-            _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodId(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
+            _alertMgr.sendAlert(alertType, vm.getDataCenterIdToDeployIn(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
                     "The Storage is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
         } catch (ConcurrentOperationException e) {
             s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
-            _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodId(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
+            _alertMgr.sendAlert(alertType, vm.getDataCenterIdToDeployIn(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
                     "The Storage is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
         } catch (OperationTimedoutException e) {
             s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
-            _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodId(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
+            _alertMgr.sendAlert(alertType, vm.getDataCenterIdToDeployIn(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc,
                     "The Storage is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
         }
         vm = _itMgr.findById(vm.getType(), vm.getId());

server/src/com/cloud/ha/UserVmDomRInvestigator.java

@@ -172,7 +172,7 @@ public class UserVmDomRInvestigator extends AbstractInvestigatorImpl {
         		|| vm.getHypervisorType() == HypervisorType.KVM){
         	otherHosts.add(router.getHostId());
         }else{
-        	otherHosts = findHostByPod(router.getPodId(), null);
+        	otherHosts = findHostByPod(router.getPodIdToDeployIn(), null);
         }
         for (Long hostId : otherHosts) {
             try {

server/src/com/cloud/host/dao/HostDao.java

@@ -171,7 +171,7 @@ public interface HostDao extends GenericDao<HostVO, Long> {
     
     List<HostVO> listManagedDirectAgents();
     
-    List<HostVO> listManagedAgents();
+    List<HostVO> listManagedRoutingAgents();
 
     HostVO findTrafficMonitorHost();
 

server/src/com/cloud/host/dao/HostDaoImpl.java

@@ -91,7 +91,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
 
     protected final SearchBuilder<HostVO> DirectConnectSearch;
     protected final SearchBuilder<HostVO> ManagedDirectConnectSearch;
-    protected final SearchBuilder<HostVO> ManagedConnectSearch;
+    protected final SearchBuilder<HostVO> ManagedRoutingServersSearch;
 
     protected final GenericSearchBuilder<HostVO, Long> HostsInStatusSearch;
     protected final GenericSearchBuilder<HostVO, Long> CountRoutingByDc;
@@ -264,9 +264,10 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         ManagedDirectConnectSearch.and("server", ManagedDirectConnectSearch.entity().getManagementServerId(), SearchCriteria.Op.NULL);
         ManagedDirectConnectSearch.done();
 
-        ManagedConnectSearch = createSearchBuilder();
-        ManagedConnectSearch.and("server", ManagedConnectSearch.entity().getManagementServerId(), SearchCriteria.Op.NNULL);
-        ManagedConnectSearch.done();
+        ManagedRoutingServersSearch = createSearchBuilder();
+        ManagedRoutingServersSearch.and("server", ManagedRoutingServersSearch.entity().getManagementServerId(), SearchCriteria.Op.NNULL);
+        ManagedRoutingServersSearch.and("type", ManagedRoutingServersSearch.entity().getType(), SearchCriteria.Op.EQ);
+        ManagedRoutingServersSearch.done();
 
         _statusAttr = _allAttributes.get("status");
         _msIdAttr = _allAttributes.get("managementServerId");
@@ -868,8 +869,9 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
     }
 
     @Override
-    public List<HostVO> listManagedAgents() {
-        SearchCriteria<HostVO> sc = ManagedConnectSearch.create();
+    public List<HostVO> listManagedRoutingAgents() {
+        SearchCriteria<HostVO> sc = ManagedRoutingServersSearch.create();
+        sc.setParameters("type", Type.Routing);
         return listBy(sc);
     }
     

server/src/com/cloud/migration/Db20to21MigrationUtil.java

@@ -570,7 +570,7 @@ public class Db20to21MigrationUtil {
 	private void migrateSystemVmGuestMacAndState(long zoneId) {
 		// for console proxy VMs
 		SearchBuilder<ConsoleProxyVO> sb = _consoleProxyDao.createSearchBuilder();
-		sb.and("zoneId", sb.entity().getDataCenterId(), Op.EQ);
+		sb.and("zoneId", sb.entity().getDataCenterIdToDeployIn(), Op.EQ);
 		sb.done();
 		
 		SearchCriteria<ConsoleProxyVO> sc = sb.create();
@@ -586,7 +586,7 @@ public class Db20to21MigrationUtil {
 				proxy.setState(State.Stopping);
 			}
 			
-			String guestIpAddress = _dcDao.allocateLinkLocalIpAddress(proxy.getDataCenterId(), proxy.getPodId(), proxy.getId(), null);
+			String guestIpAddress = _dcDao.allocateLinkLocalIpAddress(proxy.getDataCenterIdToDeployIn(), proxy.getPodIdToDeployIn(), proxy.getId(), null);
 			
 			System.out.println("Assign link loal address to proxy " + proxy.getHostName() + ", link local address: " + guestIpAddress);
 			_consoleProxyDao.update(proxy.getId(), proxy);
@@ -594,7 +594,7 @@ public class Db20to21MigrationUtil {
 		
 		// for secondary storage VMs
 		SearchBuilder<SecondaryStorageVmVO> sb2 = _secStorageVmDao.createSearchBuilder();
-		sb2.and("zoneId", sb2.entity().getDataCenterId(), Op.EQ);
+		sb2.and("zoneId", sb2.entity().getDataCenterIdToDeployIn(), Op.EQ);
 		sb2.done();
 		
 		SearchCriteria<SecondaryStorageVmVO> sc2 = sb2.create();
@@ -610,7 +610,7 @@ public class Db20to21MigrationUtil {
 				secStorageVm.setState(State.Stopping);
 			}
 			
-			String guestIpAddress = _dcDao.allocateLinkLocalIpAddress(secStorageVm.getDataCenterId(), secStorageVm.getPodId(), secStorageVm.getId(), null);
+			String guestIpAddress = _dcDao.allocateLinkLocalIpAddress(secStorageVm.getDataCenterIdToDeployIn(), secStorageVm.getPodIdToDeployIn(), secStorageVm.getId(), null);
 			
 			System.out.println("Assign link loal address to secondary storage VM " + secStorageVm.getHostName() + ", link local address: " + guestIpAddress);
 			_secStorageVmDao.update(secStorageVm.getId(), secStorageVm);
@@ -619,7 +619,7 @@ public class Db20to21MigrationUtil {
 		// for Domain Router VMs
 		// Although we can list those we are interested, but just too lazy, list all of them and check their states. 
 		SearchBuilder<DomainRouterVO> sb3 = _routerDao.createSearchBuilder();
-		sb3.and("zoneId", sb3.entity().getDataCenterId(), Op.EQ);
+		sb3.and("zoneId", sb3.entity().getDataCenterIdToDeployIn(), Op.EQ);
 		sb3.done();
 		
 		SearchCriteria<DomainRouterVO> sc3 = sb3.create();
@@ -637,8 +637,8 @@ public class Db20to21MigrationUtil {
 	
 	private void migrateVmInstanceLastHostId(long zoneId, long podId) {
 		SearchBuilder<VMInstanceVO> sb = _vmInstanceDao.createSearchBuilder();
-		sb.and("zoneId", sb.entity().getDataCenterId(), Op.EQ);
-		sb.and("podId", sb.entity().getPodId(), Op.EQ);
+		sb.and("zoneId", sb.entity().getDataCenterIdToDeployIn(), Op.EQ);
+		sb.and("podId", sb.entity().getPodIdToDeployIn(), Op.EQ);
 		sb.done();
 		
 		Random rand = new Random();

server/src/com/cloud/network/NetworkManager.java

@@ -109,7 +109,7 @@ public interface NetworkManager extends NetworkService {
             throws ConcurrentOperationException;
 
     List<NetworkVO> setupNetwork(Account owner, NetworkOfferingVO offering, Network predefined, DeploymentPlan plan, String name, String displayText, boolean isShared, boolean isDefault,
-            boolean errorIfAlreadySetup, Long domainId) throws ConcurrentOperationException;
+            boolean errorIfAlreadySetup, Long domainId, List<String> tags) throws ConcurrentOperationException;
 
     List<NetworkOfferingVO> getSystemAccountNetworkOfferings(String... offeringNames);
 
@@ -158,7 +158,7 @@ public interface NetworkManager extends NetworkService {
     boolean destroyNetwork(long networkId, ReservationContext context);
 
     Network createNetwork(long networkOfferingId, String name, String displayText, Boolean isShared, Boolean isDefault, Long zoneId, String gateway, String cidr, String vlanId, String networkDomain,
-            Account owner, boolean isSecurityGroupEnabled, Long domainId) throws ConcurrentOperationException, InsufficientCapacityException;
+            Account owner, boolean isSecurityGroupEnabled, Long domainId, List<String> tags) throws ConcurrentOperationException, InsufficientCapacityException;
 
     /**
      * @throws InsufficientCapacityException

server/src/com/cloud/network/NetworkManagerImpl.java

@@ -18,6 +18,7 @@
 package com.cloud.network;
 
 import java.net.URI;
+import java.security.InvalidParameterException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
@@ -829,13 +830,13 @@ public class NetworkManagerImpl implements NetworkManager, NetworkService, Manag
     @Override
     public List<NetworkVO> setupNetwork(Account owner, NetworkOfferingVO offering, DeploymentPlan plan, String name, String displayText, boolean isShared, boolean isDefault)
     throws ConcurrentOperationException {
-        return setupNetwork(owner, offering, null, plan, name, displayText, isShared, isDefault, false, null);
+        return setupNetwork(owner, offering, null, plan, name, displayText, isShared, isDefault, false, null, null);
     }
 
     @Override
     @DB
     public List<NetworkVO> setupNetwork(Account owner, NetworkOfferingVO offering, Network predefined, DeploymentPlan plan, String name, String displayText, boolean isShared, boolean isDefault,
-            boolean errorIfAlreadySetup, Long domainId) throws ConcurrentOperationException {
+            boolean errorIfAlreadySetup, Long domainId, List<String> tags) throws ConcurrentOperationException {
         Account locked = _accountDao.acquireInLockTable(owner.getId());
         if (locked == null) {
             throw new ConcurrentOperationException("Unable to acquire lock on " + owner);
@@ -870,21 +871,21 @@ public class NetworkManagerImpl implements NetworkManager, NetworkService, Manag
                 }
             }
 
-            List<NetworkVO> configs = new ArrayList<NetworkVO>();
+            List<NetworkVO> networks = new ArrayList<NetworkVO>();
 
             long related = -1;
 
             for (NetworkGuru guru : _networkGurus) {
-                Network config = guru.design(offering, plan, predefined, owner);
-                if (config == null) {
+                Network network = guru.design(offering, plan, predefined, owner);
+                if (network == null) {
                     continue;
                 }
 
-                if (config.getId() != -1) {
-                    if (config instanceof NetworkVO) {
-                        configs.add((NetworkVO) config);
+                if (network.getId() != -1) {
+                    if (network instanceof NetworkVO) {
+                        networks.add((NetworkVO) network);
                     } else {
-                        configs.add(_networksDao.findById(config.getId()));
+                        networks.add(_networksDao.findById(network.getId()));
                     }
                     continue;
                 }
@@ -894,20 +895,21 @@ public class NetworkManagerImpl implements NetworkManager, NetworkService, Manag
                     related = id;
                 }
 
-                NetworkVO vo = new NetworkVO(id, config, offering.getId(), plan.getDataCenterId(), guru.getName(), owner.getDomainId(), owner.getId(), related, name, displayText, isShared, isDefault,
+                NetworkVO vo = new NetworkVO(id, network, offering.getId(), plan.getDataCenterId(), guru.getName(), owner.getDomainId(), owner.getId(), related, name, displayText, isShared, isDefault,
                         predefined.isSecurityGroupEnabled());
-                configs.add(_networksDao.persist(vo, vo.getGuestType() != null));
+                vo.setTags(tags);
+                networks.add(_networksDao.persist(vo, vo.getGuestType() != null));
 
                 if (domainId != null) {
                     _networksDao.addDomainToNetwork(id, domainId);
                 }
             }
 
-            if (configs.size() < 1) {
+            if (networks.size() < 1) {
                 throw new CloudRuntimeException("Unable to convert network offering to network profile: " + offering.getId());
             }
 
-            return configs;
+            return networks;
         } finally {
             s_logger.debug("Releasing lock for " + locked);
             _accountDao.releaseFromLockTable(locked.getId());
@@ -1247,6 +1249,7 @@ public class NetworkManagerImpl implements NetworkManager, NetworkService, Manag
                 updateNic(nic, network.getId(), 1);
             } else {
                 profile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), networkRate);
+                guru.updateNicProfile(profile, network);
                 nic.setState(Nic.State.Reserved);
                 updateNic(nic, network.getId(), 1);
             }
@@ -1470,8 +1473,13 @@ public class NetworkManagerImpl implements NetworkManager, NetworkService, Manag
         Boolean isDefault = cmd.isDefault();
         Long userId = UserContext.current().getCallerUserId();
         Account caller = UserContext.current().getCaller();
+        List<String> tags = cmd.getTags();
         boolean isDomainSpecific = false;
         
+        if (tags != null && tags.size() > 1) {
+            throw new InvalidParameterException("Only one tag can be specified for a network at this time");
+        }
+
         Transaction txn = Transaction.currentTxn();
 
         // Check if network offering exists
@@ -1584,7 +1592,7 @@ public class NetworkManagerImpl implements NetworkManager, NetworkService, Manag
             domainId = cmd.getDomainId();
         }
 
-        Network network = createNetwork(networkOfferingId, name, displayText, isShared, isDefault, zoneId, gateway, cidr, vlanId, networkDomain, owner, false, domainId);
+        Network network = createNetwork(networkOfferingId, name, displayText, isShared, isDefault, zoneId, gateway, cidr, vlanId, networkDomain, owner, false, domainId, null);
 
         // Don't pass owner to create vlan when network offering is of type Direct - done to prevent accountVlanMap entry
         // creation when vlan is mapped to network
@@ -1605,7 +1613,7 @@ public class NetworkManagerImpl implements NetworkManager, NetworkService, Manag
     @Override
     @DB
     public Network createNetwork(long networkOfferingId, String name, String displayText, Boolean isShared, Boolean isDefault, Long zoneId, String gateway, String cidr, String vlanId,
-            String networkDomain, Account owner, boolean isSecurityGroupEnabled, Long domainId) throws ConcurrentOperationException, InsufficientCapacityException {
+            String networkDomain, Account owner, boolean isSecurityGroupEnabled, Long domainId, List<String> tags) throws ConcurrentOperationException, InsufficientCapacityException {
 
         NetworkOfferingVO networkOffering = _networkOfferingDao.findById(networkOfferingId);
         DataCenterVO zone = _dcDao.findById(zoneId);
@@ -1706,7 +1714,7 @@ public class NetworkManagerImpl implements NetworkManager, NetworkService, Manag
             }
         }
 
-        List<NetworkVO> networks = setupNetwork(owner, networkOffering, userNetwork, plan, name, displayText, isShared, isDefault, true, domainId);
+        List<NetworkVO> networks = setupNetwork(owner, networkOffering, userNetwork, plan, name, displayText, isShared, isDefault, true, domainId, tags);
 
         Network network = null;
         if (networks == null || networks.isEmpty()) {
@@ -2520,7 +2528,7 @@ public class NetworkManagerImpl implements NetworkManager, NetworkService, Manag
         // create new Virtual network for the user if it doesn't exist
         if (createNetwork) {
             List<? extends NetworkOffering> offerings = _configMgr.listNetworkOfferings(TrafficType.Guest, false);
-            network = createNetwork(offerings.get(0).getId(), owner.getAccountName() + "-network", owner.getAccountName() + "-network", false, null, zoneId, null, null, null, null, owner, false, null);
+            network = createNetwork(offerings.get(0).getId(), owner.getAccountName() + "-network", owner.getAccountName() + "-network", false, null, zoneId, null, null, null, null, owner, false, null, null);
 
             if (network == null) {
                 s_logger.warn("Failed to create default Virtual network for the account " + accountId + "in zone " + zoneId);
@@ -2826,7 +2834,7 @@ public class NetworkManagerImpl implements NetworkManager, NetworkService, Manag
 
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_NETWORK_UPDATE, eventDescription = "updating network", async = false)
-    public Network updateNetwork(long networkId, String name, String displayText, Account caller) {
+    public Network updateNetwork(long networkId, String name, String displayText, List<String> tags, Account caller) {
 
         // verify input parameters
         NetworkVO network = _networksDao.findById(networkId);
@@ -2834,6 +2842,10 @@ public class NetworkManagerImpl implements NetworkManager, NetworkService, Manag
             throw new InvalidParameterValueException("Network id=" + networkId + "doesn't exist in the system");
         }
         
+        if (tags != null && tags.size() > 1) {
+            throw new InvalidParameterException("Unable to support more than one tag on network yet");
+        }
+        
         // Don't allow to update system network
         NetworkOffering offering = _networkOfferingDao.findByIdIncludingRemoved(network.getNetworkOfferingId());
         if (offering.isSystemOnly()) {
@@ -2850,6 +2862,10 @@ public class NetworkManagerImpl implements NetworkManager, NetworkService, Manag
             network.setDisplayText(displayText);
         }
         
+        if (tags != null) {
+            network.setTags(tags);
+        }
+
         _networksDao.update(networkId, network);
 
         return network;

server/src/com/cloud/network/NetworkVO.java

@@ -22,11 +22,15 @@ import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 
+import javax.persistence.CollectionTable;
 import javax.persistence.Column;
+import javax.persistence.ElementCollection;
 import javax.persistence.Entity;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
+import javax.persistence.FetchType;
 import javax.persistence.Id;
+import javax.persistence.JoinColumn;
 import javax.persistence.Table;
 import javax.persistence.TableGenerator;
 import javax.persistence.Transient;
@@ -34,7 +38,6 @@ import javax.persistence.Transient;
 import com.cloud.network.Networks.BroadcastDomainType;
 import com.cloud.network.Networks.Mode;
 import com.cloud.network.Networks.TrafficType;
-import com.cloud.network.dao.NetworkDao;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.db.GenericDao;
 import com.cloud.utils.net.NetUtils;
@@ -140,7 +143,9 @@ public class NetworkVO implements Network {
     @Column(name="is_security_group_enabled")
     boolean securityGroupEnabled;
 
-    @Transient
+    @ElementCollection(targetClass = String.class, fetch=FetchType.EAGER)
+    @Column(name="tag")
+    @CollectionTable(name="network_tags", joinColumns=@JoinColumn(name="network_id"))
     List<String> tags;
 
     public NetworkVO() {
@@ -464,10 +469,4 @@ public class NetworkVO implements Network {
         buf.append(id).append("|").append(trafficType.toString()).append("|").append(networkOfferingId).append("]");
         return buf.toString();
     }
-
-    private static NetworkDao _networkDao = null;
-    static void init(NetworkDao networkDao) {
-        _networkDao = networkDao;
-    }
-
 }

server/src/com/cloud/network/guru/DirectPodBasedNetworkGuru.java

@@ -117,6 +117,10 @@ public class DirectPodBasedNetworkGuru extends DirectNetworkGuru {
             getIp(nic, dest.getPod(), vm, network);
             nic.setStrategy(ReservationStrategy.Create);
         }
+        
+        DataCenter dc = _dcDao.findById(network.getDataCenterId());
+        nic.setDns1(dc.getDns1());
+        nic.setDns2(dc.getDns2());
     }
 
     protected void getIp(NicProfile nic, Pod pod, VirtualMachineProfile<? extends VirtualMachine> vm, Network network) throws InsufficientVirtualNetworkCapcityException,

server/src/com/cloud/network/ovs/OvsNetworkManagerImpl.java

@@ -395,7 +395,7 @@ public class OvsNetworkManagerImpl implements OvsNetworkManager {
 		long hostId = dest.getHost().getId();
 		long accountId = instance.getAccountId();
 		List<UserVmVO>vms = _userVmDao.listByAccountId(accountId);
-		DomainRouterVO router = _routerDao.findBy(accountId, instance.getDataCenterId());
+		DomainRouterVO router = _routerDao.findBy(accountId, instance.getDataCenterIdToDeployIn());
 		List<VMInstanceVO>ins = new ArrayList<VMInstanceVO>();
 		ins.addAll(vms);
 		ins.add(router);
@@ -538,7 +538,7 @@ public class OvsNetworkManagerImpl implements OvsNetworkManager {
 		}
 		
 		long accountId = instance.getAccountId();
-		DomainRouterVO router = _routerDao.findBy(accountId, instance.getDataCenterId());
+		DomainRouterVO router = _routerDao.findBy(accountId, instance.getDataCenterIdToDeployIn());
 		if (router == null) {
 			return;
 		}
@@ -639,7 +639,7 @@ public class OvsNetworkManagerImpl implements OvsNetworkManager {
 		}
 		
 		if (tellRouter && instance.getType() != VirtualMachine.Type.DomainRouter) {
-			DomainRouterVO router = _routerDao.findBy(accountId, instance.getDataCenterId());
+			DomainRouterVO router = _routerDao.findBy(accountId, instance.getDataCenterIdToDeployIn());
 			if (router != null) {
 				affectedVms.add(new Long(router.getId()));
 			}

server/src/com/cloud/network/ovs/OvsTunnelManagerImpl.java

@@ -153,7 +153,7 @@ public class OvsTunnelManagerImpl implements OvsTunnelManager {
 		long hostId = dest.getHost().getId();
 		long accountId = instance.getAccountId();
 		List<UserVmVO>vms = _userVmDao.listByAccountId(accountId);
-		DomainRouterVO router = _routerDao.findBy(accountId, instance.getDataCenterId());
+		DomainRouterVO router = _routerDao.findBy(accountId, instance.getDataCenterIdToDeployIn());
 		List<VMInstanceVO>ins = new ArrayList<VMInstanceVO>();
 		if (vms != null) {
 			ins.addAll(vms);
@@ -284,7 +284,7 @@ public class OvsTunnelManagerImpl implements OvsTunnelManager {
                 return;
             }
             
-            DomainRouterVO router = _routerDao.findBy(vm.getAccountId(), vm.getDataCenterId());
+            DomainRouterVO router = _routerDao.findBy(vm.getAccountId(), vm.getDataCenterIdToDeployIn());
             if (router.getHostId() == vm.getHostId()) {
                 return;
             }

server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -430,7 +430,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
         final Transaction txn = Transaction.currentTxn();
         try {
             txn.start();
-            final UserStatisticsVO userStats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterId(), router.getNetworkId(), null, router.getId(), router.getType().toString());
+            final UserStatisticsVO userStats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterIdToDeployIn(), router.getNetworkId(), null, router.getId(), router.getType().toString());
             if (userStats != null) {
                 final RebootAnswer sa = (RebootAnswer) answer;
                 final Long received = sa.getBytesReceived();
@@ -463,7 +463,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
                 _userStatsDao.update(userStats.getId(), userStats);
                 s_logger.debug("Successfully updated user statistics as a part of domR " + router + " reboot/stop");
             } else {
-                s_logger.warn("User stats were not created for account " + router.getAccountId() + " and dc " + router.getDataCenterId());
+                s_logger.warn("User stats were not created for account " + router.getAccountId() + " and dc " + router.getDataCenterIdToDeployIn());
             }
             txn.commit();
         } catch (final Exception e) {
@@ -509,7 +509,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
         // Can reboot domain router only in Running state
         if (router == null || router.getState() != State.Running) {
             s_logger.warn("Unable to reboot, virtual router is not in the right state " + router.getState());
-            throw new ResourceUnavailableException("Unable to reboot domR, it is not in right state " + router.getState(), DataCenter.class, router.getDataCenterId());
+            throw new ResourceUnavailableException("Unable to reboot domR, it is not in right state " + router.getState(), DataCenter.class, router.getDataCenterIdToDeployIn());
         }
 
         UserVO user = _userDao.findById(UserContext.current().getCallerUserId());
@@ -651,7 +651,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
                                 continue;
                             }
                             txn.start();
-                            UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), router.getDataCenterId(), router.getNetworkId(), null, router.getId(), router.getType().toString());
+                            UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), router.getDataCenterIdToDeployIn(), router.getNetworkId(), null, router.getId(), router.getType().toString());
                             if (stats == null) {
                                 s_logger.warn("unable to find stats for account: " + router.getAccountId());
                                 continue;
@@ -1042,7 +1042,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
             s_logger.debug("Resending ipAssoc, port forwarding, load balancing rules as a part of Virtual router start");
             long networkId = router.getNetworkId();
             long ownerId = router.getAccountId();
-            long zoneId = router.getDataCenterId();
+            long zoneId = router.getDataCenterIdToDeployIn();
 
             final List<IPAddressVO> userIps = _networkMgr.listPublicIpAddressesInVirtualNetwork(ownerId, zoneId, null, null);
             List<PublicIpAddress> publicIps = new ArrayList<PublicIpAddress>();
@@ -1176,17 +1176,17 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
         }
         Answer answer = cmds.getAnswer("users");
         if (!answer.getResult()) {
-            s_logger.error("Unable to start vpn: unable add users to vpn in zone " + router.getDataCenterId() + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName()
+            s_logger.error("Unable to start vpn: unable add users to vpn in zone " + router.getDataCenterIdToDeployIn() + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName()
                     + " due to " + answer.getDetails());
-            throw new ResourceUnavailableException("Unable to start vpn: Unable to add users to vpn in zone " + router.getDataCenterId() + " for account " + vpn.getAccountId() + " on domR: "
-                    + router.getInstanceName() + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterId());
+            throw new ResourceUnavailableException("Unable to start vpn: Unable to add users to vpn in zone " + router.getDataCenterIdToDeployIn() + " for account " + vpn.getAccountId() + " on domR: "
+                    + router.getInstanceName() + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn());
         }
         answer = cmds.getAnswer("startVpn");
         if (!answer.getResult()) {
-            s_logger.error("Unable to start vpn in zone " + router.getDataCenterId() + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() + " due to "
+            s_logger.error("Unable to start vpn in zone " + router.getDataCenterIdToDeployIn() + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName() + " due to "
                     + answer.getDetails());
-            throw new ResourceUnavailableException("Unable to start vpn in zone " + router.getDataCenterId() + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName()
-                    + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterId());
+            throw new ResourceUnavailableException("Unable to start vpn in zone " + router.getDataCenterIdToDeployIn() + " for account " + vpn.getAccountId() + " on domR: " + router.getInstanceName()
+                    + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn());
         }
         return true;
     }
@@ -1291,19 +1291,19 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
         Answer answer = cmds.getAnswer("dhcp");
         if (!answer.getResult()) {
             s_logger.error("Unable to set dhcp entry for " + profile + " on domR: " + router.getHostName() + " due to " + answer.getDetails());
-            throw new ResourceUnavailableException("Unable to set dhcp entry for " + profile + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterId());
+            throw new ResourceUnavailableException("Unable to set dhcp entry for " + profile + " due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn());
         }
 
         answer = cmds.getAnswer("password");
         if (answer != null && !answer.getResult()) {
             s_logger.error("Unable to set password for " + profile + " due to " + answer.getDetails());
-            throw new ResourceUnavailableException("Unable to set password due to " + answer.getDetails(), DataCenter.class, router.getDataCenterId());
+            throw new ResourceUnavailableException("Unable to set password due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn());
         }
 
         answer = cmds.getAnswer("vmdata");
         if (answer != null && !answer.getResult()) {
             s_logger.error("Unable to set VM data for " + profile + " due to " + answer.getDetails());
-            throw new ResourceUnavailableException("Unable to set VM data due to " + answer.getDetails(), DataCenter.class, router.getDataCenterId());
+            throw new ResourceUnavailableException("Unable to set VM data due to " + answer.getDetails(), DataCenter.class, router.getDataCenterIdToDeployIn());
         }
         return router;
     }
@@ -1386,10 +1386,10 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
         Account owner = _accountMgr.getAccount(router.getAccountId());
 
         // Check if all networks are implemented for the domR; if not - implement them
-        DataCenter dc = _dcDao.findById(router.getDataCenterId());
+        DataCenter dc = _dcDao.findById(router.getDataCenterIdToDeployIn());
         HostPodVO pod = null;
-        if (router.getPodId() != null) {
-            pod = _podDao.findById(router.getPodId());
+        if (router.getPodIdToDeployIn() != null) {
+            pod = _podDao.findById(router.getPodIdToDeployIn());
         }
         DeployDestination dest = new DeployDestination(dc, pod, null, null);
 
@@ -1560,7 +1560,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
                 if (nic != null) {
                     s_logger.debug("Creating user data entry for vm " + vm + " on domR " + router);
                     String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getServiceOfferingId()).getDisplayText();
-                    String zoneName = _dcDao.findById(router.getDataCenterId()).getName();
+                    String zoneName = _dcDao.findById(router.getDataCenterIdToDeployIn()).getName();
                     cmds.addCommand("vmdata",
                             generateVmDataCommand(router, nic.getIp4Address(), vm.getUserData(), serviceOffering, zoneName, nic.getIp4Address(), vm.getHostName(), vm.getInstanceName(), vm.getId(), null));
                 }

server/src/com/cloud/network/rules/RulesManagerImpl.java

@@ -164,7 +164,7 @@ public class RulesManagerImpl implements RulesManager, RulesService, Manager {
         }
 
         // validate that userVM is in the same availability zone as the IP address
-        if (ipAddress.getDataCenterId() != userVm.getDataCenterId()) {
+        if (ipAddress.getDataCenterId() != userVm.getDataCenterIdToDeployIn()) {
             throw new InvalidParameterValueException("Unable to create ip forwarding rule, IP address " + ipAddress + " is not in the same availability zone as virtual machine " + userVm.toString());
         }
 

server/src/com/cloud/resource/ResourceManagerImpl.java

@@ -525,7 +525,8 @@ public class ResourceManagerImpl implements ResourceManager, ResourceService, Ma
         }
         _accountMgr.checkAccessAndSpecifyAuthority(UserContext.current().getCaller(), host.getDataCenterId());
         if (Host.Type.SecondaryStorage.equals(host.getType())) {
-            return _secondaryStorageMgr.destroySecStorageVm(hostId);
+            _secondaryStorageMgr.deleteHost(hostId);
+            return true;
         } else {
             return _agentMgr.deleteHost(hostId, isForced, caller);
         }

server/src/com/cloud/server/ManagementServerImpl.java

@@ -771,7 +771,7 @@ public class ManagementServerImpl implements ManagementServer {
                     DataCenterVO dc = iter.next();
                     boolean found = false;
                     for (DomainRouterVO router : routers) {
-                        if (dc.getId() == router.getDataCenterId()) {
+                        if (dc.getId() == router.getDataCenterIdToDeployIn()) {
                             found = true;
                             break;
                         }
@@ -1011,12 +1011,18 @@ public class ManagementServerImpl implements ManagementServer {
         // if a domainId is provided, we just return the so associated with this domain
         if (domainId != null) {
             if (account.getType() == Account.ACCOUNT_TYPE_ADMIN) {
+                if (account.getDomainId() != 1 && issystem){ //NON ROOT admin
+                    throw new InvalidParameterValueException("Non ROOT admins cannot access system's offering");
+                }
                 return _offeringsDao.findServiceOfferingByDomainIdAndIsSystem(domainId, issystem);// no perm check
             } else {
+                if (issystem){
+                    throw new InvalidParameterValueException("Non root users cannot access system's offering");
+                }
                 // check if the user's domain == so's domain || user's domain is a child of so's domain
                 if (isPermissible(account.getDomainId(), domainId)) {
                     // perm check succeeded
-                    return _offeringsDao.findServiceOfferingByDomainId(domainId);
+                    return _offeringsDao.findServiceOfferingByDomainIdAndIsSystem(domainId, false);
                 } else {
                     throw new PermissionDeniedException("The account:" + account.getAccountName() + " does not fall in the same domain hierarchy as the service offering");
                 }
@@ -1025,10 +1031,17 @@ public class ManagementServerImpl implements ManagementServer {
 
         // For non-root users
         if ((account.getType() == Account.ACCOUNT_TYPE_NORMAL || account.getType() == Account.ACCOUNT_TYPE_DOMAIN_ADMIN) || account.getType() == Account.ACCOUNT_TYPE_RESOURCE_DOMAIN_ADMIN) {
+            if (issystem){
+                throw new InvalidParameterValueException("Only root admins can access system's offering");
+            }
             return searchServiceOfferingsInternal(account, name, id, vmId, keyword, searchFilter);
         }
 
         // for root users, the existing flow
+        if (account.getDomainId() != 1 && issystem){ //NON ROOT admin
+            throw new InvalidParameterValueException("Non ROOT admins cannot access system's offering");
+        }
+
         if (keyword != null) {
             SearchCriteria<ServiceOfferingVO> ssc = _offeringsDao.createSearchCriteria();
             ssc.addOr("displayText", SearchCriteria.Op.LIKE, "%" + keyword + "%");
@@ -1061,7 +1074,7 @@ public class ManagementServerImpl implements ManagementServer {
         if (name != null) {
             sc.addAnd("name", SearchCriteria.Op.LIKE, "%" + name + "%");
         }
-        sc.addAnd("systemUse", SearchCriteria.Op.EQ, false);
+        sc.addAnd("systemUse", SearchCriteria.Op.EQ, issystem);
 
         return _offeringsDao.search(sc, searchFilter);
 
@@ -1307,7 +1320,7 @@ public class ManagementServerImpl implements ManagementServer {
             sc.addAnd("name", SearchCriteria.Op.LIKE, "%" + name + "%");
         }
         if (type != null) {
-            sc.addAnd("type", SearchCriteria.Op.LIKE, "%" + type + "%");
+            sc.addAnd("type", SearchCriteria.Op.LIKE, "%" + type);
         }
         if (state != null) {
             sc.addAnd("status", SearchCriteria.Op.EQ, state);
@@ -1611,7 +1624,7 @@ public class ManagementServerImpl implements ManagementServer {
 
         HypervisorType hypervisorType = HypervisorType.getType(cmd.getHypervisor());
         return listTemplates(cmd.getId(), cmd.getIsoName(), cmd.getKeyword(), isoFilter, true, cmd.isBootable(), accountId, cmd.getPageSizeVal(), cmd.getStartIndex(), cmd.getZoneId(), hypervisorType,
-                isAccountSpecific, true);
+                isAccountSpecific, true, cmd.listInReadyState());
     }
 
     @Override
@@ -1633,11 +1646,11 @@ public class ManagementServerImpl implements ManagementServer {
         HypervisorType hypervisorType = HypervisorType.getType(cmd.getHypervisor());
 
         return listTemplates(cmd.getId(), cmd.getTemplateName(), cmd.getKeyword(), templateFilter, false, null, accountId, cmd.getPageSizeVal(), cmd.getStartIndex(), cmd.getZoneId(), hypervisorType,
-                isAccountSpecific, showDomr);
+                isAccountSpecific, showDomr, cmd.listInReadyState());
     }
 
     private Set<Pair<Long, Long>> listTemplates(Long templateId, String name, String keyword, TemplateFilter templateFilter, boolean isIso, Boolean bootable, Long accountId, Long pageSize,
-            Long startIndex, Long zoneId, HypervisorType hyperType, boolean isAccountSpecific, boolean showDomr) {
+            Long startIndex, Long zoneId, HypervisorType hyperType, boolean isAccountSpecific, boolean showDomr, boolean onlyReady) {
 
         Account caller = UserContext.current().getCaller();
         VMTemplateVO template = null;
@@ -1656,10 +1669,6 @@ public class ManagementServerImpl implements ManagementServer {
             }
         }
 
-        // Show only those that are downloaded.
-        boolean onlyReady = (templateFilter == TemplateFilter.featured) || (templateFilter == TemplateFilter.selfexecutable) || (templateFilter == TemplateFilter.sharedexecutable)
-        || (templateFilter == TemplateFilter.executable && isAccountSpecific) || (templateFilter == TemplateFilter.community);
-
         Account account = null;
         DomainVO domain = null;
         if (accountId != null) {
@@ -2185,8 +2194,8 @@ public class ManagementServerImpl implements ManagementServer {
         sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ);
         sb.and("accountId", sb.entity().getAccountId(), SearchCriteria.Op.IN);
         sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ);
-        sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
-        sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ);
+        sb.and("dataCenterId", sb.entity().getDataCenterIdToDeployIn(), SearchCriteria.Op.EQ);
+        sb.and("podId", sb.entity().getPodIdToDeployIn(), SearchCriteria.Op.EQ);
         sb.and("hostId", sb.entity().getHostId(), SearchCriteria.Op.EQ);
 
         if ((accountId == null) && (domainId != null)) {
@@ -2745,7 +2754,7 @@ public class ManagementServerImpl implements ManagementServer {
     public String getConsoleAccessUrlRoot(long vmId) {
         VMInstanceVO vm = this.findVMInstanceById(vmId);
         if (vm != null) {
-            ConsoleProxyInfo proxy = getConsoleProxy(vm.getDataCenterId(), vmId);
+            ConsoleProxyInfo proxy = getConsoleProxy(vm.getDataCenterIdToDeployIn(), vmId);
             if (proxy != null) {
                 return proxy.getProxyImageUrl();
             }
@@ -3914,8 +3923,8 @@ public class ManagementServerImpl implements ManagementServer {
         sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ);
         sb.and("name", sb.entity().getHostName(), SearchCriteria.Op.LIKE);
         sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ);
-        sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
-        sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ);
+        sb.and("dataCenterId", sb.entity().getDataCenterIdToDeployIn(), SearchCriteria.Op.EQ);
+        sb.and("podId", sb.entity().getPodIdToDeployIn(), SearchCriteria.Op.EQ);
         sb.and("hostId", sb.entity().getHostId(), SearchCriteria.Op.EQ);
         sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ);
         sb.and("nulltype", sb.entity().getType(), SearchCriteria.Op.IN);

server/src/com/cloud/servlet/RegisterCompleteServlet.java

@@ -18,6 +18,7 @@
 
 package com.cloud.servlet;
 
+import java.net.URLEncoder;
 import java.util.List;
 
 import javax.servlet.ServletContextEvent;
@@ -86,6 +87,7 @@ public class RegisterCompleteServlet extends HttpServlet implements ServletConte
 			statusCode = 503;
 			responseMessage = "{ \"registration_info\" : { \"errorcode\" : \"503\", \"errortext\" : \"Missing token\" } }";
 		} else {
+			s_logger.info("Attempting to register user account with token = "+registrationToken);
 			User resourceAdminUser = _accountSvc.getActiveUserByRegistrationToken(registrationToken);
 			if (resourceAdminUser != null) {
 				if(!resourceAdminUser.isRegistered()){
@@ -101,12 +103,12 @@ public class RegisterCompleteServlet extends HttpServlet implements ServletConte
 				Configuration config = _configDao.findByName("endpointe.url");
 				
 				StringBuffer sb = new StringBuffer();
-		        sb.append("{ \"registration_info\" : { \"endpoint_url\" : \""+config.getValue()+"\", ");
+		        sb.append("{ \"registration_info\" : { \"endpoint_url\" : \""+encodeParam(config.getValue())+"\", ");
 		        sb.append("\"domain_id\" : \""+resourceAdminAccount.getDomainId()+"\", ");
-		        sb.append("\"admin_account\" : \""+resourceAdminUser.getUsername()+"\", ");
+		        sb.append("\"admin_account\" : \""+encodeParam(resourceAdminUser.getUsername())+"\", ");
 		        sb.append("\"admin_account_api_key\" : \""+resourceAdminUser.getApiKey()+"\", ");
 		        sb.append("\"admin_account_secret_key\" : \""+resourceAdminUser.getSecretKey()+"\", ");
-		        sb.append("\"user_account\" : \""+rsUser.getUsername()+"\", ");
+		        sb.append("\"user_account\" : \""+encodeParam(rsUser.getUsername())+"\", ");
 		        sb.append("\"user_account_api_key\" : \""+rsUser.getApiKey()+"\", ");
 		        sb.append("\"user_account_secret_key\" : \""+rsUser.getSecretKey()+"\" ");
 		        sb.append("} }");
@@ -125,4 +127,13 @@ public class RegisterCompleteServlet extends HttpServlet implements ServletConte
         	s_logger.error("unknown exception writing register complete response", ex);
         }
 	}
+	
+	private String encodeParam(String value) {
+		try {
+			return URLEncoder.encode(value, "UTF-8").replaceAll("\\+", "%20");
+		} catch (Exception e) {
+			s_logger.warn("Unable to encode: " + value);
+		}
+		return value;
+	}
 }

server/src/com/cloud/storage/StorageManager.java

@@ -196,4 +196,8 @@ public interface StorageManager extends Manager {
     boolean delPoolFromHost(long hostId);
 
     HostVO getSecondaryStorageHost(long zoneId, long tmpltId);
+
+    List<HostVO> getSecondaryStorageHosts(long zoneId);
+
+    List<StoragePoolVO> ListByDataCenterHypervisor(long datacenterId, HypervisorType type);
 }

server/src/com/cloud/storage/StorageManagerImpl.java

@@ -298,7 +298,7 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag
     protected int _pingInterval = 60; // seconds
     protected int _hostRetry;
     protected float _overProvisioningFactor = 1;
-    private int _maxVolumeSizeInGb;
+    private long _maxVolumeSizeInGb;
     private long _serverId;
 
     private int _snapshotTimeout;
@@ -354,6 +354,23 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag
         }
     }
     
+    @Override
+    public List<StoragePoolVO> ListByDataCenterHypervisor(long datacenterId, HypervisorType type) {
+        List<StoragePoolVO> pools = _storagePoolDao.listByDataCenterId(datacenterId);
+        List<StoragePoolVO> retPools = new ArrayList<StoragePoolVO>();
+        for (StoragePoolVO pool : pools ) {
+            if( pool.getStatus() != StoragePoolStatus.Up) {
+                continue;
+            }
+            ClusterVO cluster = _clusterDao.findById(pool.getClusterId());
+            if( type == cluster.getHypervisorType()) {
+                retPools.add(pool);
+            }
+        }
+        Collections.shuffle(retPools);
+        return retPools;
+    }
+
     @Override
     public boolean isLocalStorageActiveOnHost(Host host) {
         List<StoragePoolHostVO> storagePoolHostRefs = _storagePoolHostDao.listByHostId(host.getId());
@@ -834,7 +851,7 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag
         }
 
         String maxVolumeSizeInGbString = configDao.getValue("storage.max.volume.size");
-        _maxVolumeSizeInGb = NumbersUtil.parseInt(maxVolumeSizeInGbString, 2000);
+        _maxVolumeSizeInGb = NumbersUtil.parseLong(maxVolumeSizeInGbString, 2000);
 
         HostTemplateStatesSearch = _vmTemplateHostDao.createSearchBuilder();
         HostTemplateStatesSearch.and("id", HostTemplateStatesSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
@@ -971,6 +988,18 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag
         return hosts.get(index);
     }
     
+    @Override
+    public List<HostVO> getSecondaryStorageHosts(long zoneId) {
+        List<HostVO>  hosts = _hostDao.listSecondaryStorageHosts(zoneId);
+        if( hosts == null || hosts.size() == 0) {
+            hosts = _hostDao.listLocalSecondaryStorageHosts(zoneId);
+            if (hosts.isEmpty()) {
+                return new ArrayList<HostVO>();
+            }
+        }
+        return hosts;
+    }
+
     @Override
     public String getStoragePoolTags(long poolId) {
         return _configMgr.listToCsvTags(_storagePoolDao.searchForStoragePoolDetails(poolId, "true"));
@@ -1582,7 +1611,14 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag
             }
 
             diskOfferingId = cmd.getDiskOfferingId();
-            size = cmd.getSize() * 1024 * 1024 * 1024; // user specify size in GB
+            size = cmd.getSize();
+            if ( size != null ) {
+                if ( size > 0 ) {
+                    size = size * 1024 * 1024 * 1024; // user specify size in GB
+                } else {
+                    throw new InvalidParameterValueException("Disk size must be larger than 0");
+                }
+            }
             if (diskOfferingId == null) {
                 throw new InvalidParameterValueException("Missing parameter(s),either a positive volume size or a valid disk offering id must be specified.");
             }
@@ -2410,8 +2446,8 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag
     private boolean validateVolumeSizeRange(long size) {
         if (size < 0 || (size > 0 && size < (1024 * 1024 * 1024))) {
             throw new InvalidParameterValueException("Please specify a size of at least 1 Gb.");
-        } else if (size > _maxVolumeSizeInGb) {
-            throw new InvalidParameterValueException("The maximum size allowed is " + _maxVolumeSizeInGb + " Gb.");
+        } else if (size > (_maxVolumeSizeInGb * 1024 * 1024 * 1024) ) {
+            throw new InvalidParameterValueException("volume size " + size + ", but the maximum size allowed is " + _maxVolumeSizeInGb + " Gb.");
         }
 
         return true;
@@ -2429,7 +2465,7 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag
         } else {
             size = (size * 1024 * 1024 * 1024);
         }
-        VolumeVO vol = new VolumeVO(type, name, vm.getDataCenterId(), owner.getDomainId(), owner.getId(), offering.getId(), size);
+        VolumeVO vol = new VolumeVO(type, name, vm.getDataCenterIdToDeployIn(), owner.getDomainId(), owner.getId(), offering.getId(), size);
         if (vm != null) {
             vol.setInstanceId(vm.getId());
         }
@@ -2460,15 +2496,15 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag
         SearchCriteria<VMTemplateHostVO> sc = HostTemplateStatesSearch.create();
         sc.setParameters("id", template.getId());
         sc.setParameters("state", com.cloud.storage.VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
-        sc.setJoinParameters("host", "dcId", vm.getDataCenterId());
+        sc.setJoinParameters("host", "dcId", vm.getDataCenterIdToDeployIn());
 
         List<VMTemplateHostVO> sss = _vmTemplateHostDao.search(sc, null);
         if (sss.size() == 0) {
-            throw new CloudRuntimeException("Template " + template.getName() + " has not been completely downloaded to zone " + vm.getDataCenterId());
+            throw new CloudRuntimeException("Template " + template.getName() + " has not been completely downloaded to zone " + vm.getDataCenterIdToDeployIn());
         }
         VMTemplateHostVO ss = sss.get(0);
 
-        VolumeVO vol = new VolumeVO(type, name, vm.getDataCenterId(), owner.getDomainId(), owner.getId(), offering.getId(), ss.getSize());
+        VolumeVO vol = new VolumeVO(type, name, vm.getDataCenterIdToDeployIn(), owner.getDomainId(), owner.getId(), offering.getId(), ss.getSize());
         if (vm != null) {
             vol.setInstanceId(vm.getId());
         }
@@ -2518,7 +2554,7 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag
         if (vm.getType() == VirtualMachine.Type.User) {
             UserVmVO userVM = (UserVmVO) vm.getVirtualMachine();
             if (userVM.getIsoId() != null) {
-                Pair<String, String> isoPathPair = getAbsoluteIsoPath(userVM.getIsoId(), userVM.getDataCenterId());
+                Pair<String, String> isoPathPair = getAbsoluteIsoPath(userVM.getIsoId(), userVM.getDataCenterIdToDeployIn());
                 if (isoPathPair != null) {
                     String isoPath = isoPathPair.first();
                     VolumeTO iso = new VolumeTO(vm.getId(), Volume.Type.ISO, StoragePoolType.ISO, null, null, null, isoPath, 0, null, null);

server/src/com/cloud/storage/dao/SnapshotDao.java

@@ -37,5 +37,7 @@ public interface SnapshotDao extends GenericDao<SnapshotVO, Long> {
     List<SnapshotVO> listByVolumeIdVersion(long volumeId, String version);
     Long getSecHostId(long volumeId);
     long updateSnapshotSecHost(long dcId, long secHostId);
+    List<SnapshotVO> listByHostId(Filter filter, long hostId);
+    List<SnapshotVO> listByHostId(long hostId);
 
 }

server/src/com/cloud/storage/dao/SnapshotDaoImpl.java

@@ -28,6 +28,7 @@ import org.apache.log4j.Logger;
 
 import com.cloud.storage.SnapshotVO;
 import com.cloud.storage.Snapshot.Type;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
 import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.GenericDaoBase;
 import com.cloud.utils.db.SearchBuilder;
@@ -47,6 +48,7 @@ public class SnapshotDaoImpl extends GenericDaoBase<SnapshotVO, Long> implements
     private final SearchBuilder<SnapshotVO> ParentIdSearch;
     private final SearchBuilder<SnapshotVO> backupUuidSearch;   
     private final SearchBuilder<SnapshotVO> VolumeIdVersionSearch;
+    private final SearchBuilder<SnapshotVO> HostIdSearch;
     
     @Override
     public SnapshotVO findNextSnapshot(long snapshotId) {
@@ -85,6 +87,19 @@ public class SnapshotDaoImpl extends GenericDaoBase<SnapshotVO, Long> implements
         return listBy(sc, filter);
     }
     
+    @Override
+    public List<SnapshotVO> listByHostId(long hostId) {
+        return listByHostId(null, hostId);
+    }
+    
+    @Override
+    public List<SnapshotVO> listByHostId(Filter filter, long hostId ) {
+        SearchCriteria<SnapshotVO> sc = HostIdSearch.create();
+        sc.setParameters("hostId", hostId);
+        sc.setParameters("status", Status.DOWNLOADED);
+        return listBy(sc, filter);
+    }
+    
     @Override
     public List<SnapshotVO> listByVolumeIdIncludingRemoved(long volumeId) {
         SearchCriteria<SnapshotVO> sc = VolumeIdSearch.create();
@@ -111,6 +126,11 @@ public class SnapshotDaoImpl extends GenericDaoBase<SnapshotVO, Long> implements
         VolumeIdSearch.and("volumeId", VolumeIdSearch.entity().getVolumeId(), SearchCriteria.Op.EQ);
         VolumeIdSearch.done();
         
+        HostIdSearch = createSearchBuilder();
+        HostIdSearch.and("hostId", HostIdSearch.entity().getSecHostId(), SearchCriteria.Op.EQ);
+        HostIdSearch.and("status", HostIdSearch.entity().getStatus(), SearchCriteria.Op.EQ);
+        HostIdSearch.done();
+        
         VolumeIdTypeSearch = createSearchBuilder();
         VolumeIdTypeSearch.and("volumeId", VolumeIdTypeSearch.entity().getVolumeId(), SearchCriteria.Op.EQ);
         VolumeIdTypeSearch.and("type", VolumeIdTypeSearch.entity().getsnapshotType(), SearchCriteria.Op.EQ);

server/src/com/cloud/storage/dao/StoragePoolDaoImpl.java

@@ -280,7 +280,7 @@ public class StoragePoolDaoImpl extends GenericDaoBase<StoragePoolVO, Long>  imp
 	    sql.delete(sql.length() - 4, sql.length());
 	    sql.append(DetailsSqlSuffix);
 	    Transaction txn = Transaction.currentTxn();
-	    PreparedStatement pstmt = s_initStmt;
+	    PreparedStatement pstmt = null;
 	    try {
 	        pstmt = txn.prepareAutoCloseStatement(sql.toString());
 	        int i = 1;
@@ -297,7 +297,7 @@ public class StoragePoolDaoImpl extends GenericDaoBase<StoragePoolVO, Long>  imp
 	        }
 	        return pools;
 	    } catch (SQLException e) {
-	        throw new CloudRuntimeException("Unable to execute " + pstmt.toString(), e);
+	        throw new CloudRuntimeException("Unable to execute " + pstmt, e);
 	    }
 	}
 	

server/src/com/cloud/storage/dao/VMTemplateDao.java

@@ -71,5 +71,6 @@ public interface VMTemplateDao extends GenericDao<VMTemplateVO, Long> {
     VMTemplateVO findSystemVMTemplate(long zoneId, HypervisorType hType);
 
     VMTemplateVO findRoutingTemplate(HypervisorType type);
+    List<Long> listPrivateTemplatesByHost(Long hostId);
 	
 }

server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java

@@ -21,11 +21,13 @@ package com.cloud.storage.dao;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TimeZone;
 
 import javax.ejb.Local;
 import javax.naming.ConfigurationException;
@@ -39,6 +41,7 @@ import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.info.RunningHostCountInfo;
 import com.cloud.storage.Storage;
 import com.cloud.storage.Storage.TemplateType;
 import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
@@ -46,6 +49,7 @@ import com.cloud.storage.VMTemplateVO;
 import com.cloud.storage.VMTemplateZoneVO;
 import com.cloud.template.VirtualMachineTemplate.TemplateFilter;
 import com.cloud.user.Account;
+import com.cloud.utils.DateUtil;
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.Inject;
 import com.cloud.utils.db.DB;
@@ -139,6 +143,30 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
 		return listBy(sc, filter);
 	}
 
+    @Override
+    public List<Long> listPrivateTemplatesByHost(Long hostId) {
+
+        String sql = "select * from template_host_ref as thr INNER JOIN vm_template as t ON t.id=thr.template_id " 
+            + "where thr.host_id=? and t.public=0 and t.featured=0 and t.type='USER' and t.removed is NULL";
+
+        List<Long> l = new ArrayList<Long>();
+
+        Transaction txn = Transaction.currentTxn();
+
+        PreparedStatement pstmt = null;
+        try {
+            pstmt = txn.prepareAutoCloseStatement(sql);
+            pstmt.setLong(1, hostId);
+            ResultSet rs = pstmt.executeQuery();
+            while (rs.next()) {
+                l.add(rs.getLong(1));
+            }
+        } catch (SQLException e) {
+        } catch (Throwable e) {
+        }
+        return l;
+    }
+	
 	@Override
 	public List<VMTemplateVO> listReadyTemplates() {
 		SearchCriteria<VMTemplateVO> sc = createSearchCriteria();

server/src/com/cloud/storage/dao/VMTemplateHostDao.java

@@ -56,6 +56,8 @@ public interface VMTemplateHostDao extends GenericDao<VMTemplateHostVO, Long> {
 
     boolean templateAvailable(long templateId, long hostId);
 
-    List<VMTemplateHostVO> listByZoneTemplate(long dcId, long templateId);
+    List<VMTemplateHostVO> listByZoneTemplate(long dcId, long templateId, boolean readyOnly);
+
+    void deleteByHost(Long hostId);
 	
 }

server/src/com/cloud/storage/dao/VMTemplateHostDaoImpl.java

@@ -24,15 +24,22 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 import java.util.TimeZone;
 
 import javax.ejb.Local;
+import javax.naming.ConfigurationException;
 
 import org.apache.log4j.Logger;
+
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
 import com.cloud.storage.VMTemplateHostVO;
 import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
 import com.cloud.utils.DateUtil;
+import com.cloud.utils.component.Inject;
 import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.JoinBuilder;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.Transaction;
@@ -40,7 +47,8 @@ import com.cloud.utils.db.Transaction;
 @Local(value={VMTemplateHostDao.class})
 public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long> implements VMTemplateHostDao {
 	public static final Logger s_logger = Logger.getLogger(VMTemplateHostDaoImpl.class.getName());
-	
+    @Inject
+    HostDao   _hostDao;
 	protected final SearchBuilder<VMTemplateHostVO> HostSearch;
 	protected final SearchBuilder<VMTemplateHostVO> TemplateSearch;
 	protected final SearchBuilder<VMTemplateHostVO> HostTemplateSearch;
@@ -49,10 +57,11 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
 	protected final SearchBuilder<VMTemplateHostVO> HostTemplatePoolSearch;
 	protected final SearchBuilder<VMTemplateHostVO> TemplateStatusSearch;
 	protected final SearchBuilder<VMTemplateHostVO> TemplateStatesSearch;
+	protected SearchBuilder<VMTemplateHostVO> ZONE_TEMPLATE_SEARCH;
 
 	
 	protected static final String UPDATE_TEMPLATE_HOST_REF =
-		"UPDATE type_host_ref SET download_state = ?, download_pct= ?, last_updated = ? "
+		"UPDATE template_host_ref SET download_state = ?, download_pct= ?, last_updated = ? "
 	+   ", error_str = ?, local_path = ?, job_id = ? "
 	+   "WHERE host_id = ? and type_id = ?";
 			
@@ -71,12 +80,6 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
 		"SELECT * FROM template_host_ref t "
 	+	" where t.template_id=? and t.download_state=?";
 	
-    protected static final String ZONE_TEMPLATE_SEARCH=
-        "SELECT t.id, t.host_id, t.pool_id, t.template_id, t.created, t.last_updated, t.job_id, " 
-    +   "t.download_pct, t.size, t.physical_size, t.download_state, t.error_str, t.local_path, "
-    +   "t.install_path, t.url, t.destroyed, t.is_copy FROM template_host_ref t, host h"
-    +   " where t.host_id = h.id and h.data_center_id=? and t.template_id=? ";
-	
 	public VMTemplateHostDaoImpl () {
 		HostSearch = createSearchBuilder();
 		HostSearch.and("host_id", HostSearch.entity().getHostId(), SearchCriteria.Op.EQ);
@@ -120,6 +123,18 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
 	}
 	
 	@Override
+	public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
+	    boolean result = super.configure(name, params);
+	    ZONE_TEMPLATE_SEARCH = createSearchBuilder();
+	    ZONE_TEMPLATE_SEARCH.and("template_id", ZONE_TEMPLATE_SEARCH.entity().getTemplateId(), SearchCriteria.Op.EQ);
+	    ZONE_TEMPLATE_SEARCH.and("state", ZONE_TEMPLATE_SEARCH.entity().getDownloadState(), SearchCriteria.Op.EQ);
+	    SearchBuilder<HostVO> hostSearch = _hostDao.createSearchBuilder();
+	    hostSearch.and("zone_id", hostSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+	    ZONE_TEMPLATE_SEARCH.join("tmplHost", hostSearch, hostSearch.entity().getId(), ZONE_TEMPLATE_SEARCH.entity().getHostId(), JoinBuilder.JoinType.INNER);
+	    ZONE_TEMPLATE_SEARCH.done();
+	    return result;
+	}
+	@Override
     public void update(VMTemplateHostVO instance) {
         Transaction txn = Transaction.currentTxn();
 		PreparedStatement pstmt = null;
@@ -275,25 +290,18 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
 	}
 
     @Override
-    public List<VMTemplateHostVO> listByZoneTemplate(long dcId, long templateId) {
-        Transaction txn = Transaction.currentTxn();
-        PreparedStatement pstmt = null;
-        List<VMTemplateHostVO> result = new ArrayList<VMTemplateHostVO>();
-        try {
-            String sql = ZONE_TEMPLATE_SEARCH;
-            pstmt = txn.prepareStatement(sql);
-            
-            pstmt.setLong(1, dcId);
-            pstmt.setLong(2, templateId);
-            ResultSet rs = pstmt.executeQuery();
-            while (rs.next()) {
-                result.add(toEntityBean(rs, false));
+    public List<VMTemplateHostVO> listByZoneTemplate(long dcId, long templateId, boolean readyOnly) {
+        SearchCriteria<VMTemplateHostVO> sc = ZONE_TEMPLATE_SEARCH.create();
+        sc.setParameters("template_id", templateId);
+        sc.setJoinParameters("tmplHost", "zone_id", dcId);
+        if (readyOnly) {
+            sc.setParameters("state", VMTemplateHostVO.Status.DOWNLOADED);
+            List<VMTemplateHostVO> tmplHost = new ArrayList<VMTemplateHostVO>();
+            tmplHost.add(findOneBy(sc));
+            return tmplHost;
+        } else {
+            return listBy(sc);
         }
-            
-        } catch (Exception e) {
-            s_logger.warn("Exception: ", e);
-        }
-        return result;
     }
 	
 	@Override
@@ -323,4 +331,12 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
 	    	return lockOneRandomRow(sc, true);
 	}
 
+    @Override
+    public void deleteByHost(Long hostId) {
+        List<VMTemplateHostVO> tmpltHosts = listByHostId(hostId);
+        for (VMTemplateHostVO tmpltHost : tmpltHosts ) {
+            remove(tmpltHost.getId());
+        }
+    }
+
 }

server/src/com/cloud/storage/download/DownloadMonitorImpl.java

@@ -535,11 +535,11 @@ public class DownloadMonitorImpl implements  DownloadMonitor {
             if( tmplt.isPublicTemplate() || tmplt.isFeatured() ) {
                 continue;
             }
-            List<VMTemplateHostVO> tmpltHosts = _vmTemplateHostDao.listByZoneTemplate(dcId, tmplt.getId());
+            List<VMTemplateHostVO> tmpltHosts = _vmTemplateHostDao.listByZoneTemplate(dcId, tmplt.getId(), false);
             for ( VMTemplateHostVO tmpltHost : tmpltHosts ) {
                if ( tmpltHost.getDownloadState() == Status.DOWNLOADED || tmpltHost.getDownloadState() == Status.DOWNLOAD_IN_PROGRESS) {
                    iter.remove();
-                   continue;
+                   break;
                }
             }
         }
@@ -659,6 +659,10 @@ public class DownloadMonitorImpl implements  DownloadMonitor {
 		if (toBeDownloaded.size() > 0) {
 			/*Only download templates whose hypervirsor type is in the zone*/
 			List<HypervisorType> availHypers = _clusterDao.getAvailableHypervisorInZone(ssHost.getDataCenterId());
+			if (availHypers.isEmpty()) {
+			    /*This is for cloudzone, local secondary storage resource started before cluster created*/
+			    availHypers.add(HypervisorType.KVM);
+			}
 			/* Baremetal need not to download any template */
 			availHypers.remove(HypervisorType.BareMetal);
 			availHypers.add(HypervisorType.None); //bug 9809: resume ISO download.

server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java

@@ -71,9 +71,11 @@ import com.cloud.network.Networks.TrafficType;
 import com.cloud.offerings.NetworkOfferingVO;
 import com.cloud.service.ServiceOfferingVO;
 import com.cloud.service.dao.ServiceOfferingDao;
+import com.cloud.storage.SnapshotVO;
 import com.cloud.storage.VMTemplateHostVO;
 import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
 import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.dao.SnapshotDao;
 import com.cloud.storage.dao.StoragePoolHostDao;
 import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.storage.dao.VMTemplateHostDao;
@@ -162,6 +164,8 @@ public class SecondaryStorageManagerImpl implements SecondaryStorageVmManager, V
     private AgentManager _agentMgr;
     @Inject
     private NetworkManager _networkMgr;
+    @Inject
+    protected SnapshotDao _snapshotDao;
     
     @Inject
     private ClusterManager _clusterMgr;
@@ -282,6 +286,26 @@ public class SecondaryStorageManagerImpl implements SecondaryStorageVmManager, V
         return true;
     }
     
+    
+    @Override 
+    public boolean deleteHost(Long hostId) {
+        List<SnapshotVO> snapshots = _snapshotDao.listByHostId(hostId);
+        if( snapshots != null && !snapshots.isEmpty()) {
+            throw new CloudRuntimeException("Can not delete this secondary storage due to there are still snapshots on it ");
+        }
+        List<Long> list = _templateDao.listPrivateTemplatesByHost(hostId);
+        if( list != null && !list.isEmpty()) {
+            throw new CloudRuntimeException("Can not delete this secondary storage due to there are still private template on it ");
+        }
+        _vmTemplateHostDao.deleteByHost(hostId);
+        HostVO host = _hostDao.findById(hostId);
+        host.setGuid(null);
+        _hostDao.update(hostId, host);
+        _hostDao.remove(hostId);
+        
+        return true;
+    }
+    
     @Override
     public boolean generateVMSetupCommand(Long ssAHostId) {
         HostVO ssAHost = _hostDao.findById(ssAHostId);
@@ -356,7 +380,7 @@ public class SecondaryStorageManagerImpl implements SecondaryStorageVmManager, V
         SecStorageFirewallCfgCommand thiscpc = new SecStorageFirewallCfgCommand();
         thiscpc.addPortConfig(thisSecStorageVm.getPublicIpAddress(), copyPort, true, TemplateConstants.DEFAULT_TMPLT_COPY_INTF);
         for (SecondaryStorageVmVO ssVm : alreadyRunning) {
-            if ( ssVm.getDataCenterId() == zoneId ) {
+            if ( ssVm.getDataCenterIdToDeployIn() == zoneId ) {
                 continue;
             }
             if (ssVm.getPublicIpAddress() != null) {
@@ -844,7 +868,7 @@ public class SecondaryStorageManagerImpl implements SecondaryStorageVmManager, V
                 }
 
                 SubscriptionMgr.getInstance().notifySubscribers(ALERT_SUBJECT, this,
-                        new SecStorageVmAlertEventArgs(SecStorageVmAlertEventArgs.SSVM_REBOOTED, secStorageVm.getDataCenterId(), secStorageVm.getId(), secStorageVm, null));
+                        new SecStorageVmAlertEventArgs(SecStorageVmAlertEventArgs.SSVM_REBOOTED, secStorageVm.getDataCenterIdToDeployIn(), secStorageVm.getId(), secStorageVm, null));
 
                 return true;
             } else {
@@ -971,7 +995,7 @@ public class SecondaryStorageManagerImpl implements SecondaryStorageVmManager, V
             buf.append(" bootproto=dhcp");
         }
 
-        DataCenterVO dc = _dcDao.findById(profile.getVirtualMachine().getDataCenterId());
+        DataCenterVO dc = _dcDao.findById(profile.getVirtualMachine().getDataCenterIdToDeployIn());
         buf.append(" internaldns1=").append(dc.getInternalDns1());
         if (dc.getInternalDns2() != null) {
             buf.append(" internaldns2=").append(dc.getInternalDns2());

server/src/com/cloud/storage/secondary/SecondaryStorageVmManager.java

@@ -44,4 +44,5 @@ public interface SecondaryStorageVmManager extends Manager {
 	
 	public Pair<HostVO, SecondaryStorageVmVO> assignSecStorageVm(long zoneId, Command cmd);
     boolean generateSetupCommand(Long hostId);
+    boolean deleteHost(Long hostId);
 }

server/src/com/cloud/storage/upload/UploadMonitorImpl.java

@@ -194,15 +194,24 @@ public class UploadMonitorImpl implements UploadMonitor {
 	    String errorString = "";
 	    boolean success = false;
 	    List<HostVO> storageServers = _serverDao.listByTypeDataCenter(Host.Type.SecondaryStorage, dataCenterId);
-	    if(storageServers == null ) {
+	    if(storageServers == null || storageServers.size() == 0) {
             throw new CloudRuntimeException("No Storage Server found at the datacenter - " +dataCenterId);
         }
 	    
 	    Type type = (template.getFormat() == ImageFormat.ISO) ? Type.ISO : Type.TEMPLATE ;
 	    
-	    //Check if ssvm is up
-	    HostVO sserver = storageServers.get(0);
-	    if(sserver.getStatus() != com.cloud.host.Status.Up){
+	    List<HostVO> storageServerVMs = _serverDao.listByTypeDataCenter(Host.Type.SecondaryStorageVM, dataCenterId);
+        //Check if one ssvm is up
+        boolean no_vm_up = true;
+        HostVO use_ssvm = null;
+        for (HostVO ssvm: storageServerVMs){
+            if(ssvm.getStatus() == com.cloud.host.Status.Up){
+                no_vm_up = false;
+                use_ssvm = ssvm;
+                break;
+            }  
+        }
+        if(no_vm_up){
             throw new CloudRuntimeException("Couldnt create extract link - Secondary Storage Vm is not up");
         }
 	    
@@ -213,15 +222,15 @@ public class UploadMonitorImpl implements UploadMonitor {
         }
 	    
 	    // It doesn't exist so create a DB entry.	    
-	    UploadVO uploadTemplateObj = new UploadVO(sserver.getId(), template.getId(), new Date(), 
+	    UploadVO uploadTemplateObj = new UploadVO(use_ssvm.getId(), template.getId(), new Date(), 
 	                                                Status.DOWNLOAD_URL_NOT_CREATED, 0, type, Mode.HTTP_DOWNLOAD); 
 	    uploadTemplateObj.setInstallPath(vmTemplateHost.getInstallPath());	                                                
 	    _uploadDao.persist(uploadTemplateObj);
 	    try{
     	    // Create Symlink at ssvm
 	    	String uuid = UUID.randomUUID().toString() + ".vhd";
-    	    CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(vmTemplateHost.getInstallPath(), uuid);
-    	    long result = send(sserver.getId(), cmd, null);
+    	    CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(storageServers.get(0).getParent(), vmTemplateHost.getInstallPath(), uuid);
+    	    long result = send(use_ssvm.getId(), cmd, null);
     	    if (result == -1){
     	        errorString = "Unable to create a link for " +type+ " id:"+template.getId();
                 s_logger.error(errorString);

server/src/com/cloud/template/TemplateManagerImpl.java

@@ -262,13 +262,21 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe
         	}
         }
 
-        HostVO secondaryStorageHost = _storageMgr.getSecondaryStorageHost(zoneId);
+        List<HostVO> sservers = _storageMgr.getSecondaryStorageHosts(zoneId);
+
         VMTemplateHostVO tmpltHostRef = null;
-        if (secondaryStorageHost != null) {
+        if (sservers != null) {
+            for(HostVO secondaryStorageHost: sservers){
                 tmpltHostRef = _tmpltHostDao.findByHostTemplate(secondaryStorageHost.getId(), templateId);
-            if (tmpltHostRef != null && tmpltHostRef.getDownloadState() != com.cloud.storage.VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
+                if (tmpltHostRef != null){
+                    if (tmpltHostRef.getDownloadState() != com.cloud.storage.VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
                         throw new InvalidParameterValueException("The " + desc + " has not been downloaded ");
                     }
+                    else {
+                        break;
+                    }
+                }
+            }
         }
         
         Upload.Mode extractMode;
@@ -819,7 +827,7 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe
              vm.setIsoId(iso.getId());
             _userVmDao.update(vmId, vm);
         } 
-        if ( !attach ) {
+        if ( success && !attach ) {
             vm.setIsoId(null);
             _userVmDao.update(vmId, vm);
         }    

server/src/com/cloud/upgrade/DatabaseUpgradeChecker.java

@@ -75,10 +75,9 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker {
         _upgradeMap.put("2.2.5", new DbUpgrade[] { new Upgrade225to226()});
     }
 
-    protected void runScript(File file) {
+    protected void runScript(Connection conn, File file) {
         try {
             FileReader reader = new FileReader(file);
-            Connection conn = Transaction.getStandaloneConnection();
             ScriptRunner runner = new ScriptRunner(conn, false, true);
             runner.runScript(reader);
         } catch (FileNotFoundException e) {
@@ -142,7 +141,7 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker {
                 File[] scripts = upgrade.getPrepareScripts();
                 if (scripts != null) {
                     for (File script : scripts) {
-                        runScript(script);
+                        runScript(conn, script);
                     }
                 }
 
@@ -204,10 +203,18 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker {
 
                             txn.start();
                             
+                            Connection conn;
+                            try {
+                                conn = txn.getConnection();
+                            } catch (SQLException e) {
+                                s_logger.error("Unable to cleanup the database", e);
+                                throw new CloudRuntimeException("Unable to cleanup the database", e);
+                            }
+
                             File[] scripts = upgrade.getCleanupScripts();
                             if (scripts != null) {
                                 for (File script : scripts) {
-                                    runScript(script);
+                                    runScript(conn, script);
                                     s_logger.debug("Cleanup script " + script.getAbsolutePath() + " is executed successfully");
                                 }
                             }

server/src/com/cloud/user/AccountManagerImpl.java

@@ -511,23 +511,34 @@ public class AccountManagerImpl implements AccountManager, AccountService, Manag
                     throw new InvalidParameterValueException("Failed to list limits for account " + accountName + " no domain id specified.");
                 }
 
-                Account userAccount = _accountDao.findActiveAccount(accountName, domainId);
+                DomainVO domain = _domainDao.findById(domainId);
+                if (domain == null) {
+                    throw new InvalidParameterValueException("Unable to find domain by id " + domainId);
+                }
 
+                Account userAccount = _accountDao.findActiveAccount(accountName, domainId);
                 if (userAccount == null) {
                     throw new InvalidParameterValueException("Unable to find account " + accountName + " in domain " + domainId);
-                } else if (account != null
-                        && (account.getType() == Account.ACCOUNT_TYPE_DOMAIN_ADMIN || account.getType() == Account.ACCOUNT_TYPE_READ_ONLY_ADMIN || account.getType() == Account.ACCOUNT_TYPE_READ_ONLY_ADMIN)) {
-                    // If this is a non-root admin, make sure that the admin and the user account belong in the same domain or
-                    // that the user account's domain is a child domain of the parent
-                    if (account.getDomainId() != userAccount.getDomainId() && !_domainDao.isChildDomain(account.getDomainId(), userAccount.getDomainId())) {
-                        throw new PermissionDeniedException("You do not have permission to access limits for this account: " + accountName);
                 }
+
+                if (account != null) {
+                    checkAccess(account, domain);
                 }
 
                 accountId = userAccount.getId();
                 domainId = null;
             } else if (domainId != null) {
                 // Look up limits for the specified domain
+
+                DomainVO domain = _domainDao.findById(domainId);
+                if (domain == null) {
+                    throw new InvalidParameterValueException("Unable to find domain by id " + domainId);
+                }
+
+                if (account != null) {
+                    checkAccess(account, domain);
+                }
+
                 accountId = null;
             } else if (account == null) {
                 // Look up limits for the ROOT domain
@@ -934,7 +945,7 @@ public class AccountManagerImpl implements AccountManager, AccountService, Manag
                     s_logger.error("Unable to destroy vm: " + vm.getId());
                     accountCleanupNeeded = true;
                 }
-                UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VM_DESTROY, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), vm.getHostName(), vm.getServiceOfferingId(),
+                UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VM_DESTROY, vm.getAccountId(), vm.getDataCenterIdToDeployIn(), vm.getId(), vm.getHostName(), vm.getServiceOfferingId(),
                         vm.getTemplateId(), vm.getHypervisorType().toString());
                 _usageEventDao.persist(usageEvent);
             }

server/src/com/cloud/vm/UserVmManagerImpl.java

@@ -523,7 +523,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
         }
 
         // Check that the VM and the volume are in the same zone
-        if (vm.getDataCenterId() != volume.getDataCenterId()) {
+        if (vm.getDataCenterIdToDeployIn() != volume.getDataCenterId()) {
             throw new InvalidParameterValueException("Please specify a VM that is in the same zone as the volume.");
         }
 
@@ -561,8 +561,8 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
         if (volume.getState().equals(Volume.State.Allocated)) {
             /* Need to create the volume */
             VMTemplateVO rootDiskTmplt = _templateDao.findById(vm.getTemplateId());
-            DataCenterVO dcVO = _dcDao.findById(vm.getDataCenterId());
-            HostPodVO pod = _podDao.findById(vm.getPodId());
+            DataCenterVO dcVO = _dcDao.findById(vm.getDataCenterIdToDeployIn());
+            HostPodVO pod = _podDao.findById(vm.getPodIdToDeployIn());
             StoragePoolVO rootDiskPool = _storagePoolDao.findById(rootVolumeOfVm.getPoolId());
             ServiceOfferingVO svo = _serviceOfferingDao.findById(vm.getServiceOfferingId());
             DiskOfferingVO diskVO = _diskOfferingDao.findById(volume.getDiskOfferingId());
@@ -838,7 +838,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
         if (tmplt.getTemplateType() == TemplateType.PERHOST) {
             isoPath = tmplt.getName();
         } else {
-            isoPathPair = _storageMgr.getAbsoluteIsoPath(isoId, vm.getDataCenterId());
+            isoPathPair = _storageMgr.getAbsoluteIsoPath(isoId, vm.getDataCenterIdToDeployIn());
             if (isoPathPair == null) {
                 s_logger.warn("Couldn't get absolute iso path");
                 return false;
@@ -1105,7 +1105,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
 
         _accountMgr.incrementResourceCount(account.getId(), ResourceType.volume, new Long(volumes.size()));
 
-        UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VM_CREATE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), vm.getHostName(), vm.getServiceOfferingId(), vm.getTemplateId(), vm
+        UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VM_CREATE, vm.getAccountId(), vm.getDataCenterIdToDeployIn(), vm.getId(), vm.getHostName(), vm.getServiceOfferingId(), vm.getTemplateId(), vm
                 .getHypervisorType().toString());
         _usageEventDao.persist(usageEvent);
         txn.commit();
@@ -1456,11 +1456,11 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
                 volumeId = snapshot.getVolumeId();
 
                 String origTemplateInstallPath = null;
-                List<StoragePoolVO> storagePools = _storagePoolDao.listByDataCenterId(zoneId);
-                if (storagePools == null || storagePools.size() == 0) {
+                List<StoragePoolVO> pools = _storageMgr.ListByDataCenterHypervisor(zoneId, snapshot.getHypervisorType());
+                if (pools == null ||  pools.size() == 0 ) {
                     throw new CloudRuntimeException("Unable to find storage pools in zone " + zoneId);
                 }
-                pool = storagePools.get(0);
+                pool = pools.get(0);
                 if (snapshot.getVersion() != null && snapshot.getVersion().equalsIgnoreCase("2.1")) {
                     VolumeVO volume = _volsDao.findByIdIncludingRemoved(volumeId);
                     if (volume == null) {
@@ -1625,7 +1625,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
                         s_logger.warn("Unable to delete volume:" + volume.getId() + " for vm:" + vmId + " whilst transitioning to error state");
                     }
                 }
-                UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VM_DESTROY, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), vm.getHostName());
+                UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VM_DESTROY, vm.getAccountId(), vm.getDataCenterIdToDeployIn(), vm.getId(), vm.getHostName());
                 _usageEventDao.persist(usageEvent);
             }
         }
@@ -2192,7 +2192,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
                 if (virtualNetworks.isEmpty()) {
                     s_logger.debug("Creating default Virtual network for account " + owner + " as a part of deployVM process");
                     Network newNetwork = _networkMgr.createNetwork(defaultVirtualOffering.get(0).getId(), owner.getAccountName() + "-network", owner.getAccountName() + "-network", false, null,
-                            zone.getId(), null, null, null, null, owner, false, null);
+                            zone.getId(), null, null, null, null, owner, false, null, null);
                     defaultNetwork = _networkDao.findById(newNetwork.getId());
                 } else if (virtualNetworks.size() > 1) {
                     throw new InvalidParameterValueException("More than 1 default Virtaul networks are found for account " + owner + "; please specify networkIds");
@@ -2205,7 +2205,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
                     if (defaultVirtualOffering.get(0).getAvailability() == Availability.Optional) {
                         s_logger.debug("Creating default Virtual network for account " + owner + " as a part of deployVM process");
                         Network newNetwork = _networkMgr.createNetwork(defaultVirtualOffering.get(0).getId(), owner.getAccountName() + "-network", owner.getAccountName() + "-network", false, null,
-                                zone.getId(), null, null, null, null, owner, false, null);
+                                zone.getId(), null, null, null, null, owner, false, null, null);
                         defaultNetwork = _networkDao.findById(newNetwork.getId());
                     } else {
                         throw new InvalidParameterValueException("Unable to find default networks for account " + owner);
@@ -2400,7 +2400,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
             sshPublicKey = pair.getPublicKey();
         }
 
-        _accountMgr.checkAccess(caller, template);
+       // _accountMgr.checkAccess(caller, template);
 
         DataCenterDeployment plan = new DataCenterDeployment(zone.getId());
 
@@ -2569,7 +2569,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
             DataCenterDeployment plan = null;
             if (destinationHost != null) {
                 s_logger.debug("Destination Host to deploy the VM is specified, specifying a deployment plan to deploy the VM");
-                plan = new DataCenterDeployment(vm.getDataCenterId(), destinationHost.getPodId(), destinationHost.getClusterId(), destinationHost.getId(), null);
+                plan = new DataCenterDeployment(vm.getDataCenterIdToDeployIn(), destinationHost.getPodId(), destinationHost.getClusterId(), destinationHost.getId(), null);
             }
 
             vm = _itMgr.start(vm, params, caller, owner, plan);
@@ -2604,7 +2604,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
         }
         if (template != null && template.getFormat() == ImageFormat.ISO && vm.getIsoId() != null) {
             String isoPath = null;
-            Pair<String, String> isoPathPair = _storageMgr.getAbsoluteIsoPath(template.getId(), vm.getDataCenterId());
+            Pair<String, String> isoPathPair = _storageMgr.getAbsoluteIsoPath(template.getId(), vm.getDataCenterIdToDeployIn());
             if (isoPathPair == null) {
                 s_logger.warn("Couldn't get absolute iso path");
                 return false;
@@ -2656,7 +2656,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
     @Override
     public boolean finalizeStart(VirtualMachineProfile<UserVmVO> profile, long hostId, Commands cmds, ReservationContext context) {
         UserVmVO vm = profile.getVirtualMachine();
-        UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VM_START, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), vm.getHostName(), vm.getServiceOfferingId(), vm.getTemplateId(), vm
+        UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VM_START, vm.getAccountId(), vm.getDataCenterIdToDeployIn(), vm.getId(), vm.getHostName(), vm.getServiceOfferingId(), vm.getTemplateId(), vm
                 .getHypervisorType().toString());
         _usageEventDao.persist(usageEvent);
         Answer startAnswer = cmds.getAnswer(StartAnswer.class);
@@ -2677,7 +2677,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
         for (NicVO nic : nics) {
             NetworkVO network = _networkDao.findById(nic.getNetworkId());
             long isDefault = (nic.isDefaultNic()) ? 1 : 0;
-            usageEvent = new UsageEventVO(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), vm.getHostName(), network.getNetworkOfferingId(), null, isDefault);
+            usageEvent = new UsageEventVO(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vm.getAccountId(), vm.getDataCenterIdToDeployIn(), vm.getId(), vm.getHostName(), network.getNetworkOfferingId(), null, isDefault);
             _usageEventDao.persist(usageEvent);
             if (network.getTrafficType() == TrafficType.Guest) {
                 originalIp = nic.getIp4Address();
@@ -2698,7 +2698,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
             }
         }
         if (ipChanged) {
-            DataCenterVO dc = _dcDao.findById(vm.getDataCenterId());
+            DataCenterVO dc = _dcDao.findById(vm.getDataCenterIdToDeployIn());
             UserVmVO userVm = profile.getVirtualMachine();
             if (dc.getDhcpProvider().equalsIgnoreCase(Provider.ExternalDhcpServer.getName())){
                 _nicDao.update(guestNic.getId(), guestNic);
@@ -2769,13 +2769,13 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
     @Override
     public void finalizeStop(VirtualMachineProfile<UserVmVO> profile, StopAnswer answer) {
         VMInstanceVO vm = profile.getVirtualMachine();
-        UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VM_STOP, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), vm.getHostName());
+        UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VM_STOP, vm.getAccountId(), vm.getDataCenterIdToDeployIn(), vm.getId(), vm.getHostName());
         _usageEventDao.persist(usageEvent);
 
         List<NicVO> nics = _nicDao.listByVmId(vm.getId());
         for (NicVO nic : nics) {
             NetworkVO network = _networkDao.findById(nic.getNetworkId());
-            usageEvent = new UsageEventVO(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), null, network.getNetworkOfferingId(), null, 0L);
+            usageEvent = new UsageEventVO(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterIdToDeployIn(), vm.getId(), null, network.getNetworkOfferingId(), null, 0L);
             _usageEventDao.persist(usageEvent);
         }
     }
@@ -2853,7 +2853,7 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
                     _usageEventDao.persist(usageEvent);
                 }
             }
-            UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VM_DESTROY, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), vm.getHostName());
+            UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VM_DESTROY, vm.getAccountId(), vm.getDataCenterIdToDeployIn(), vm.getId(), vm.getHostName());
             _usageEventDao.persist(usageEvent);
 
             if (vmState != State.Error) {
@@ -2990,8 +2990,8 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager
         sb.and("stateEQ", sb.entity().getState(), SearchCriteria.Op.EQ);
         sb.and("stateNEQ", sb.entity().getState(), SearchCriteria.Op.NEQ);
         sb.and("stateNIN", sb.entity().getState(), SearchCriteria.Op.NIN);
-        sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
-        sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ);
+        sb.and("dataCenterId", sb.entity().getDataCenterIdToDeployIn(), SearchCriteria.Op.EQ);
+        sb.and("podId", sb.entity().getPodIdToDeployIn(), SearchCriteria.Op.EQ);
         sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ);
         sb.and("hostIdEQ", sb.entity().getHostId(), SearchCriteria.Op.EQ);
         sb.and("hostIdIN", sb.entity().getHostId(), SearchCriteria.Op.IN);

server/src/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -586,7 +586,7 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene
         ServiceOfferingVO offering = _offeringDao.findById(vm.getServiceOfferingId());
         VMTemplateVO template = _templateDao.findById(vm.getTemplateId());
 
-        DataCenterDeployment plan = new DataCenterDeployment(vm.getDataCenterId(), vm.getPodId(), null, null, null);
+        DataCenterDeployment plan = new DataCenterDeployment(vm.getDataCenterIdToDeployIn(), vm.getPodIdToDeployIn(), null, null, null);
         if(planToDeploy != null){
             if (s_logger.isDebugEnabled()) {
                 s_logger.debug("advanceStart: DeploymentPlan is provided, using that plan to deploy");
@@ -1318,13 +1318,13 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene
     ConcurrentOperationException, ResourceUnavailableException {
         T rebootedVm = null;
 
-        DataCenter dc = _configMgr.getZone(vm.getDataCenterId());
-        HostPodVO pod = _configMgr.getPod(vm.getPodId());
+        DataCenter dc = _configMgr.getZone(vm.getDataCenterIdToDeployIn());
         Host host = _hostDao.findById(vm.getHostId());
         Cluster cluster = null;
         if (host != null) {
             cluster = _configMgr.getCluster(host.getClusterId());
         }
+        HostPodVO pod = _configMgr.getPod(host.getPodId());
         DeployDestination dest = new DeployDestination(dc, pod, cluster, host);
 
         try {
@@ -1433,7 +1433,7 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene
         Command command = null;
 
         if (s_logger.isDebugEnabled()) {
-            s_logger.debug("VM " + serverName + ": server state = " + serverState.toString() + " and agent state = " + agentState.toString());
+            s_logger.debug("VM " + serverName + ": server state = " + serverState + " and agent state = " + agentState);
         }
 
         if (agentState == State.Error) {
@@ -1446,12 +1446,12 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene
                 alertType = AlertManager.ALERT_TYPE_CONSOLE_PROXY;
             }
 
-            HostPodVO podVO = _podDao.findById(vm.getPodId());
-            DataCenterVO dcVO = _dcDao.findById(vm.getDataCenterId());
+            HostPodVO podVO = _podDao.findById(vm.getPodIdToDeployIn());
+            DataCenterVO dcVO = _dcDao.findById(vm.getDataCenterIdToDeployIn());
             HostVO hostVO = _hostDao.findById(vm.getHostId());
 
             String hostDesc = "name: " + hostVO.getName() + " (id:" + hostVO.getId() + "), availability zone: " + dcVO.getName() + ", pod: " + podVO.getName();
-            _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodId(), "VM (name: " + vm.getInstanceName() + ", id: " + vm.getId() + ") stopped on host " + hostDesc + " due to storage failure",
+            _alertMgr.sendAlert(alertType, vm.getDataCenterIdToDeployIn(), vm.getPodIdToDeployIn(), "VM (name: " + vm.getInstanceName() + ", id: " + vm.getId() + ") stopped on host " + hostDesc + " due to storage failure",
                     "Virtual Machine " + vm.getInstanceName() + " (id: " + vm.getId() + ") running on host [" + vm.getHostId() + "] stopped due to storage failure.");
         }
 

server/src/com/cloud/vm/dao/ConsoleProxyDaoImpl.java

@@ -117,7 +117,7 @@ public class ConsoleProxyDaoImpl extends GenericDaoBase<ConsoleProxyVO, Long> im
     
     public ConsoleProxyDaoImpl() {
         DataCenterStatusSearch = createSearchBuilder();
-        DataCenterStatusSearch.and("dc", DataCenterStatusSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        DataCenterStatusSearch.and("dc", DataCenterStatusSearch.entity().getDataCenterIdToDeployIn(), SearchCriteria.Op.EQ);
         DataCenterStatusSearch.and("states", DataCenterStatusSearch.entity().getState(), SearchCriteria.Op.IN);
         DataCenterStatusSearch.done();
         
@@ -171,7 +171,7 @@ public class ConsoleProxyDaoImpl extends GenericDaoBase<ConsoleProxyVO, Long> im
         UpdateBuilder ub = getUpdateBuilder(proxy);
         ub.set(proxy, "state", State.Destroyed);
         ub.set(proxy, "privateIpAddress", null);
-        update(id, ub);
+        update(id, ub, proxy);
         
         boolean result = super.remove(id);
         txn.commit();

server/src/com/cloud/vm/dao/DomainRouterDaoImpl.java

@@ -29,9 +29,9 @@ import com.cloud.network.dao.NetworkDaoImpl;
 import com.cloud.network.router.VirtualRouter.Role;
 import com.cloud.utils.component.ComponentLocator;
 import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.JoinBuilder.JoinType;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
-import com.cloud.utils.db.JoinBuilder.JoinType;
 import com.cloud.utils.db.SearchCriteria.Op;
 import com.cloud.utils.db.Transaction;
 import com.cloud.utils.db.UpdateBuilder;
@@ -49,7 +49,7 @@ public class DomainRouterDaoImpl extends GenericDaoBase<DomainRouterVO, Long> im
 
     protected DomainRouterDaoImpl() {
         AllFieldsSearch = createSearchBuilder();
-        AllFieldsSearch.and("dc", AllFieldsSearch.entity().getDataCenterId(), Op.EQ);
+        AllFieldsSearch.and("dc", AllFieldsSearch.entity().getDataCenterIdToDeployIn(), Op.EQ);
         AllFieldsSearch.and("account", AllFieldsSearch.entity().getAccountId(), Op.EQ);
         AllFieldsSearch.and("role", AllFieldsSearch.entity().getRole(), Op.EQ);
         AllFieldsSearch.and("domainId", AllFieldsSearch.entity().getDomainId(), Op.EQ);
@@ -57,7 +57,7 @@ public class DomainRouterDaoImpl extends GenericDaoBase<DomainRouterVO, Long> im
         AllFieldsSearch.and("lastHost", AllFieldsSearch.entity().getLastHostId(), Op.EQ);
         AllFieldsSearch.and("state", AllFieldsSearch.entity().getState(), Op.EQ);
         AllFieldsSearch.and("network", AllFieldsSearch.entity().getNetworkId(), Op.EQ);
-        AllFieldsSearch.and("podId", AllFieldsSearch.entity().getPodId(), Op.EQ);
+        AllFieldsSearch.and("podId", AllFieldsSearch.entity().getPodIdToDeployIn(), Op.EQ);
         AllFieldsSearch.done();
 
         IdStatesSearch = createSearchBuilder();
@@ -83,7 +83,7 @@ public class DomainRouterDaoImpl extends GenericDaoBase<DomainRouterVO, Long> im
         router.setPublicIpAddress(null);
         UpdateBuilder ub = getUpdateBuilder(router);
         ub.set(router, "state", State.Destroyed);
-        update(id, ub);
+        update(id, ub, router);
 
         boolean result = super.remove(id);
         txn.commit();

server/src/com/cloud/vm/dao/SecondaryStorageVmDaoImpl.java

@@ -54,7 +54,7 @@ public class SecondaryStorageVmDaoImpl extends GenericDaoBase<SecondaryStorageVm
     
     public SecondaryStorageVmDaoImpl() {
         DataCenterStatusSearch = createSearchBuilder();
-        DataCenterStatusSearch.and("dc", DataCenterStatusSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        DataCenterStatusSearch.and("dc", DataCenterStatusSearch.entity().getDataCenterIdToDeployIn(), SearchCriteria.Op.EQ);
         DataCenterStatusSearch.and("states", DataCenterStatusSearch.entity().getState(), SearchCriteria.Op.IN);
         DataCenterStatusSearch.and("role", DataCenterStatusSearch.entity().getRole(), SearchCriteria.Op.EQ);
         DataCenterStatusSearch.done();
@@ -86,7 +86,7 @@ public class SecondaryStorageVmDaoImpl extends GenericDaoBase<SecondaryStorageVm
         HostUpSearch.done();
         
         ZoneSearch = createSearchBuilder();
-        ZoneSearch.and("zone", ZoneSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        ZoneSearch.and("zone", ZoneSearch.entity().getDataCenterIdToDeployIn(), SearchCriteria.Op.EQ);
         ZoneSearch.and("role", ZoneSearch.entity().getRole(), SearchCriteria.Op.EQ);
         ZoneSearch.done();
         
@@ -113,7 +113,7 @@ public class SecondaryStorageVmDaoImpl extends GenericDaoBase<SecondaryStorageVm
         UpdateBuilder ub = getUpdateBuilder(proxy);
         ub.set(proxy, "state", State.Destroyed);
         ub.set(proxy, "privateIpAddress", null);
-        update(id, ub);
+        update(id, ub, proxy);
         
         boolean result = super.remove(id);
         txn.commit();
@@ -125,8 +125,9 @@ public class SecondaryStorageVmDaoImpl extends GenericDaoBase<SecondaryStorageVm
         SearchCriteria<SecondaryStorageVmVO> sc = DataCenterStatusSearch.create();
         sc.setParameters("states", (Object[])states);
         sc.setParameters("dc", dataCenterId);
-        if(role != null)
+        if(role != null) {
             sc.setParameters("role", role);
+        }
         return listBy(sc);
     }
 
@@ -134,8 +135,9 @@ public class SecondaryStorageVmDaoImpl extends GenericDaoBase<SecondaryStorageVm
     public List<SecondaryStorageVmVO> getSecStorageVmListInStates(SecondaryStorageVm.Role role, State... states) {
         SearchCriteria<SecondaryStorageVmVO> sc = StateSearch.create();
         sc.setParameters("states", (Object[])states);
-        if(role != null)
+        if(role != null) {
             sc.setParameters("role", role);
+        }
         
         return listBy(sc);
     }
@@ -144,8 +146,9 @@ public class SecondaryStorageVmDaoImpl extends GenericDaoBase<SecondaryStorageVm
     public List<SecondaryStorageVmVO> listByHostId(SecondaryStorageVm.Role role, long hostId) {
         SearchCriteria<SecondaryStorageVmVO> sc = HostSearch.create();
         sc.setParameters("host", hostId);
-        if(role != null)
+        if(role != null) {
             sc.setParameters("role", role);
+        }
         return listBy(sc);
     }
     
@@ -154,8 +157,9 @@ public class SecondaryStorageVmDaoImpl extends GenericDaoBase<SecondaryStorageVm
         SearchCriteria<SecondaryStorageVmVO> sc = HostUpSearch.create();
         sc.setParameters("host", hostId);
         sc.setParameters("states", new Object[] {State.Destroyed, State.Stopped, State.Expunging});
-        if(role != null)
+        if(role != null) {
             sc.setParameters("role", role);
+        }
         return listBy(sc);
     }
     
@@ -166,12 +170,13 @@ public class SecondaryStorageVmDaoImpl extends GenericDaoBase<SecondaryStorageVm
         PreparedStatement pstmt = null;
         try {
         	String sql;
-        	if(role == null)
+        	if(role == null) {
                 sql = "SELECT s.id FROM secondary_storage_vm s, vm_instance v, host h " +
         			"WHERE s.id=v.id AND v.state='Running' AND v.host_id=h.id AND h.mgmt_server_id=?";
-        	else
+            } else {
                 sql = "SELECT s.id FROM secondary_storage_vm s, vm_instance v, host h " +
     				"WHERE s.id=v.id AND v.state='Running' AND s.role=? AND v.host_id=h.id AND h.mgmt_server_id=?";
+            }
         	
             pstmt = txn.prepareAutoCloseStatement(sql);
             
@@ -209,8 +214,9 @@ public class SecondaryStorageVmDaoImpl extends GenericDaoBase<SecondaryStorageVm
 	public List<SecondaryStorageVmVO> listByZoneId(SecondaryStorageVm.Role role, long zoneId) {
 		SearchCriteria<SecondaryStorageVmVO> sc = ZoneSearch.create();
         sc.setParameters("zone", zoneId);
-        if(role != null)
+        if(role != null) {
             sc.setParameters("role", role);
+        }
         return listBy(sc);
 	}
 
@@ -219,8 +225,9 @@ public class SecondaryStorageVmDaoImpl extends GenericDaoBase<SecondaryStorageVm
 		SearchCriteria<SecondaryStorageVmVO> sc = LastHostSearch.create();
 		sc.setParameters("lastHost", hostId);
 		sc.setParameters("state", State.Stopped);
-        if(role != null)
+        if(role != null) {
             sc.setParameters("role", role);
+        }
 		
 		return listBy(sc);
 	}
@@ -233,10 +240,11 @@ public class SecondaryStorageVmDaoImpl extends GenericDaoBase<SecondaryStorageVm
         PreparedStatement pstmt = null;
         try {
         	String sql;
-        	if(role == null)
+        	if(role == null) {
                 sql = "SELECT s.id, count(l.id) as count FROM secondary_storage_vm s INNER JOIN vm_instance v ON s.id=v.id LEFT JOIN cmd_exec_log l ON s.id=l.instance_id WHERE v.state='Running' AND v.data_center_id=? GROUP BY s.id ORDER BY count";
-        	else
+            } else {
                 sql = "SELECT s.id, count(l.id) as count FROM secondary_storage_vm s INNER JOIN vm_instance v ON s.id=v.id LEFT JOIN cmd_exec_log l ON s.id=l.instance_id WHERE v.state='Running' AND v.data_center_id=? AND s.role=? GROUP BY s.id ORDER BY count";
+            }
         	
             pstmt = txn.prepareAutoCloseStatement(sql);
             

server/src/com/cloud/vm/dao/UserVmDaoImpl.java

@@ -97,12 +97,12 @@ public class UserVmDaoImpl extends GenericDaoBase<UserVmVO, Long> implements Use
         
         AccountPodSearch = createSearchBuilder();
         AccountPodSearch.and("account", AccountPodSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
-        AccountPodSearch.and("pod", AccountPodSearch.entity().getPodId(), SearchCriteria.Op.EQ);
+        AccountPodSearch.and("pod", AccountPodSearch.entity().getPodIdToDeployIn(), SearchCriteria.Op.EQ);
         AccountPodSearch.done();
 
         AccountDataCenterSearch = createSearchBuilder();
         AccountDataCenterSearch.and("account", AccountDataCenterSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
-        AccountDataCenterSearch.and("dc", AccountDataCenterSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        AccountDataCenterSearch.and("dc", AccountDataCenterSearch.entity().getDataCenterIdToDeployIn(), SearchCriteria.Op.EQ);
         AccountDataCenterSearch.done();
 
         StateChangeSearch = createSearchBuilder();
@@ -125,7 +125,7 @@ public class UserVmDaoImpl extends GenericDaoBase<UserVmVO, Long> implements Use
         CountByAccountPod = createSearchBuilder(Long.class);
         CountByAccountPod.select(null, Func.COUNT, null);
         CountByAccountPod.and("account", CountByAccountPod.entity().getAccountId(), SearchCriteria.Op.EQ);
-        CountByAccountPod.and("pod", CountByAccountPod.entity().getPodId(), SearchCriteria.Op.EQ);
+        CountByAccountPod.and("pod", CountByAccountPod.entity().getPodIdToDeployIn(), SearchCriteria.Op.EQ);
         CountByAccountPod.done();
 
         _updateTimeAttr = _allAttributes.get("updateTime");
@@ -211,7 +211,7 @@ public class UserVmDaoImpl extends GenericDaoBase<UserVmVO, Long> implements Use
 
             AccountDataCenterVirtualSearch = createSearchBuilder();
             AccountDataCenterVirtualSearch.and("account", AccountDataCenterVirtualSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
-            AccountDataCenterVirtualSearch.and("dc", AccountDataCenterVirtualSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+            AccountDataCenterVirtualSearch.and("dc", AccountDataCenterVirtualSearch.entity().getDataCenterIdToDeployIn(), SearchCriteria.Op.EQ);
             AccountDataCenterVirtualSearch.join("nicSearch", nicSearch, AccountDataCenterVirtualSearch.entity().getId(), nicSearch.entity().getInstanceId(), JoinBuilder.JoinType.INNER);
             AccountDataCenterVirtualSearch.done();
         }

server/src/com/cloud/vm/dao/VMInstanceDaoImpl.java

@@ -67,13 +67,13 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
         AllFieldsSearch.and("host", AllFieldsSearch.entity().getHostId(), Op.EQ);
         AllFieldsSearch.and("lastHost", AllFieldsSearch.entity().getLastHostId(), Op.EQ);
         AllFieldsSearch.and("state", AllFieldsSearch.entity().getState(), Op.EQ);
-        AllFieldsSearch.and("zone", AllFieldsSearch.entity().getDataCenterId(), Op.EQ);
+        AllFieldsSearch.and("zone", AllFieldsSearch.entity().getDataCenterIdToDeployIn(), Op.EQ);
         AllFieldsSearch.and("type", AllFieldsSearch.entity().getType(), Op.EQ);
         AllFieldsSearch.and("account", AllFieldsSearch.entity().getAccountId(), Op.EQ);
         AllFieldsSearch.done();
 
         ZoneTemplateNonExpungedSearch = createSearchBuilder();
-        ZoneTemplateNonExpungedSearch.and("zone", ZoneTemplateNonExpungedSearch.entity().getDataCenterId(), Op.EQ);
+        ZoneTemplateNonExpungedSearch.and("zone", ZoneTemplateNonExpungedSearch.entity().getDataCenterIdToDeployIn(), Op.EQ);
         ZoneTemplateNonExpungedSearch.and("template", ZoneTemplateNonExpungedSearch.entity().getTemplateId(), Op.EQ);
         ZoneTemplateNonExpungedSearch.and("state", ZoneTemplateNonExpungedSearch.entity().getState(), Op.NEQ);
         ZoneTemplateNonExpungedSearch.done();

server/test/com/cloud/network/dao/NetworkDaoTest.java

@@ -0,0 +1,57 @@
+package com.cloud.network.dao;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import junit.framework.TestCase;
+
+import com.cloud.network.Network.GuestIpType;
+import com.cloud.network.NetworkVO;
+import com.cloud.network.Networks.BroadcastDomainType;
+import com.cloud.network.Networks.Mode;
+import com.cloud.network.Networks.TrafficType;
+import com.cloud.utils.component.ComponentLocator;
+
+
+public class NetworkDaoTest extends TestCase {
+    public void testTags() {
+        NetworkDaoImpl dao = ComponentLocator.inject(NetworkDaoImpl.class);
+        
+        dao.expunge(1001l);
+        NetworkVO network = new NetworkVO(1001, TrafficType.Control, GuestIpType.Direct, Mode.Dhcp, BroadcastDomainType.Native, 1, 1, 1, 1, 1001, "Name", "DisplayText", false, true);
+        network.setGuruName("guru_name");
+        List<String> tags = new ArrayList<String>();
+
+        tags.add("a");
+        tags.add("b");
+        network.setTags(tags);
+
+        network = dao.persist(network);
+        List<String> saveTags = network.getTags();
+        assert(saveTags.size() == 2 && saveTags.contains("a") && saveTags.contains("b"));
+
+        NetworkVO retrieved = dao.findById(1001l);
+        List<String> retrievedTags = retrieved.getTags();
+        assert(retrievedTags.size() == 2 && retrievedTags.contains("a") && retrievedTags.contains("b"));
+        
+        List<String> updateTags = new ArrayList<String>();
+        updateTags.add("e");
+        updateTags.add("f");
+        retrieved.setTags(updateTags);
+        dao.update(retrieved.getId(), retrieved);
+        
+        retrieved = dao.findById(1001l);
+        retrievedTags = retrieved.getTags();
+        assert(retrievedTags.size() == 2 && retrievedTags.contains("e") && retrievedTags.contains("f"));
+        
+        
+        dao.expunge(1001l);
+    }
+    
+    public void testListBy() {
+        NetworkDaoImpl dao = ComponentLocator.inject(NetworkDaoImpl.class);
+        
+        dao.listBy(1l, 1l, 1l, "192.168.192.0/24");
+    }
+
+}

server/test/com/cloud/vm/dao/UserVmDaoImplTest.java

@@ -0,0 +1,24 @@
+package com.cloud.vm.dao;
+
+import junit.framework.TestCase;
+
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.utils.component.ComponentLocator;
+import com.cloud.vm.UserVmVO;
+import com.cloud.vm.VirtualMachine;
+
+
+public class UserVmDaoImplTest extends TestCase {
+    public void testPersist() {
+        UserVmDao dao = ComponentLocator.inject(UserVmDaoImpl.class);
+        
+        dao.expunge(1000l);
+        
+        UserVmVO vo = new UserVmVO(1000l, "instancename", "displayname", 1, HypervisorType.XenServer, 1, true, true, 1, 1, 1, "userdata", "name");
+        dao.persist(vo);
+        
+        vo = dao.findById(1000l);
+        assert (vo.getType() == VirtualMachine.Type.User) : "Incorrect type " + vo.getType();
+    }
+
+}

setup/db/create-schema.sql

@@ -185,7 +185,7 @@ CREATE TABLE `cloud`.`networks` (
   `is_security_group_enabled` tinyint NOT NULL DEFAULT 0 COMMENT '1: enabled, 0: not',
   PRIMARY KEY (`id`),
   CONSTRAINT `fk_networks__network_offering_id` FOREIGN KEY (`network_offering_id`) REFERENCES `network_offerings`(`id`),
-  CONSTRAINT `fk_networks__data_center_id` FOREIGN KEY (`data_center_id`) REFERENCES `data_center`(`id`),
+  CONSTRAINT `fk_networks__data_center_id` FOREIGN KEY (`data_center_id`) REFERENCES `data_center`(`id`) ON DELETE CASCADE,
   CONSTRAINT `fk_networks__related` FOREIGN KEY(`related`) REFERENCES `networks`(`id`) ON DELETE CASCADE,
   CONSTRAINT `fk_networks__account_id` FOREIGN KEY(`account_id`) REFERENCES `account`(`id`),
   CONSTRAINT `fk_networks__domain_id` FOREIGN KEY(`domain_id`) REFERENCES `domain`(`id`),
@@ -197,7 +197,7 @@ CREATE TABLE `cloud`.`network_tags` (
   `network_id` bigint unsigned NOT NULL COMMENT 'id of the network',
   `tag` varchar(255) NOT NULL COMMENT 'tag',
   PRIMARY KEY (`id`),
-  CONSTRAINT `fk_network_tags__network_id` FOREIGN KEY (`network_id`) REFERENCES `networks`(`id`),
+  CONSTRAINT `fk_network_tags__network_id` FOREIGN KEY (`network_id`) REFERENCES `networks`(`id`) ON DELETE CASCADE,
   UNIQUE KEY(`network_id`, `tag`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
@@ -280,7 +280,7 @@ CREATE TABLE `cloud`.`cluster` (
   `allocation_state` varchar(32) NOT NULL DEFAULT 'Enabled' COMMENT 'Is this cluster enabled for allocation for new resources',
   `removed` datetime COMMENT 'date removed if not null',
   PRIMARY KEY (`id`),
-  CONSTRAINT `fk_cluster__data_center_id` FOREIGN KEY (`data_center_id`) REFERENCES `cloud`.`data_center`(`id`),
+  CONSTRAINT `fk_cluster__data_center_id` FOREIGN KEY (`data_center_id`) REFERENCES `cloud`.`data_center`(`id`) ON DELETE CASCADE,
   CONSTRAINT `fk_cluster__pod_id` FOREIGN KEY (`pod_id`) REFERENCES `cloud`.`host_pod_ref`(`id`),
   UNIQUE `i_cluster__pod_id__name`(`pod_id`, `name`),
   INDEX `i_cluster__allocation_state`(`allocation_state`)
@@ -490,6 +490,7 @@ CREATE TABLE  `cloud`.`data_center` (
   `is_security_group_enabled` tinyint NOT NULL DEFAULT 0 COMMENT '1: enabled, 0: not',
   `allocation_state` varchar(32) NOT NULL DEFAULT 'Enabled' COMMENT 'Is this data center enabled for allocation for new resources',
   `zone_token` varchar(255),
+  `removed` datetime COMMENT 'date removed if not null',
   PRIMARY KEY  (`id`),
   CONSTRAINT `fk_data_center__domain_id` FOREIGN KEY (`domain_id`) REFERENCES `domain`(`id`),
   INDEX `i_data_center__domain_id`(`domain_id`),
@@ -839,6 +840,7 @@ CREATE TABLE  `cloud`.`vm_instance` (
   `created` datetime NOT NULL COMMENT 'date created',
   `removed` datetime COMMENT 'date removed if not null',
   `type` varchar(32) NOT NULL COMMENT 'type of vm it is',
+  `vm_type` varchar(32) NOT NULL COMMENT 'vm type',
   `account_id` bigint unsigned NOT NULL COMMENT 'user id of owner',
   `domain_id` bigint unsigned NOT NULL,
   `service_offering_id` bigint unsigned NOT NULL COMMENT 'service offering id',