Merge branch 'master' into storage-offering-domains-zones

.travis.yml

@@ -15,12 +15,12 @@
 # specific language governing permissions and limitations
 # under the License.
 sudo: required
-dist: trusty
+dist: xenial
 group: edge
 
 language: java
 jdk:
-  - oraclejdk8
+  - openjdk8
 python:
   - "2.7"
 

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -50,6 +50,7 @@ public class ApiConstants {
     public static final String CERTIFICATE_CHAIN = "certchain";
     public static final String CERTIFICATE_FINGERPRINT = "fingerprint";
     public static final String CERTIFICATE_ID = "certid";
+    public static final String COPY_IMAGE_TAGS = "copyimagetags";
     public static final String CSR = "csr";
     public static final String PRIVATE_KEY = "privatekey";
     public static final String DOMAIN_SUFFIX = "domainsuffix";

api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java

@@ -16,16 +16,22 @@
 // under the License.
 package org.apache.cloudstack.api.command.user.account;
 
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiConstants.DomainDetails;
 import org.apache.cloudstack.api.BaseListDomainResourcesCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.ResponseObject.ResponseView;
 import org.apache.cloudstack.api.response.AccountResponse;
 import org.apache.cloudstack.api.response.ListResponse;
 
+import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.user.Account;
 
 @APICommand(name = "listAccounts", description = "Lists accounts and provides detailed account information for listed accounts", responseObject = AccountResponse.class, responseView = ResponseView.Restricted, entityType = {Account.class},
@@ -55,6 +61,12 @@ public class ListAccountsCmd extends BaseListDomainResourcesCmd {
     @Parameter(name = ApiConstants.STATE, type = CommandType.STRING, description = "list accounts by state. Valid states are enabled, disabled, and locked.")
     private String state;
 
+    @Parameter(name = ApiConstants.DETAILS,
+               type = CommandType.LIST,
+               collectionType = CommandType.STRING,
+               description = "comma separated list of account details requested, value can be a list of [ all, resource, min]")
+    private List<String> viewDetails;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -79,6 +91,25 @@ public class ListAccountsCmd extends BaseListDomainResourcesCmd {
         return state;
     }
 
+    public EnumSet<DomainDetails> getDetails() throws InvalidParameterValueException {
+        EnumSet<DomainDetails> dv;
+        if (viewDetails == null || viewDetails.size() <= 0) {
+            dv = EnumSet.of(DomainDetails.all);
+        } else {
+            try {
+                ArrayList<DomainDetails> dc = new ArrayList<DomainDetails>();
+                for (String detail : viewDetails) {
+                    dc.add(DomainDetails.valueOf(detail));
+                }
+                dv = EnumSet.copyOf(dc);
+            } catch (IllegalArgumentException e) {
+                throw new InvalidParameterValueException("The details parameter contains a non permitted value. The allowed values are " +
+                    EnumSet.allOf(DomainDetails.class));
+            }
+        }
+        return dv;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/project/ListProjectsCmd.java

@@ -16,15 +16,19 @@
 // under the License.
 package org.apache.cloudstack.api.command.user.project;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiConstants.DomainDetails;
 import org.apache.cloudstack.api.BaseListAccountResourcesCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.response.ListResponse;
@@ -61,6 +65,12 @@ public class ListProjectsCmd extends BaseListAccountResourcesCmd {
     @Parameter(name = ApiConstants.TAGS, type = CommandType.MAP, description = "List projects by tags (key/value pairs)")
     private Map tags;
 
+    @Parameter(name = ApiConstants.DETAILS,
+               type = CommandType.LIST,
+               collectionType = CommandType.STRING,
+               description = "comma separated list of project details requested, value can be a list of [ all, resource, min]")
+    private List<String> viewDetails;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -105,6 +115,25 @@ public class ListProjectsCmd extends BaseListAccountResourcesCmd {
         return tagsMap;
     }
 
+    public EnumSet<DomainDetails> getDetails() throws InvalidParameterValueException {
+        EnumSet<DomainDetails> dv;
+        if (viewDetails == null || viewDetails.size() <= 0) {
+            dv = EnumSet.of(DomainDetails.all);
+        } else {
+            try {
+                ArrayList<DomainDetails> dc = new ArrayList<DomainDetails>();
+                for (String detail : viewDetails) {
+                    dc.add(DomainDetails.valueOf(detail));
+                }
+                dv = EnumSet.copyOf(dc);
+            } catch (IllegalArgumentException e) {
+                throw new InvalidParameterValueException("The details parameter contains a non permitted value. The allowed values are " +
+                    EnumSet.allOf(DomainDetails.class));
+            }
+        }
+        return dv;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java

@@ -203,6 +203,9 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
     @Parameter(name = ApiConstants.EXTRA_CONFIG, type = CommandType.STRING, since = "4.12", description = "an optional URL encoded string that can be passed to the virtual machine upon successful deployment", length = 5120)
     private String extraConfig;
 
+    @Parameter(name = ApiConstants.COPY_IMAGE_TAGS, type = CommandType.BOOLEAN, since = "4.13", description = "if true the image tags (if any) will be copied to the VM, default value is false")
+    private Boolean copyImageTags;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -489,6 +492,10 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
         return extraConfig;
     }
 
+    public boolean getCopyImageTags() {
+        return copyImageTags == null ? false : copyImageTags;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

debian/control

@@ -15,7 +15,7 @@ Description: A common package which contains files which are shared by several C
 
 Package: cloudstack-management
 Architecture: all
-Depends: ${python:Depends}, openjdk-8-jre-headless | java8-runtime-headless | java8-runtime | openjdk-9-jre-headless, cloudstack-common (= ${source:Version}), sudo, python-mysql.connector, libmysql-java, augeas-tools, mysql-client, adduser, bzip2, ipmitool, lsb-release, init-system-helpers (>= 1.14~)
+Depends: ${python:Depends}, openjdk-8-jre-headless | java8-runtime-headless | java8-runtime | openjdk-9-jre-headless, cloudstack-common (= ${source:Version}), sudo, python-mysql.connector, libmysql-java, augeas-tools, mysql-client, adduser, bzip2, ipmitool, file, lsb-release, init-system-helpers (>= 1.14~)
 Conflicts: cloud-server, cloud-client, cloud-client-ui
 Description: CloudStack server library
  The CloudStack management server

engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java

@@ -122,7 +122,7 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
 
     List<Long> listHostIdsByVmCount(long dcId, Long podId, Long clusterId, long accountId);
 
-    Long countRunningByAccount(long accountId);
+    Long countRunningAndStartingByAccount(long accountId);
 
     Long countByZoneAndState(long zoneId, State state);
 

engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java

@@ -87,7 +87,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
     protected SearchBuilder<VMInstanceVO> HostNameAndZoneSearch;
     protected GenericSearchBuilder<VMInstanceVO, Long> FindIdsOfVirtualRoutersByAccount;
     protected GenericSearchBuilder<VMInstanceVO, Long> CountActiveByHost;
-    protected GenericSearchBuilder<VMInstanceVO, Long> CountRunningByAccount;
+    protected GenericSearchBuilder<VMInstanceVO, Long> CountRunningAndStartingByAccount;
     protected GenericSearchBuilder<VMInstanceVO, Long> CountByZoneAndState;
     protected SearchBuilder<VMInstanceVO> NetworkTypeSearch;
     protected GenericSearchBuilder<VMInstanceVO, String> DistinctHostNameSearch;
@@ -245,11 +245,11 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
         CountActiveByHost.and("state", CountActiveByHost.entity().getState(), SearchCriteria.Op.IN);
         CountActiveByHost.done();
 
-        CountRunningByAccount = createSearchBuilder(Long.class);
+        CountRunningAndStartingByAccount = createSearchBuilder(Long.class);
-        CountRunningByAccount.select(null, Func.COUNT, null);
+        CountRunningAndStartingByAccount.select(null, Func.COUNT, null);
-        CountRunningByAccount.and("account", CountRunningByAccount.entity().getAccountId(), SearchCriteria.Op.EQ);
+        CountRunningAndStartingByAccount.and("account", CountRunningAndStartingByAccount.entity().getAccountId(), SearchCriteria.Op.EQ);
-        CountRunningByAccount.and("state", CountRunningByAccount.entity().getState(), SearchCriteria.Op.EQ);
+        CountRunningAndStartingByAccount.and("states", CountRunningAndStartingByAccount.entity().getState(), SearchCriteria.Op.IN);
-        CountRunningByAccount.done();
+        CountRunningAndStartingByAccount.done();
 
         CountByZoneAndState = createSearchBuilder(Long.class);
         CountByZoneAndState.select(null, Func.COUNT, null);
@@ -749,10 +749,10 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
     }
 
     @Override
-    public Long countRunningByAccount(long accountId) {
+    public Long countRunningAndStartingByAccount(long accountId) {
-        SearchCriteria<Long> sc = CountRunningByAccount.create();
+        SearchCriteria<Long> sc = CountRunningAndStartingByAccount.create();
         sc.setParameters("account", accountId);
-        sc.setParameters("state", State.Running);
+        sc.setParameters("states", new Object[] {State.Starting, State.Running});
         return customSearch(sc, null).get(0);
     }
 

engine/schema/src/main/resources/META-INF/db/schema-41200to41300.sql

@@ -23,6 +23,10 @@
 INSERT IGNORE INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) values (UUID(), 'VMware', '6.7', 128, 0, 13, 32, 1, 1);
 INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'VMware', '6.7', guest_os_name, guest_os_id, utc_timestamp(), 0  FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='VMware' AND hypervisor_version='6.5';
 
+-- XenServer 7.1.2
+INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, max_data_volumes_limit, storage_motion_supported) values (UUID(), 'XenServer', '7.1.2', 500, 13, 1);
+INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '7.1.2', guest_os_name, guest_os_id, utc_timestamp(), 0  FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='7.1.0';
+
 -- XenServer 7.6
 INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, max_data_volumes_limit, storage_motion_supported) values (UUID(), 'XenServer', '7.6.0', 500, 13, 1);
 INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '7.6.0', guest_os_name, guest_os_id, utc_timestamp(), 0  FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='7.5.0';
@@ -324,4 +328,34 @@ CREATE VIEW `cloud`.`data_center_view` AS
             left join
         `cloud`.`dedicated_resources` ON data_center.id = dedicated_resources.data_center_id
             left join
-        `cloud`.`affinity_group` ON dedicated_resources.affinity_group_id = affinity_group.id;
+        `cloud`.`affinity_group` ON dedicated_resources.affinity_group_id = affinity_group.id;
+
+-- Remove key/value tags from project_view
+DROP VIEW IF EXISTS `cloud`.`project_view`;
+CREATE VIEW `cloud`.`project_view` AS
+    select
+        projects.id,
+        projects.uuid,
+        projects.name,
+        projects.display_text,
+        projects.state,
+        projects.removed,
+        projects.created,
+        projects.project_account_id,
+        account.account_name owner,
+        pacct.account_id,
+        domain.id domain_id,
+        domain.uuid domain_uuid,
+        domain.name domain_name,
+        domain.path domain_path
+    from
+        `cloud`.`projects`
+            inner join
+        `cloud`.`domain` ON projects.domain_id = domain.id
+            inner join
+        `cloud`.`project_account` ON projects.id = project_account.project_id
+            and project_account.account_role = 'Admin'
+            inner join
+        `cloud`.`account` ON account.id = project_account.account_id
+            left join
+        `cloud`.`project_account` pacct ON projects.id = pacct.project_id;

framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java

@@ -44,7 +44,7 @@ import java.util.Map;
 import java.util.TimeZone;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
-import com.google.common.base.Strings;
+
 import javax.naming.ConfigurationException;
 import javax.persistence.AttributeOverride;
 import javax.persistence.Column;
@@ -55,16 +55,6 @@ import javax.persistence.Enumerated;
 import javax.persistence.Table;
 import javax.persistence.TableGenerator;
 
-import net.sf.cglib.proxy.Callback;
-import net.sf.cglib.proxy.CallbackFilter;
-import net.sf.cglib.proxy.Enhancer;
-import net.sf.cglib.proxy.Factory;
-import net.sf.cglib.proxy.MethodInterceptor;
-import net.sf.cglib.proxy.NoOp;
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheManager;
-import net.sf.ehcache.Element;
-
 import org.apache.log4j.Logger;
 
 import com.cloud.utils.DateUtil;
@@ -79,6 +69,17 @@ import com.cloud.utils.db.SearchCriteria.SelectType;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.net.Ip;
 import com.cloud.utils.net.NetUtils;
+import com.google.common.base.Strings;
+
+import net.sf.cglib.proxy.Callback;
+import net.sf.cglib.proxy.CallbackFilter;
+import net.sf.cglib.proxy.Enhancer;
+import net.sf.cglib.proxy.Factory;
+import net.sf.cglib.proxy.MethodInterceptor;
+import net.sf.cglib.proxy.NoOp;
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.Element;
 
 /**
  *  GenericDaoBase is a simple way to implement DAOs.  It DOES NOT
@@ -2014,8 +2015,6 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
             }
         }
 
-        // we have to disable group by in getting count, since count for groupBy clause will be different.
-        //List<Object> groupByValues = addGroupBy(str, sc);
         final TransactionLegacy txn = TransactionLegacy.currentTxn();
         final String sql = str.toString();
 
@@ -2033,14 +2032,6 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
                 i = addJoinAttributes(i, pstmt, joins);
             }
 
-            /*
-            if (groupByValues != null) {
-                for (Object value : groupByValues) {
-                    pstmt.setObject(i++, value);
-                }
-            }
-             */
-
             final ResultSet rs = pstmt.executeQuery();
             while (rs.next()) {
                 return rs.getInt(1);
@@ -2056,6 +2047,13 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
     @DB()
     protected StringBuilder createCountSelect(SearchCriteria<?> sc, final boolean whereClause) {
         StringBuilder sql = new StringBuilder(_count);
+        if (sc != null) {
+            Pair<GroupBy<?, ?, ?>, List<Object>> groupBys = sc.getGroupBy();
+            if (groupBys != null) {
+                final SqlGenerator generator = new SqlGenerator(_entityBeanType);
+                sql = new StringBuilder(generator.buildCountSqlWithGroupBy(groupBys.first()));
+            }
+        }
 
         if (!whereClause) {
             sql.delete(sql.length() - (_discriminatorClause == null ? 6 : 4), sql.length());

framework/db/src/main/java/com/cloud/utils/db/GroupBy.java

@@ -63,6 +63,13 @@ public class GroupBy<J extends SearchBase<?, T, R>, T, R> {
 
     public void toSql(final StringBuilder builder) {
         builder.append(" GROUP BY ");
+        appendGroupByAttributes(builder);
+        if (_having != null) {
+            _having.toSql(builder);
+        }
+    }
+
+    public void appendGroupByAttributes(final StringBuilder builder) {
         for (final Pair<Func, Attribute> groupBy : _groupBys) {
             if (groupBy.first() != null) {
                 String func = groupBy.first().toString();
@@ -71,14 +78,9 @@ public class GroupBy<J extends SearchBase<?, T, R>, T, R> {
             } else {
                 builder.append(groupBy.second().table + "." + groupBy.second().columnName);
             }
-
             builder.append(", ");
         }
-
         builder.delete(builder.length() - 2, builder.length());
-        if (_having != null) {
-            _having.toSql(builder);
-        }
     }
 
     protected class Having {

framework/db/src/main/java/com/cloud/utils/db/SqlGenerator.java

@@ -675,6 +675,14 @@ public class SqlGenerator {
         return sql.append("SELECT COUNT(*) FROM ").append(buildTableReferences()).append(" WHERE ").append(buildDiscriminatorClause().first()).toString();
     }
 
+    public String buildCountSqlWithGroupBy(final GroupBy groupBy) {
+        StringBuilder sql = new StringBuilder();
+        sql.append("SELECT COUNT(DISTINCT ");
+        groupBy.appendGroupByAttributes(sql);
+        sql.append(") FROM ").append(buildTableReferences()).append(" WHERE ").append(buildDiscriminatorClause().first());
+        return sql.toString();
+    }
+
     public String buildDistinctIdSql() {
         StringBuilder sql = new StringBuilder();
 

framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java

@@ -53,7 +53,7 @@ public class QuotaTypes extends UsageTypes {
         quotaTypeList.put(VM_DISK_IO_READ, new QuotaTypes(VM_DISK_IO_READ, "VM_DISK_IO_READ", "GB", "VM Disk usage(I/O Read)"));
         quotaTypeList.put(VM_DISK_IO_WRITE, new QuotaTypes(VM_DISK_IO_WRITE, "VM_DISK_IO_WRITE", "GB", "VM Disk usage(I/O Write)"));
         quotaTypeList.put(VM_DISK_BYTES_READ, new QuotaTypes(VM_DISK_BYTES_READ, "VM_DISK_BYTES_READ", "GB", "VM Disk usage(Bytes Read)"));
-        quotaTypeList.put(VM_DISK_BYTES_WRITE, new QuotaTypes(VM_DISK_BYTES_WRITE, "VPN_USERS", "GB", "VM Disk usage(Bytes Write)"));
+        quotaTypeList.put(VM_DISK_BYTES_WRITE, new QuotaTypes(VM_DISK_BYTES_WRITE, "VM_DISK_BYTES_WRITE", "GB", "VM Disk usage(Bytes Write)"));
         quotaTypeList.put(VM_SNAPSHOT, new QuotaTypes(VM_SNAPSHOT, "VM_SNAPSHOT", "GB-Month", "VM Snapshot storage usage"));
         quotaTypeList.put(VOLUME_SECONDARY, new QuotaTypes(VOLUME_SECONDARY, "VOLUME_SECONDARY", "GB-Month", "Volume secondary storage usage"));
         quotaTypeList.put(VM_SNAPSHOT_ON_PRIMARY, new QuotaTypes(VM_SNAPSHOT_ON_PRIMARY, "VM_SNAPSHOT_ON_PRIMARY", "GB-Month", "VM Snapshot primary storage usage"));

plugins/deployment-planners/user-dispersing/src/main/java/com/cloud/deploy/UserDispersingPlanner.java

@@ -146,7 +146,7 @@ public class UserDispersingPlanner extends FirstFitPlanner implements Deployment
         //normalize the vmCountMap
         LinkedHashMap<Long, Double> normalisedVmCountIdMap = new LinkedHashMap<Long, Double>();
 
-        Long totalVmsOfAccount = vmInstanceDao.countRunningByAccount(accountId);
+        Long totalVmsOfAccount = vmInstanceDao.countRunningAndStartingByAccount(accountId);
         if (s_logger.isDebugEnabled()) {
             s_logger.debug("Total VMs for account: " + totalVmsOfAccount);
         }

plugins/network-elements/juniper-srx/src/main/java/com/cloud/network/resource/JuniperSrxResource.java

@@ -177,7 +177,15 @@ public class JuniperSrxResource implements ServerResource {
         private static final Logger s_logger = Logger.getLogger(JuniperSrxResource.class);
 
         private SrxXml(String filename) {
-            xml = getXml(filename);
+            String contents = getXml(filename);
+
+            // Strip the apache header and add the filename as a header to aid debugging
+            contents = contents.replaceAll( "(?s)<!--.*?-->", "" ).trim();
+            if (!contents.startsWith("<?")) {
+                contents = "<!-- " + filename + " -->" + contents;
+            }
+
+            xml = contents;
         }
 
         public String getXml() {
@@ -2031,62 +2039,69 @@ public class JuniperSrxResource implements ServerResource {
                 xml = replaceXmlValue(xml, "rule-name", ruleName_private);
                 return sendRequestAndCheckResponse(command, xml, "name", ruleName_private);
             case ADD:
-                if (manageStaticNatRule(SrxCommand.CHECK_IF_EXISTS, publicIp, privateIp)) {
+                if (!manageStaticNatRule(SrxCommand.CHECK_IF_EXISTS, publicIp, privateIp)) {
-                    return true;
+                    xml = SrxXml.STATIC_NAT_RULE_ADD.getXml();
+                    xml = replaceXmlValue(xml, "rule-set", _publicZone);
+                    xml = replaceXmlValue(xml, "from-zone", _publicZone);
+                    xml = replaceXmlValue(xml, "rule-name", ruleName);
+                    xml = replaceXmlValue(xml, "original-ip", publicIp);
+                    xml = replaceXmlValue(xml, "translated-ip", privateIp);
+
+                    if (!sendRequestAndCheckResponse(command, xml)) {
+                        throw new ExecutionException(String.format("Failed to add static NAT rule %s -> %s on %s ", publicIp, privateIp, _publicZone));
+                    }
+                } else {
+                    s_logger.debug(String.format("Static NAT rule %s -> %s on %s already exists", publicIp, privateIp, _publicZone));
                 }
 
-                xml = SrxXml.STATIC_NAT_RULE_ADD.getXml();
+                if (!manageStaticNatRule(SrxCommand.CHECK_PRIVATE_IF_EXISTS, publicIp, privateIp)) {
-                xml = replaceXmlValue(xml, "rule-set", _publicZone);
-                xml = replaceXmlValue(xml, "from-zone", _publicZone);
-                xml = replaceXmlValue(xml, "rule-name", ruleName);
-                xml = replaceXmlValue(xml, "original-ip", publicIp);
-                xml = replaceXmlValue(xml, "translated-ip", privateIp);
-
-                if (!sendRequestAndCheckResponse(command, xml)) {
-                    throw new ExecutionException("Failed to add static NAT rule from public IP " + publicIp + " to private IP " + privateIp);
-                } else {
                     xml = SrxXml.STATIC_NAT_RULE_ADD.getXml();
                     xml = replaceXmlValue(xml, "rule-set", _privateZone);
                     xml = replaceXmlValue(xml, "from-zone", _privateZone);
                     xml = replaceXmlValue(xml, "rule-name", ruleName_private);
                     xml = replaceXmlValue(xml, "original-ip", publicIp);
                     xml = replaceXmlValue(xml, "translated-ip", privateIp);
-                    if (!sendRequestAndCheckResponse(command, xml))
+                    if (!sendRequestAndCheckResponse(command, xml)) {
-                    {
+                        throw new ExecutionException(String.format("Failed to add static NAT rule %s -> %s on %s ", publicIp, privateIp, _privateZone));
-                        throw new ExecutionException("Failed to add trust static NAT rule from public IP " + publicIp + " to private IP " + privateIp);
                     }
-                    return true;
+                } else {
+                    s_logger.debug(String.format("Static NAT rule %s -> %s on %s already exists", publicIp, privateIp, _privateZone));
                 }
 
+                return true;
+
             case DELETE:
-                if (!manageStaticNatRule(SrxCommand.CHECK_IF_EXISTS, publicIp, privateIp)) {
+                if (manageStaticNatRule(SrxCommand.CHECK_IF_EXISTS, publicIp, privateIp)) {
-                    return true;
+                    xml = SrxXml.STATIC_NAT_RULE_GETONE.getXml();
-                }
+                    xml = setDelete(xml, true);
+                    xml = replaceXmlValue(xml, "rule-set", _publicZone);
+                    xml = replaceXmlValue(xml, "from-zone", _publicZone);
+                    xml = replaceXmlValue(xml, "rule-name", ruleName);
 
-                xml = SrxXml.STATIC_NAT_RULE_GETONE.getXml();
+                    if (!sendRequestAndCheckResponse(command, xml, "name", ruleName)) {
-                xml = setDelete(xml, true);
+                        throw new ExecutionException(String.format("Failed to delete static NAT rule %s -> %s on %s", publicIp, privateIp, _publicZone));
-                xml = replaceXmlValue(xml, "rule-set", _publicZone);
-                xml = replaceXmlValue(xml, "from-zone", _publicZone);
-                xml = replaceXmlValue(xml, "rule-name", ruleName);
-
-                if (!sendRequestAndCheckResponse(command, xml, "name", ruleName)) {
-                    throw new ExecutionException("Failed to delete static NAT rule from public IP " + publicIp + " to private IP " + privateIp);
-                } else {
-                    if (manageStaticNatRule(SrxCommand.CHECK_PRIVATE_IF_EXISTS, publicIp, privateIp)){
-                        xml = SrxXml.STATIC_NAT_RULE_GETONE.getXml();
-                        xml = setDelete(xml, true);
-                        xml = replaceXmlValue(xml, "rule-set", _privateZone);
-                        xml = replaceXmlValue(xml, "from-zone", _privateZone);
-                        xml = replaceXmlValue(xml, "rule-name", ruleName_private);
-
-                        if (!sendRequestAndCheckResponse(command, xml, "name", ruleName_private))
-                        {
-                            throw new ExecutionException("Failed to delete trust static NAT rule from public IP " + publicIp + " to private IP " + privateIp);
-                        }
                     }
-                    return true;
+                } else {
+                    s_logger.debug(String.format("Static NAT rule %s -> %s on %s not found", publicIp, privateIp, _publicZone));
                 }
 
+                if (manageStaticNatRule(SrxCommand.CHECK_PRIVATE_IF_EXISTS, publicIp, privateIp)){
+                    xml = SrxXml.STATIC_NAT_RULE_GETONE.getXml();
+                    xml = setDelete(xml, true);
+                    xml = replaceXmlValue(xml, "rule-set", _privateZone);
+                    xml = replaceXmlValue(xml, "from-zone", _privateZone);
+                    xml = replaceXmlValue(xml, "rule-name", ruleName_private);
+
+                    if (!sendRequestAndCheckResponse(command, xml, "name", ruleName_private))
+                    {
+                        throw new ExecutionException(String.format("Failed to delete static NAT rule %s -> %s on %s", publicIp, privateIp, _privateZone));
+                    }
+                } else {
+                    s_logger.debug(String.format("Static NAT rule %s -> %s on %s not found", publicIp, privateIp, _privateZone));
+                }
+
+                return true;
+
             default:
                 throw new ExecutionException("Unrecognized command.");
 
@@ -2163,39 +2178,39 @@ public class JuniperSrxResource implements ServerResource {
                 return sendRequestAndCheckResponse(command, xml, "pool-name", poolName);
 
             case ADD:
-                if (manageDestinationNatPool(SrxCommand.CHECK_IF_EXISTS, privateIp, destPort)) {
+                if (!manageDestinationNatPool(SrxCommand.CHECK_IF_EXISTS, privateIp, destPort)) {
-                    return true;
+                    xml = SrxXml.DEST_NAT_POOL_ADD.getXml();
-                }
+                    xml = replaceXmlValue(xml, "pool-name", poolName);
+                    xml = replaceXmlValue(xml, "private-address", privateIp + "/32");
+                    xml = replaceXmlValue(xml, "dest-port", String.valueOf(destPort));
 
-                xml = SrxXml.DEST_NAT_POOL_ADD.getXml();
+                    if (!sendRequestAndCheckResponse(command, xml)) {
-                xml = replaceXmlValue(xml, "pool-name", poolName);
+                        throw new ExecutionException(String.format("Failed to add Destination NAT pool for %s:%s", privateIp, destPort));
-                xml = replaceXmlValue(xml, "private-address", privateIp + "/32");
+                    }
-                xml = replaceXmlValue(xml, "dest-port", String.valueOf(destPort));
-
-                if (!sendRequestAndCheckResponse(command, xml)) {
-                    throw new ExecutionException("Failed to add destination NAT pool for private IP " + privateIp + " and private port " + destPort);
                 } else {
+                    s_logger.debug(String.format("Destination NAT pool for %s:%s already exists", privateIp, destPort));
                     return true;
                 }
 
+                return true;
+
             case DELETE:
-                if (!manageDestinationNatPool(SrxCommand.CHECK_IF_EXISTS, privateIp, destPort)) {
+                if (manageDestinationNatPool(SrxCommand.CHECK_IF_EXISTS, privateIp, destPort)) {
-                    return true;
+                    if (!manageDestinationNatPool(SrxCommand.CHECK_IF_IN_USE, privateIp, destPort)) {
-                }
+                        xml = SrxXml.DEST_NAT_POOL_GETONE.getXml();
+                        xml = setDelete(xml, true);
+                        xml = replaceXmlValue(xml, "pool-name", poolName);
 
-                if (manageDestinationNatPool(SrxCommand.CHECK_IF_IN_USE, privateIp, destPort)) {
+                        if (!sendRequestAndCheckResponse(command, xml)) {
-                    return true;
+                            throw new ExecutionException(String.format("Failed to delete Destination NAT pool for %s:%s", privateIp, destPort));
-                }
+                        }
-
+                    } else {
-                xml = SrxXml.DEST_NAT_POOL_GETONE.getXml();
+                        s_logger.debug(String.format("Destination NAT pool for %s:%s is in use, not deleting", privateIp, destPort));
-                xml = setDelete(xml, true);
+                    }
-                xml = replaceXmlValue(xml, "pool-name", poolName);
-
-                if (!sendRequestAndCheckResponse(command, xml)) {
-                    throw new ExecutionException("Failed to delete destination NAT pool for private IP " + privateIp + " and private port " + destPort);
                 } else {
-                    return true;
+                    s_logger.debug(String.format("Did not find Destination NAT pool for %s:%s to delete", privateIp, destPort));
                 }
+                return true;
 
             default:
                 throw new ExecutionException("Unrecognized command.");
@@ -2234,28 +2249,31 @@ public class JuniperSrxResource implements ServerResource {
                 xml = replaceXmlValue(xml, "rule-name", ruleName_private);
                 return sendRequestAndCheckResponse(command, xml, "name", ruleName_private);
             case ADD:
-                if (manageDestinationNatRule(SrxCommand.CHECK_IF_EXISTS, publicIp, privateIp, srcPort, destPort)) {
+                // Add untrust rule
-                    return true;
+                if (!manageDestinationNatRule(SrxCommand.CHECK_IF_EXISTS, publicIp, privateIp, srcPort, destPort)) {
-                }
+                    if (!manageDestinationNatPool(SrxCommand.CHECK_IF_EXISTS, privateIp, destPort)) {       // Added elsewhere
+                        throw new ExecutionException(String.format("Destination NAT pool for %s:%s does not exist", privateIp, destPort));
+                    }
 
-                if (!manageDestinationNatPool(SrxCommand.CHECK_IF_EXISTS, privateIp, destPort)) {
+                    xml = SrxXml.DEST_NAT_RULE_ADD.getXml();
-                    throw new ExecutionException("The destination NAT pool corresponding to private IP: " + privateIp + " and destination port: " + destPort +
+                    xml = replaceXmlValue(xml, "rule-set", _publicZone);
-                        " does not exist.");
+                    xml = replaceXmlValue(xml, "from-zone", _publicZone);
-                }
+                    xml = replaceXmlValue(xml, "rule-name", ruleName);
+                    xml = replaceXmlValue(xml, "public-address", publicIp);
+                    xml = replaceXmlValue(xml, "src-port", String.valueOf(srcPort));
+                    xml = replaceXmlValue(xml, "pool-name", poolName);
 
-                xml = SrxXml.DEST_NAT_RULE_ADD.getXml();
+                    if (!sendRequestAndCheckResponse(command, xml)) {
-                xml = replaceXmlValue(xml, "rule-set", _publicZone);
+                        throw new ExecutionException(String.format("Failed to add Destination NAT rule %s:%s -> %s:%s on %s",
-                xml = replaceXmlValue(xml, "from-zone", _publicZone);
+                            publicIp, srcPort, privateIp, destPort, _publicZone));
-                xml = replaceXmlValue(xml, "rule-name", ruleName);
+                    }
-                xml = replaceXmlValue(xml, "public-address", publicIp);
-                xml = replaceXmlValue(xml, "src-port", String.valueOf(srcPort));
-                xml = replaceXmlValue(xml, "pool-name", poolName);
-
-                if (!sendRequestAndCheckResponse(command, xml)) {
-                    throw new ExecutionException("Failed to add destination NAT rule from public IP " + publicIp + ", public port " + srcPort + ", private IP " +
-                        privateIp + ", and private port " + destPort);
                 } else {
+                    s_logger.debug(String.format("Destination NAT rule for %s:%s -> %s:%s on %s already exists",
+                        publicIp, srcPort, privateIp, destPort, _publicZone));
+                }
 
+                // Add trust rule
+                if (!manageDestinationNatRule(SrxCommand.CHECK_PRIVATE_IF_EXISTS, publicIp, privateIp, srcPort, destPort)) {
                     xml = SrxXml.DEST_NAT_RULE_ADD.getXml();
                     xml = replaceXmlValue(xml, "rule-set", _privateZone);
                     xml = replaceXmlValue(xml, "from-zone", _privateZone);
@@ -2266,45 +2284,54 @@ public class JuniperSrxResource implements ServerResource {
 
                     if (!sendRequestAndCheckResponse(command, xml))
                     {
-                        s_logger.debug("Purple: loopback Failed to add " + _privateZone + " destination NAT rule from public IP " + publicIp + ", public port " + srcPort + ", private IP " +
+                        throw new ExecutionException(String.format("Failed to add Destination NAT rule %s:%s -> %s:%s on %s",
-                                privateIp + ", and private port " + destPort);
+                            publicIp, srcPort, privateIp, destPort, _privateZone));
                     }
-                    return true;
+                } else {
+                    s_logger.debug(String.format("Destination NAT rule for %s:%s -> %s:%s on %s already exists",
+                        publicIp, srcPort, privateIp, destPort, _privateZone));
                 }
 
+                return true;
+
             case DELETE:
-                if (!manageDestinationNatRule(SrxCommand.CHECK_IF_EXISTS, publicIp, privateIp, srcPort, destPort)) {
+                // Delete public rule
-                    return true;
+                if (manageDestinationNatRule(SrxCommand.CHECK_IF_EXISTS, publicIp, privateIp, srcPort, destPort)) {
-                }
+                    xml = SrxXml.DEST_NAT_RULE_GETONE.getXml();
+                    xml = setDelete(xml, true);
+                    xml = replaceXmlValue(xml, "rule-set", _publicZone);
+                    xml = replaceXmlValue(xml, "from-zone", _publicZone);
+                    xml = replaceXmlValue(xml, "rule-name", ruleName);
 
-                xml = SrxXml.DEST_NAT_RULE_GETONE.getXml();
+                    if (!sendRequestAndCheckResponse(command, xml)) {
-                xml = setDelete(xml, true);
+                        throw new ExecutionException(String.format("Failed to delete destination NAT rule %s[%s] -> %s[%s] on rule %s",
-                xml = replaceXmlValue(xml, "rule-set", _publicZone);
+                            publicIp, srcPort, privateIp, destPort, _publicZone));
-                xml = replaceXmlValue(xml, "from-zone", _publicZone);
-                xml = replaceXmlValue(xml, "rule-name", ruleName);
-
-                if (!sendRequestAndCheckResponse(command, xml)) {
-                    throw new ExecutionException("Failed to delete destination NAT rule from public IP " + publicIp + ", public port " + srcPort + ", private IP " +
-                        privateIp + ", and private port " + destPort);
-                } else {
-                    if (manageDestinationNatRule(SrxCommand.CHECK_PRIVATE_IF_EXISTS, publicIp, privateIp, srcPort, destPort))
-                    {
-                        xml = SrxXml.DEST_NAT_RULE_GETONE.getXml();
-                        xml = setDelete(xml, true);
-                        xml = replaceXmlValue(xml, "rule-set", _privateZone);
-                        xml = replaceXmlValue(xml, "from-zone", _privateZone);
-                        xml = replaceXmlValue(xml, "rule-name", ruleName_private);
-
-                        if (!sendRequestAndCheckResponse(command, xml))
-                        {
-                            s_logger.debug("Purple: Failed to delete " + _privateZone + " destination NAT rule from public IP " + publicIp + ", public port " + srcPort + ", private IP " +
-                                    privateIp + ", and private port " + destPort);
-                        }
                     }
-
+                } else {
-                    return true;
+                    s_logger.debug(String.format("Destination NAT rule %s[%s] -> %s[%s] not found on %s, not deleting",
+                        publicIp, srcPort, privateIp, destPort, _publicZone));
                 }
 
+                // Delete private rule
+                if (manageDestinationNatRule(SrxCommand.CHECK_PRIVATE_IF_EXISTS, publicIp, privateIp, srcPort, destPort)) {
+                    xml = SrxXml.DEST_NAT_RULE_GETONE.getXml();
+                    xml = setDelete(xml, true);
+                    xml = replaceXmlValue(xml, "rule-set", _privateZone);
+                    xml = replaceXmlValue(xml, "from-zone", _privateZone);
+                    xml = replaceXmlValue(xml, "rule-name", ruleName_private);
+
+                    if (!sendRequestAndCheckResponse(command, xml))
+                    {
+                        throw new ExecutionException(String.format("Failed to delete destination NAT rule %s[%s] -> %s[%s] on rule %s",
+                            publicIp, srcPort, privateIp, destPort, _privateZone));
+                    }
+                } else {
+                    s_logger.debug(String.format("Destination NAT rule %s[%s] -> %s[%s] not found on %s, not deleting",
+                        publicIp, srcPort, privateIp, destPort, _privateZone));
+                }
+
+                return true;
+
             default:
                 s_logger.debug("Unrecognized command.");
                 return false;
@@ -2345,7 +2372,7 @@ public class JuniperSrxResource implements ServerResource {
                             NodeList destPortEntries = ruleMatchEntry.getChildNodes();
                             for (int destPortIndex = 0; destPortIndex < destPortEntries.getLength(); destPortIndex++) {
                                 Node destPortEntry = destPortEntries.item(destPortIndex);
-                                if (destPortEntry.getNodeName().equals("dst-port")) {
+                                if (destPortEntry.getNodeName().equals("dst-port") || destPortEntry.getNodeName().equals("name")) {
                                     ruleSrcPort = destPortEntry.getFirstChild().getNodeValue();
                                 }
                             }

pom.xml

@@ -909,6 +909,7 @@
                             <exclude>**/*.iso</exclude>
                             <exclude>**/*.tgz</exclude>
                             <exclude>**/*.zip</exclude>
+                            <exclude>**/*.vhd*</exclude>
                             <exclude>**/target/**</exclude>
                             <exclude>**/.vagrant</exclude>
                             <exclude>**/*.json</exclude>
@@ -934,15 +935,7 @@
                             <exclude>debian/source/format</exclude>
                             <exclude>dist/console-proxy/js/jquery.js</exclude>
                             <exclude>plugins/hypervisors/hyperv/conf/agent.properties</exclude>
-                            <exclude>plugins/hypervisors/hyperv/DotNet/ServerResource/ServerResource.sln</exclude>
+                            <exclude>plugins/hypervisors/hyperv/DotNet/ServerResource/**</exclude>
-                            <exclude>plugins/hypervisors/hyperv/DotNet/ServerResource/packages/**</exclude>
-                            <exclude>plugins/hypervisors/hyperv/DotNet/ServerResource/.nuget/**</exclude>
-                            <exclude>plugins/hypervisors/hyperv/DotNet/ServerResource/**/obj/**</exclude>
-                            <exclude>plugins/hypervisors/hyperv/DotNet/ServerResource/**/bin/**</exclude>
-                            <exclude>plugins/hypervisors/hyperv/DotNet/ServerResource/**/packages.config</exclude>
-                            <exclude>plugins/hypervisors/hyperv/DotNet/ServerResource/**/App.config</exclude>
-                            <exclude>plugins/hypervisors/hyperv/DotNet/ServerResource/**/*.csproj</exclude>
-                            <exclude>plugins/hypervisors/hyperv/DotNet/ServerResource/**/*.settings</exclude>
                             <exclude>scripts/installer/windows/acs_license.rtf</exclude>
                             <exclude>scripts/vm/systemvm/id_rsa.cloud</exclude>
                             <exclude>services/console-proxy/server/conf/agent.properties</exclude>

scripts/network/juniper/dest-nat-rule-add.xml

@@ -32,7 +32,7 @@ under the License.
 <dst-addr>%public-address%</dst-addr>
 </destination-address>
 <destination-port>
-<dst-port>%src-port%</dst-port>
+<low>%src-port%</low>
 </destination-port>
 </dest-nat-rule-match>
 <then>

server/src/main/java/com/cloud/api/ApiDBUtils.java

@@ -1803,12 +1803,8 @@ public class ApiDBUtils {
         return s_userAccountJoinDao.newUserView(usr);
     }
 
-    public static ProjectResponse newProjectResponse(ProjectJoinVO proj) {
+    public static ProjectResponse newProjectResponse(EnumSet<DomainDetails> details, ProjectJoinVO proj) {
-        return s_projectJoinDao.newProjectResponse(proj);
+        return s_projectJoinDao.newProjectResponse(details, proj);
-    }
-
-    public static ProjectResponse fillProjectDetails(ProjectResponse rsp, ProjectJoinVO proj) {
-        return s_projectJoinDao.setProjectResponse(rsp, proj);
     }
 
     public static List<ProjectJoinVO> newProjectView(Project proj) {
@@ -1912,8 +1908,8 @@ public class ApiDBUtils {
         return s_domainJoinDao.newDomainResponse(view, details, ve);
     }
 
-    public static AccountResponse newAccountResponse(ResponseView view, AccountJoinVO ve) {
+    public static AccountResponse newAccountResponse(ResponseView view, EnumSet<DomainDetails> details, AccountJoinVO ve) {
-        AccountResponse response = s_accountJoinDao.newAccountResponse(view, ve);
+        AccountResponse response = s_accountJoinDao.newAccountResponse(view, details, ve);
         // Populate account role information
         if (ve.getRoleId() != null) {
             Role role = s_roleService.findRole(ve.getRoleId());

server/src/main/java/com/cloud/api/ApiResponseHelper.java

@@ -35,6 +35,7 @@ import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.acl.ControlledEntity.ACLType;
 import org.apache.cloudstack.affinity.AffinityGroup;
 import org.apache.cloudstack.affinity.AffinityGroupResponse;
+import org.apache.cloudstack.api.ApiConstants.DomainDetails;
 import org.apache.cloudstack.api.ApiConstants.HostDetails;
 import org.apache.cloudstack.api.ApiConstants.VMDetails;
 import org.apache.cloudstack.api.ResponseGenerator;
@@ -382,13 +383,13 @@ public class ApiResponseHelper implements ResponseGenerator {
     // creates an account + user)
     @Override
     public AccountResponse createUserAccountResponse(ResponseView view, UserAccount user) {
-        return ApiDBUtils.newAccountResponse(view, ApiDBUtils.findAccountViewById(user.getAccountId()));
+        return ApiDBUtils.newAccountResponse(view, EnumSet.of(DomainDetails.all), ApiDBUtils.findAccountViewById(user.getAccountId()));
     }
 
     @Override
     public AccountResponse createAccountResponse(ResponseView view, Account account) {
         AccountJoinVO vUser = ApiDBUtils.newAccountView(account);
-        return ApiDBUtils.newAccountResponse(view, vUser);
+        return ApiDBUtils.newAccountResponse(view, EnumSet.of(DomainDetails.all), vUser);
     }
 
     @Override
@@ -2269,7 +2270,7 @@ public class ApiResponseHelper implements ResponseGenerator {
     @Override
     public ProjectResponse createProjectResponse(Project project) {
         List<ProjectJoinVO> viewPrjs = ApiDBUtils.newProjectView(project);
-        List<ProjectResponse> listPrjs = ViewResponseHelper.createProjectResponse(viewPrjs.toArray(new ProjectJoinVO[viewPrjs.size()]));
+        List<ProjectResponse> listPrjs = ViewResponseHelper.createProjectResponse(EnumSet.of(DomainDetails.all), viewPrjs.toArray(new ProjectJoinVO[viewPrjs.size()]));
         assert listPrjs != null && listPrjs.size() == 1 : "There should be one project  returned";
         return listPrjs.get(0);
     }

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -1277,7 +1277,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
     public ListResponse<ProjectResponse> listProjects(ListProjectsCmd cmd) {
         Pair<List<ProjectJoinVO>, Integer> projects = listProjectsInternal(cmd);
         ListResponse<ProjectResponse> response = new ListResponse<ProjectResponse>();
-        List<ProjectResponse> projectResponses = ViewResponseHelper.createProjectResponse(projects.first().toArray(new ProjectJoinVO[projects.first().size()]));
+        List<ProjectResponse> projectResponses = ViewResponseHelper.createProjectResponse(cmd.getDetails(), projects.first().toArray(new ProjectJoinVO[projects.first().size()]));
         response.setResponses(projectResponses, projects.second());
         return response;
     }
@@ -1937,7 +1937,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
             respView = ResponseView.Full;
         }
 
-        List<AccountResponse> accountResponses = ViewResponseHelper.createAccountResponse(respView, result.first().toArray(new AccountJoinVO[result.first().size()]));
+        List<AccountResponse> accountResponses = ViewResponseHelper.createAccountResponse(respView, cmd.getDetails(), result.first().toArray(new AccountJoinVO[result.first().size()]));
         response.setResponses(accountResponses, result.second());
         return response;
     }

server/src/main/java/com/cloud/api/query/ViewResponseHelper.java

@@ -195,19 +195,16 @@ public class ViewResponseHelper {
         return new ArrayList<SecurityGroupResponse>(vrDataList.values());
     }
 
-    public static List<ProjectResponse> createProjectResponse(ProjectJoinVO... projects) {
+    public static List<ProjectResponse> createProjectResponse(EnumSet<DomainDetails> details, ProjectJoinVO... projects) {
         Hashtable<Long, ProjectResponse> prjDataList = new Hashtable<Long, ProjectResponse>();
         // Initialise the prjdatalist with the input data
         for (ProjectJoinVO p : projects) {
             ProjectResponse pData = prjDataList.get(p.getId());
             if (pData == null) {
                 // first time encountering this vm
-                pData = ApiDBUtils.newProjectResponse(p);
+                pData = ApiDBUtils.newProjectResponse(details, p);
-            } else {
+                prjDataList.put(p.getId(), pData);
-                // update those  1 to many mapping fields
-                pData = ApiDBUtils.fillProjectDetails(pData, p);
             }
-            prjDataList.put(p.getId(), pData);
         }
         return new ArrayList<ProjectResponse>(prjDataList.values());
     }
@@ -541,10 +538,10 @@ public class ViewResponseHelper {
         }
     }
 
-    public static List<AccountResponse> createAccountResponse(ResponseView view, AccountJoinVO... accounts) {
+    public static List<AccountResponse> createAccountResponse(ResponseView view, EnumSet<DomainDetails> details, AccountJoinVO... accounts) {
         List<AccountResponse> respList = new ArrayList<AccountResponse>();
         for (AccountJoinVO vt : accounts){
-            respList.add(ApiDBUtils.newAccountResponse(view, vt));
+            respList.add(ApiDBUtils.newAccountResponse(view, details, vt));
         }
         return respList;
     }

server/src/main/java/com/cloud/api/query/dao/AccountJoinDao.java

@@ -16,6 +16,9 @@
 // under the License.
 package com.cloud.api.query.dao;
 
+import java.util.EnumSet;
+
+import org.apache.cloudstack.api.ApiConstants.DomainDetails;
 import org.apache.cloudstack.api.ResponseObject.ResponseView;
 import org.apache.cloudstack.api.response.AccountResponse;
 import org.apache.cloudstack.api.response.ResourceLimitAndCountResponse;
@@ -26,7 +29,7 @@ import com.cloud.utils.db.GenericDao;
 
 public interface AccountJoinDao extends GenericDao<AccountJoinVO, Long> {
 
-    AccountResponse newAccountResponse(ResponseView view, AccountJoinVO vol);
+    AccountResponse newAccountResponse(ResponseView view, EnumSet<DomainDetails> details, AccountJoinVO vol);
 
     AccountJoinVO newAccountView(Account vol);
 

server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.api.query.dao;
 
+import java.util.EnumSet;
 import java.util.List;
 
 import javax.inject.Inject;
@@ -23,6 +24,7 @@ import javax.inject.Inject;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
+import org.apache.cloudstack.api.ApiConstants.DomainDetails;
 import org.apache.cloudstack.api.ResponseObject.ResponseView;
 import org.apache.cloudstack.api.response.AccountResponse;
 import org.apache.cloudstack.api.response.ResourceLimitAndCountResponse;
@@ -57,7 +59,7 @@ public class AccountJoinDaoImpl extends GenericDaoBase<AccountJoinVO, Long> impl
     }
 
     @Override
-    public AccountResponse newAccountResponse(ResponseView view, AccountJoinVO account) {
+    public AccountResponse newAccountResponse(ResponseView view, EnumSet<DomainDetails> details, AccountJoinVO account) {
         AccountResponse accountResponse = new AccountResponse();
         accountResponse.setId(account.getUuid());
         accountResponse.setName(account.getAccountName());
@@ -76,17 +78,19 @@ public class AccountJoinDaoImpl extends GenericDaoBase<AccountJoinVO, Long> impl
         accountResponse.setBytesReceived(account.getBytesReceived());
         accountResponse.setBytesSent(account.getBytesSent());
 
-        boolean fullView = (view == ResponseView.Full && _acctMgr.isRootAdmin(account.getId()));
+        if (details.contains(DomainDetails.all) || details.contains(DomainDetails.resource)) {
-        setResourceLimits(account, fullView, accountResponse);
+            boolean fullView = (view == ResponseView.Full && _acctMgr.isRootAdmin(account.getId()));
+            setResourceLimits(account, fullView, accountResponse);
 
-        //get resource limits for projects
+            //get resource limits for projects
-        long projectLimit = ApiDBUtils.findCorrectResourceLimit(account.getProjectLimit(), account.getId(), ResourceType.project);
+            long projectLimit = ApiDBUtils.findCorrectResourceLimit(account.getProjectLimit(), account.getId(), ResourceType.project);
-        String projectLimitDisplay = (fullView || projectLimit == -1) ? "Unlimited" : String.valueOf(projectLimit);
+            String projectLimitDisplay = (fullView || projectLimit == -1) ? "Unlimited" : String.valueOf(projectLimit);
-        long projectTotal = (account.getProjectTotal() == null) ? 0 : account.getProjectTotal();
+            long projectTotal = (account.getProjectTotal() == null) ? 0 : account.getProjectTotal();
-        String projectAvail = (fullView || projectLimit == -1) ? "Unlimited" : String.valueOf(projectLimit - projectTotal);
+            String projectAvail = (fullView || projectLimit == -1) ? "Unlimited" : String.valueOf(projectLimit - projectTotal);
-        accountResponse.setProjectLimit(projectLimitDisplay);
+            accountResponse.setProjectLimit(projectLimitDisplay);
-        accountResponse.setProjectTotal(projectTotal);
+            accountResponse.setProjectTotal(projectTotal);
-        accountResponse.setProjectAvailable(projectAvail);
+            accountResponse.setProjectAvailable(projectAvail);
+        }
 
         // set async job
         if (account.getJobId() != null) {

server/src/main/java/com/cloud/api/query/dao/ProjectJoinDao.java

@@ -16,8 +16,10 @@
 // under the License.
 package com.cloud.api.query.dao;
 
+import java.util.EnumSet;
 import java.util.List;
 
+import org.apache.cloudstack.api.ApiConstants.DomainDetails;
 import org.apache.cloudstack.api.response.ProjectResponse;
 
 import com.cloud.api.query.vo.ProjectJoinVO;
@@ -26,9 +28,7 @@ import com.cloud.utils.db.GenericDao;
 
 public interface ProjectJoinDao extends GenericDao<ProjectJoinVO, Long> {
 
-    ProjectResponse newProjectResponse(ProjectJoinVO proj);
+    ProjectResponse newProjectResponse(EnumSet<DomainDetails> details, ProjectJoinVO proj);
-
-    ProjectResponse setProjectResponse(ProjectResponse rsp, ProjectJoinVO proj);
 
     List<ProjectJoinVO> newProjectView(Project proj);
 

server/src/main/java/com/cloud/api/query/dao/ProjectJoinDaoImpl.java

@@ -17,13 +17,16 @@
 package com.cloud.api.query.dao;
 
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.List;
 
+
 import javax.inject.Inject;
 
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
+import org.apache.cloudstack.api.ApiConstants.DomainDetails;
 import org.apache.cloudstack.api.response.ProjectResponse;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 
@@ -32,6 +35,7 @@ import com.cloud.api.query.vo.AccountJoinVO;
 import com.cloud.api.query.vo.ProjectJoinVO;
 import com.cloud.api.query.vo.ResourceTagJoinVO;
 import com.cloud.projects.Project;
+import com.cloud.server.ResourceTag.ResourceObjectType;
 import com.cloud.user.Account;
 import com.cloud.user.dao.AccountDao;
 import com.cloud.utils.db.GenericDaoBase;
@@ -67,7 +71,7 @@ public class ProjectJoinDaoImpl extends GenericDaoBase<ProjectJoinVO, Long> impl
     }
 
     @Override
-    public ProjectResponse newProjectResponse(ProjectJoinVO proj) {
+    public ProjectResponse newProjectResponse(EnumSet<DomainDetails> details, ProjectJoinVO proj) {
         ProjectResponse response = new ProjectResponse();
         response.setId(proj.getUuid());
         response.setName(proj.getName());
@@ -81,37 +85,23 @@ public class ProjectJoinDaoImpl extends GenericDaoBase<ProjectJoinVO, Long> impl
         response.setOwner(proj.getOwner());
 
         // update tag information
-        Long tag_id = proj.getTagId();
+        List<ResourceTagJoinVO> tags = ApiDBUtils.listResourceTagViewByResourceUUID(proj.getUuid(), ResourceObjectType.Project);
-        if (tag_id != null && tag_id.longValue() > 0) {
+        for (ResourceTagJoinVO vtag : tags) {
-            ResourceTagJoinVO vtag = ApiDBUtils.findResourceTagViewById(tag_id);
+            response.addTag(ApiDBUtils.newResourceTagResponse(vtag, false));
-            if (vtag != null) {
-                response.addTag(ApiDBUtils.newResourceTagResponse(vtag, false));
-            }
         }
 
         //set resource limit/count information for the project (by getting the info of the project's account)
         Account account = _accountDao.findByIdIncludingRemoved(proj.getProjectAccountId());
-        AccountJoinVO accountJn = ApiDBUtils.newAccountView(account);
+        if (details.contains(DomainDetails.all) || details.contains(DomainDetails.resource)) {
-        _accountJoinDao.setResourceLimits(accountJn, false, response);
+            AccountJoinVO accountJn = ApiDBUtils.newAccountView(account);
-        response.setProjectAccountName(accountJn.getAccountName());
+            _accountJoinDao.setResourceLimits(accountJn, false, response);
+        }
+        response.setProjectAccountName(account.getAccountName());
 
         response.setObjectName("project");
         return response;
     }
 
-    @Override
-    public ProjectResponse setProjectResponse(ProjectResponse rsp, ProjectJoinVO proj) {
-        // update tag information
-        Long tag_id = proj.getTagId();
-        if (tag_id != null && tag_id.longValue() > 0) {
-            ResourceTagJoinVO vtag = ApiDBUtils.findResourceTagViewById(tag_id);
-            if (vtag != null) {
-                rsp.addTag(ApiDBUtils.newResourceTagResponse(vtag, false));
-            }
-        }
-        return rsp;
-    }
-
     @Override
     public List<ProjectJoinVO> newProjectView(Project proj) {
         SearchCriteria<ProjectJoinVO> sc = prjIdSearch.create();

server/src/main/java/com/cloud/api/query/vo/ProjectJoinVO.java

@@ -29,7 +29,6 @@ import org.apache.cloudstack.api.Identity;
 import org.apache.cloudstack.api.InternalIdentity;
 
 import com.cloud.projects.Project.State;
-import com.cloud.server.ResourceTag.ResourceObjectType;
 import com.cloud.utils.db.GenericDao;
 
 @Entity
@@ -77,37 +76,6 @@ public class ProjectJoinVO extends BaseViewVO implements InternalIdentity, Ident
     @Column(name = "domain_path")
     private String domainPath;
 
-    @Column(name = "tag_id")
-    private long tagId;
-
-    @Column(name = "tag_uuid")
-    private String tagUuid;
-
-    @Column(name = "tag_key")
-    private String tagKey;
-
-    @Column(name = "tag_value")
-    private String tagValue;
-
-    @Column(name = "tag_domain_id")
-    private long tagDomainId;
-
-    @Column(name = "tag_account_id")
-    private long tagAccountId;
-
-    @Column(name = "tag_resource_id")
-    private long tagResourceId;
-
-    @Column(name = "tag_resource_uuid")
-    private String tagResourceUuid;
-
-    @Column(name = "tag_resource_type")
-    @Enumerated(value = EnumType.STRING)
-    private ResourceObjectType tagResourceType;
-
-    @Column(name = "tag_customer")
-    private String tagCustomer;
-
     @Column(name = "project_account_id")
     private long projectAccountId;
 
@@ -164,46 +132,6 @@ public class ProjectJoinVO extends BaseViewVO implements InternalIdentity, Ident
         return owner;
     }
 
-    public long getTagId() {
-        return tagId;
-    }
-
-    public String getTagUuid() {
-        return tagUuid;
-    }
-
-    public String getTagKey() {
-        return tagKey;
-    }
-
-    public String getTagValue() {
-        return tagValue;
-    }
-
-    public long getTagDomainId() {
-        return tagDomainId;
-    }
-
-    public long getTagAccountId() {
-        return tagAccountId;
-    }
-
-    public long getTagResourceId() {
-        return tagResourceId;
-    }
-
-    public String getTagResourceUuid() {
-        return tagResourceUuid;
-    }
-
-    public ResourceObjectType getTagResourceType() {
-        return tagResourceType;
-    }
-
-    public String getTagCustomer() {
-        return tagCustomer;
-    }
-
     public long getAccountId() {
         return accountId;
     }

server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java

@@ -376,8 +376,13 @@ public class NetworkHelperImpl implements NetworkHelper {
         final List<Long> networkIds = _routerDao.getRouterNetworks(router.getId());
 
         DomainRouterVO routerToBeAvoid = null;
+        List<DomainRouterVO> routerList = null;
         if (networkIds.size() != 0) {
-            final List<DomainRouterVO> routerList = _routerDao.findByNetwork(networkIds.get(0));
+            routerList = _routerDao.findByNetwork(networkIds.get(0));
+        } else if (router.getVpcId() != null) {
+            routerList = _routerDao.listByVpcId(router.getVpcId());
+        }
+        if (routerList != null) {
             for (final DomainRouterVO rrouter : routerList) {
                 if (rrouter.getHostId() != null && rrouter.getIsRedundantRouter() && rrouter.getState() == State.Running) {
                     if (routerToBeAvoid != null) {

server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java

@@ -545,7 +545,7 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
         }
         if (icmpCode != null) {
             if (icmpCode.longValue() != -1 && !NetUtils.validateIcmpCode(icmpCode.longValue())) {
-                throw new InvalidParameterValueException(String.format("Invalid icmp code [%d]. It should belong to [0-15] range and can be defined when icmpType belongs to [0-40] range", icmpCode));
+                throw new InvalidParameterValueException(String.format("Invalid icmp code [%d]. It should belong to [0-16] range and can be defined when icmpType belongs to [0-40] range", icmpCode));
             }
         }
     }

server/src/main/java/com/cloud/template/TemplateManagerImpl.java

@@ -1523,9 +1523,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
         }
 
         if (isExtractable != null) {
-            // Only Root admins allowed to change it for templates
+            // Only Root admins and owners are allowed to change it for templates
-            if (!template.getFormat().equals(ImageFormat.ISO) && !_accountMgr.isRootAdmin(caller.getId())) {
+            if (!template.getFormat().equals(ImageFormat.ISO) && caller.getId() != ownerId && !isAdmin) {
-                throw new InvalidParameterValueException("Only ROOT admins are allowed to modify isExtractable attribute.");
+                throw new InvalidParameterValueException("Only ROOT admins and template owners are allowed to modify isExtractable attribute.");
             } else {
                 // For Isos normal user can change it, as their are no derivatives.
                 updatedTemplate.setExtractable(isExtractable.booleanValue());

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -223,6 +223,7 @@ import com.cloud.org.Grouping;
 import com.cloud.resource.ResourceManager;
 import com.cloud.resource.ResourceState;
 import com.cloud.server.ManagementService;
+import com.cloud.server.ResourceTag;
 import com.cloud.service.ServiceOfferingVO;
 import com.cloud.service.dao.ServiceOfferingDao;
 import com.cloud.service.dao.ServiceOfferingDetailsDao;
@@ -251,6 +252,8 @@ import com.cloud.storage.dao.SnapshotDao;
 import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.storage.dao.VMTemplateZoneDao;
 import com.cloud.storage.dao.VolumeDao;
+import com.cloud.tags.ResourceTagVO;
+import com.cloud.tags.dao.ResourceTagDao;
 import com.cloud.template.TemplateApiService;
 import com.cloud.template.TemplateManager;
 import com.cloud.template.VirtualMachineTemplate;
@@ -477,6 +480,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
     private ConfigurationDao _configDao;
     @Inject
     private DpdkHelper dpdkHelper;
+    @Inject
+    private ResourceTagDao resourceTagDao;
 
     private ScheduledExecutorService _executor = null;
     private ScheduledExecutorService _vmIpFetchExecutor = null;
@@ -4958,6 +4963,18 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             addExtraConfig(vm, caller, extraConfig);
         }
 
+        if (cmd.getCopyImageTags()) {
+            VMTemplateVO templateOrIso = _templateDao.findById(templateId);
+            if (templateOrIso != null) {
+                final ResourceTag.ResourceObjectType templateType = (templateOrIso.getFormat() == ImageFormat.ISO) ? ResourceTag.ResourceObjectType.ISO : ResourceTag.ResourceObjectType.Template;
+                final List<? extends ResourceTag> resourceTags = resourceTagDao.listBy(templateId, templateType);
+                for (ResourceTag resourceTag : resourceTags) {
+                    final ResourceTagVO copyTag = new ResourceTagVO(resourceTag.getKey(), resourceTag.getValue(), resourceTag.getAccountId(), resourceTag.getDomainId(), vm.getId(), ResourceTag.ResourceObjectType.UserVm, resourceTag.getCustomer(), vm.getUuid());
+                    resourceTagDao.persist(copyTag);
+                }
+            }
+        }
+
         return vm;
     }
 

server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java

@@ -685,7 +685,7 @@ public class NetworkACLServiceImplTest {
     @Test(expected = InvalidParameterValueException.class)
     public void validateIcmpTypeAndCodeTestIcmpTypeValidAndIcmpCodeInvalid() {
         Mockito.when(networkAclItemVoMock.getIcmpType()).thenReturn(255);
-        Mockito.when(networkAclItemVoMock.getIcmpCode()).thenReturn(16);
+        Mockito.when(networkAclItemVoMock.getIcmpCode()).thenReturn(17);
 
         networkAclServiceImpl.validateIcmpTypeAndCode(networkAclItemVoMock);
     }

systemvm/debian/opt/cloud/bin/checkrouter.sh

@@ -27,10 +27,10 @@ fi
 ROUTER_TYPE=$(cat /etc/cloudstack/cmdline.json | grep type | awk '{print $2;}' | sed -e 's/[,\"]//g')
 if [ "$ROUTER_TYPE" = "router" ]
 then
-	ROUTER_STATE=$(ip addr | grep eth0 | grep inet | wc -l | xargs bash -c  'if [ $0 == 2 ]; then echo "MASTER"; else echo "BACKUP"; fi')
+	ROUTER_STATE=$(ip addr show dev eth0 | grep inet | wc -l | xargs bash -c  'if [ $0 == 2 ]; then echo "MASTER"; else echo "BACKUP"; fi')
 	STATUS=$ROUTER_STATE
 else
-	ROUTER_STATE=$(ip addr | grep eth1 | grep state | awk '{print $9;}')
+	ROUTER_STATE=$(ip addr show dev eth1 | grep state | awk '{print $9;}')
 	if [ "$ROUTER_STATE" = "UP" ]
 	then
 	    STATUS=MASTER

systemvm/debian/opt/cloud/bin/configure.py

@@ -21,6 +21,8 @@ import logging
 import os
 import re
 import sys
+import urllib
+import urllib2
 import time
 
 from collections import OrderedDict
@@ -62,10 +64,15 @@ class CsPassword(CsDataBag):
             server_ip = ip.split('/')[0]
             proc = CsProcess(['/opt/cloud/bin/passwd_server_ip.py', server_ip])
             if proc.find():
-                update_command = 'curl --header "DomU_Request: save_password" "http://{SERVER_IP}:8080/" -F "ip={VM_IP}" -F "password={PASSWORD}" ' \
+                url = "http://%s:8080/" % server_ip
-                                 '-F "token={TOKEN}" >/dev/null 2>/dev/null &'.format(SERVER_IP=server_ip, VM_IP=vm_ip, PASSWORD=password, TOKEN=token)
+                payload = {"ip": vm_ip, "password": password, "token": token}
-                result = CsHelper.execute(update_command)
+                data = urllib.urlencode(payload)
-                logging.debug("Update password server result ==> %s" % result)
+                request = urllib2.Request(url, data=data, headers={"DomU_Request": "save_password"})
+                try:
+                    resp = urllib2.urlopen(request, data)
+                    logging.debug("Update password server result: http:%s, content:%s" % (resp.code, resp.read()))
+                except Exception as e:
+                    logging.error("Failed to update password server due to: %s" % e)
 
 
 class CsAcl(CsDataBag):

test/integration/smoke/test_create_list_domain_account_project.py

@@ -0,0 +1,205 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+""" tests for create/list domain,account,project
+"""
+# Import Local Modules
+from nose.plugins.attrib import attr
+from marvin.cloudstackTestCase import cloudstackTestCase
+from marvin.lib.utils import cleanup_resources
+from marvin.lib.base import (Account,
+                             Project,
+                             Domain)
+from marvin.lib.common import get_domain
+
+class Services:
+
+    """Test Project Services
+    """
+
+    def __init__(self):
+        self.services = {
+            "domain": {
+                "name": "Test Domain",
+            },
+            "project": {
+                "name": "Test Project",
+                "displaytext": "Test project",
+            },
+            "account": {
+                "email": "administrator@cloudstack.apache.org",
+                "firstname": "Test",
+                "lastname": "User",
+                "username": "test",
+                "password": "password",
+            },
+        }
+
+
+class TestDomainAccountProject(cloudstackTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.testClient = super(
+            TestDomainAccountProject,
+            cls).getClsTestClient()
+        cls.api_client = cls.testClient.getApiClient()
+
+        cls.services = Services().services
+        cls.domain = get_domain(cls.api_client)
+        cls._cleanup = []
+        return
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            # Cleanup resources used
+            cleanup_resources(cls.api_client, cls._cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def setUp(self):
+        self.apiclient = self.testClient.getApiClient()
+        self.cleanup = []
+        return
+
+    def tearDown(self):
+        try:
+            # Clean up, terminate the created accounts, domains etc
+            cleanup_resources(self.apiclient, self.cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def test_01_create_list_domain_account_project(self):
+        """ Verify list domain, account and project return expected response
+        """
+        # Validate the following
+        # 1. Create domain
+        # 2. list domain, 'cpulimit' should be included in response
+        # 3. list domain with details=min, 'cpulimit' should not be included in response.
+
+        # 4. create account in the domain
+        # 5. list account, 'cpulimit' should be included in response
+        # 6. list account with details=min, 'cpulimit' should not be included in response.
+
+        # 7. create project in the domain
+        # 8. list project, 'cpulimit' should be included in response
+        # 9. list project with details=min, 'cpulimit' should not be included in response.
+
+        # Create new domain
+        self.user_domain = Domain.create(
+            self.apiclient,
+            self.services["domain"],
+            parentdomainid=self.domain.id)
+
+        list_domain_response = Domain.list(
+            self.apiclient,
+            id = self.user_domain.id)
+
+        self.assertEqual(
+            isinstance(list_domain_response, list),
+            True,
+            "Check list response returns a valid list"
+        )
+
+        self.assertIsNotNone(list_domain_response[0].cpulimit, "'cpulimit' should be included in response")
+
+        list_domain_response = Domain.list(
+            self.apiclient,
+            details="min",
+            id = self.user_domain.id)
+
+        self.assertEqual(
+            isinstance(list_domain_response, list),
+            True,
+            "Check list response returns a valid list"
+        )
+
+        self.assertIsNone(list_domain_response[0].cpulimit, "'cpulimit' should not be included in response")
+
+        # Create account        
+        self.account = Account.create(
+            self.apiclient,
+            self.services["account"],
+            admin=True,
+            domainid=self.user_domain.id
+        )
+
+        list_account_response = Account.list(
+            self.apiclient,
+            id = self.account.id)
+
+        self.assertEqual(
+            isinstance(list_account_response, list),
+            True,
+            "Check list response returns a valid list"
+        )
+
+        self.assertIsNotNone(list_account_response[0].cpulimit, "'cpulimit' should be included in response")
+
+        list_account_response = Account.list(
+            self.apiclient,
+            details="min",
+            id = self.account.id)
+
+        self.assertEqual(
+            isinstance(list_account_response, list),
+            True,
+            "Check list response returns a valid list"
+        )
+
+        self.assertIsNone(list_account_response[0].cpulimit, "'cpulimit' should not be included in response")
+
+        # Create project
+        self.project = Project.create(
+            self.apiclient,
+            self.services["project"],
+            account=self.account.name,
+            domainid=self.account.domainid
+        )
+
+        list_project_response = Project.list(
+            self.apiclient,
+            listall="true",
+            id = self.project.id)
+
+        self.assertEqual(
+            isinstance(list_project_response, list),
+            True,
+            "Check list response returns a valid list"
+        )
+
+        self.assertIsNotNone(list_project_response[0].cpulimit, "'cpulimit' should be included in response")
+
+        list_project_response = Project.list(
+            self.apiclient,
+            details="min",
+            listall="true",
+            id = self.project.id)
+
+        self.assertEqual(
+            isinstance(list_project_response, list),
+            True,
+            "Check list response returns a valid list"
+        )
+
+        self.assertIsNone(list_project_response[0].cpulimit, "'cpulimit' should not be included in response")
+
+        self.cleanup.append(self.project)
+        self.cleanup.append(self.account)
+        self.cleanup.append(self.user_domain)

tools/travis/install.sh

@@ -25,14 +25,15 @@ export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/
 set -e
 
 if [ $TEST_SEQUENCE_NUMBER -eq 1 ]; then
+   DIR=$(pwd)
    # Pylint/pep8 systemvm python codebase
-   cd systemvm/test && bash -x runtests.sh && cd ../..
+   cd systemvm/test && bash -x runtests.sh
    # Build noredist
-   git clone https://github.com/rhtyd/cloudstack-nonoss.git nonoss
+   git clone https://github.com/rhtyd/cloudstack-nonoss.git nonoss && cd nonoss && bash -x install-non-oss.sh
-   cd nonoss && bash -x install-non-oss.sh && cd ..
+   cd $DIR && echo $DIR
    git clean -fdx .
    # Perform rat checks
-   mvn -P developer,systemvm -Dsimulator -Dnoredist --projects='org.apache.cloudstack:cloudstack' org.apache.rat:apache-rat-plugin:0.12:check
+   mvn -P developer,systemvm -Dsimulator -Dnoredist -pl . org.apache.rat:apache-rat-plugin:0.12:check
    mvn -q -B -P developer,systemvm -Dsimulator -Dnoredist clean install
 else
    mvn -Pdeveloper -Dsimulator clean install -DskipTests -T4 | egrep "Building|Tests|SUCCESS|FAILURE"

ui/css/cloudstack3.css

@@ -8154,7 +8154,6 @@ div.container div.panel div#details-tab-addloadBalancer.detail-group div.loadBal
   display: block;
   float: left;
   max-width: 90%;
-  white-space: nowrap;
   text-overflow: ellipsis;
   overflow: hidden;
 }

ui/l10n/en.js

@@ -993,6 +993,7 @@ var dictionary = {
 "label.lb.protocol.http":"HTTP",
 "label.lb.protocol.ssl":"SSL",
 "label.lb.protocol.tcp":"TCP",
+"label.lb.protocol.tcp.proxy":"TCP-proxy",
 "label.lb.protocol.udp":"UDP",
 "label.ldap.configuration":"LDAP Configuration",
 "label.ldap.group.name":"LDAP Group",

ui/scripts/accounts.js

@@ -149,6 +149,10 @@
                             }
                         }
 
+                        $.extend(data, {
+                            details: 'min'
+                        });
+
                         $.ajax({
                             url: createURL('listAccounts'),
                             data: data,

ui/scripts/instances.js

@@ -750,9 +750,13 @@
                 if ("sshkeypairs" in args.context) {
                     $.extend(data, {
                         domainid: args.context.sshkeypairs[0].domainid,
-                        account: args.context.sshkeypairs[0].account,
                         keypair: args.context.sshkeypairs[0].name
                     });
+                    if (!cloudStack.context || !cloudStack.context.projects) {
+                        // In case we are in project mode sshkeypairs provides project account name which
+                        // should not be passed as part of API params. So only extend if NOT in project mode.
+                        $.extend(data, { account: args.context.sshkeypairs[0].account});
+                    }
                 }
 
                 $.ajax({
@@ -1473,6 +1477,11 @@
                                     displayName: args.data.displayname
                                 });
                             }
+                            if (args.data.name != args.context.instances[0].name) {
+                                $.extend(data, {
+                                    name: args.data.name
+                                });
+                            }
                             $.ajax({
                                 url: createURL('updateVirtualMachine'),
                                 data: data,
@@ -2378,6 +2387,7 @@
                                         };
                                         $.ajax({
                                             url: createURL('listAccounts', {
+                                                details: 'min',
                                                 ignoreProject: true
                                             }),
                                             data: dataObj,
@@ -2411,6 +2421,7 @@
                                         var dataObj = {
                                             domainId: args.domainid,
                                             state: 'Active',
+                                            details: 'min',
                                             listAll: true,
                                         };
                                         $.ajax({
@@ -2794,7 +2805,8 @@
                                 converter: cloudStack.converters.toLocalDate
                             },
                             name: {
-                                label: 'label.name'
+                                label: 'label.name',
+                                isEditable: true
                             },
                             id: {
                                 label: 'label.id'

ui/scripts/lbCertificatePolicy.js

@@ -36,6 +36,10 @@
                                 }
                             }
 
+                            $.extend(data, {
+                                details: 'min'
+                            });
+
                             $.ajax({
                                 url: createURL('listAccounts'),
                                 async: false,
@@ -187,4 +191,4 @@
             }
         }
     };
-}(jQuery, cloudStack));
+}(jQuery, cloudStack));

ui/scripts/network.js

@@ -3398,6 +3398,10 @@
                                                             id: 'tcp',
                                                             name: 'tcp',
                                                             description: _l('label.lb.protocol.tcp')
+                                                        }, {
+                                                            id: 'tcp-proxy',
+                                                            name: 'tcp-proxy',
+                                                            description: _l('label.lb.protocol.tcp.proxy')
                                                         }, {
                                                             id: 'udp',
                                                             name: 'udp',

ui/scripts/projects.js

@@ -635,6 +635,7 @@
             var page = 1;
             var getNextPage = function() {
                 var data2 = $.extend({}, data1, {
+                    details: 'min',
                     page: page,
                     pageSize: 500
                 });
@@ -777,6 +778,10 @@
                             });
                         }
 
+                        $.extend(data, {
+                            details: 'min',
+                        });
+
                         $.ajax({
                             url: createURL('listProjects', {
                                 ignoreProject: true

ui/scripts/sharedFunctions.js

@@ -512,7 +512,7 @@ var addGuestNetworkDialog = {
                     select: function(args) {
                         var items = [];
                         $.ajax({
-                            url: createURL("listProjects&listAll=true"),
+                            url: createURL("listProjects&listAll=true&details=min"),
                             dataType: "json",
                             async: false,
                             success: function(json) {
@@ -2429,7 +2429,7 @@ cloudStack.api = {
                     url: createURL(updateCommand),
                     data: {
                         id: args.context[objType].id,
-                        sortKey: g_sortKeyIsAscending ? (-1 * args.index) : args.index
+                        sortKey: args.sortKey
                     },
                     success: function(json) {
                         args.response.success();
@@ -2567,6 +2567,39 @@ function strOrFunc(arg, args) {
     return arg;
 }
 
+function sortArrayByKey(arrayToSort, sortKey, reverse) {
+
+    if(!arrayToSort){
+        return;
+    }
+    // Move smaller items towards the front
+    // or back of the array depending on if
+    // we want to sort the array in reverse
+    // order or not.
+    var moveSmaller = reverse ? 1 : -1;
+
+    // Move larger items towards the front
+    // or back of the array depending on if
+    // we want to sort the array in reverse
+    // order or not.
+    var moveLarger = reverse ? -1 : 1;
+
+    /**
+     * @param  {*} a
+     * @param  {*} b
+     * @return {Number}
+     */
+    arrayToSort.sort(function(a, b) {
+        if (a[sortKey] < b[sortKey]) {
+            return moveSmaller;
+        }
+        if (a[sortKey] > b[sortKey]) {
+            return moveLarger;
+        }
+        return 0;
+    });
+}
+
 $.validator.addMethod("netmask", function(value, element) {
     if (this.optional(element) && value.length == 0)
         return true;

ui/scripts/storage.js

@@ -530,29 +530,7 @@
                                                 if (this.iscustomized == true) {
                                                     items.push({
                                                         id: this.id,
-                                                        description: this.displaytext
+                                                        description: this.name
-                                                    });
-                                                }
-                                            });
-                                            args.response.success({
-                                                data: items
-                                            });
-                                        }
-                                    },
-                                    diskOffering: {
-                                        label: 'label.custom.disk.offering',
-                                        docID: 'helpVolumeDiskOffering',
-                                        select: function(args) {
-                                            var diskOfferings = cloudStack.listDiskOfferings({});
-                                            var items = [{
-                                                id: '',
-                                                description: ''
-                                            }];
-                                            $(diskOfferings).each(function() {
-                                                if (this.iscustomized == true) {
-                                                    items.push({
-                                                        id: this.id,
-                                                        description: this.displaytext
                                                     });
                                                 }
                                             });

ui/scripts/system.js

@@ -2044,6 +2044,7 @@
                                                             $.ajax({
                                                                 url: createURL('listAccounts&domainid=' + args.domainId),
                                                                 data: {
+                                                                    details: 'min',
                                                                     listAll: true
                                                                 },
                                                                 success: function (json) {
@@ -2068,7 +2069,7 @@
                                                         select: function(args) {
                                                             var items = [];
                                                             $.ajax({
-                                                                url: createURL("listProjects&domainid=" + args.domainId),
+                                                                url: createURL("listProjects&details=min&domainid=" + args.domainId),
                                                                 dataType: "json",
                                                                 async: false,
                                                                 success: function(json) {
@@ -11027,7 +11028,7 @@
                                     }
                                 }
                                 $.ajax({
-                                    url: createURL("listAccounts&listAll=true&page=" + args.page + "&pagesize=" + pageSize + array1.join("")),
+                                    url: createURL("listAccounts&listAll=true&details=min&page=" + args.page + "&pagesize=" + pageSize + array1.join("")),
                                     success: function (json) {
                                         var accountObjs = json.listaccountsresponse.account;
                                         if (accountObjs != null) {

ui/scripts/templates.js

@@ -1595,12 +1595,7 @@
                                     isextractable: {
                                         label: 'label.extractable.lower',
                                         isBoolean: true,
-                                        isEditable: function() {
+                                        isEditable: true,
-                                            if (isAdmin())
-                                                return true;
-                                            else
-                                                return false;
-                                        },
                                         converter: cloudStack.converters.toBooleanText
                                     },
                                     passwordenabled: {
@@ -2039,12 +2034,7 @@
                                                 isextractable: {
                                                     label: 'label.extractable.lower',
                                                     isBoolean: true,
-                                                    isEditable: function() {
+                                                    isEditable: true,
-                                                        if (isAdmin())
-                                                            return true;
-                                                        else
-                                                            return false;
-                                                    },
                                                     converter: cloudStack.converters.toBooleanText
                                                 },
                                                 passwordenabled: {
@@ -3327,12 +3317,7 @@
                                                 isextractable: {
                                                     label: 'label.extractable.lower',
                                                     isBoolean: true,
-                                                    isEditable: function() {
+                                                    isEditable: true,
-                                                        if (isAdmin())
-                                                            return true;
-                                                        else
-                                                            return false;
-                                                    },
                                                     converter: cloudStack.converters.toBooleanText
                                                 },
                                                 bootable: {
@@ -3449,7 +3434,9 @@
             || (jsonObj.isready == false) || jsonObj.templatetype == "SYSTEM") {
             //do nothing
         } else {
-            allowedActions.push("downloadTemplate");
+            if (jsonObj.isextractable){
+                allowedActions.push("downloadTemplate");
+            }
         }
 
         // "Delete Template"

ui/scripts/ui-custom/instanceWizard.js

@@ -809,6 +809,9 @@
                                     $step.find('.main-desc, p.no-affinity-groups').remove();
 
                                     if (args.data.affinityGroups && args.data.affinityGroups.length) {
+
+                                        sortArrayByKey(args.data.affinityGroups, 'name');
+
                                         $step.prepend(
                                             $('<div>').addClass('main-desc').append(
                                                 $('<p>').html(_l('message.select.affinity.groups'))
@@ -855,6 +858,9 @@
                                     $step.find('.main-desc, p.no-sshkey-pairs').remove();
 
                                     if (args.data.sshkeyPairs && args.data.sshkeyPairs.length) {
+
+                                        sortArrayByKey(args.data.sshkeyPairs, 'name');
+
                                         $step.prepend(
                                             $('<div>').addClass('main-desc').append(
                                                 $('<p>').html(_l('message.please.select.ssh.key.pair.use.with.this.vm'))
@@ -1024,6 +1030,9 @@
 
                                     // Populate VPC drop-down
                                     $vpcSelect.html('');
+
+                                    sortArrayByKey(vpcs, 'name');
+
                                     $(vpcs).map(function(index, vpc) {
                                         var $option = $('<option>');
                                         var id = vpc.id;

ui/scripts/ui/widgets/listView.js

@@ -1335,13 +1335,18 @@
                         true, {},
                         $tr.closest('.list-view').data('view-args').context
                     );
-                    var rowIndex = $tr.closest('tbody').find('tr').length - ($tr.index());
+                    var sortKey;
+                    if (g_sortKeyIsAscending) {
+                        sortKey = $tr.index() + 1;
+                    } else {
+                        sortKey = ($tr.closest('tbody').find('tr').length - ($tr.index()));
+                    }
 
                     context[viewArgs.activeSection] = $tr.data('json-obj');
 
                     action.action({
                         context: context,
-                        index: rowIndex,
+                        sortKey: sortKey,
                         response: {
                             success: function(args) {},
                             error: function(args) {

utils/src/main/java/com/cloud/utils/net/NetUtils.java

@@ -34,6 +34,7 @@ import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Formatter;
 import java.util.List;
+import java.util.Collections;
 import java.util.Random;
 import java.util.Set;
 import java.util.SortedSet;
@@ -410,10 +411,11 @@ public class NetUtils {
     }
 
     public static String[] getNetworkParams(final NetworkInterface nic) {
-        final List<InterfaceAddress> addrs = nic.getInterfaceAddresses();
+        List<InterfaceAddress> addrs = nic.getInterfaceAddresses();
         if (addrs == null || addrs.size() == 0) {
             return null;
         }
+        Collections.reverse(addrs); // reverse addresses because it has reverse order as "ip addr show"
         InterfaceAddress addr = null;
         for (final InterfaceAddress iaddr : addrs) {
             final InetAddress inet = iaddr.getAddress();
@@ -1213,9 +1215,9 @@ public class NetUtils {
 
     public static boolean validateIcmpCode(final long icmpCode) {
 
-        //Source - http://www.erg.abdn.ac.uk/~gorry/course/inet-pages/icmp-code.html
+        // Reference: https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xhtml#icmp-parameters-codes-9/#table-icmp-parameters-ext-classes
-        if (!(icmpCode >= 0 && icmpCode <= 15)) {
+        if (!(icmpCode >= 0 && icmpCode <= 16)) {
-            s_logger.warn("Icmp code should be within 0-15 range");
+            s_logger.warn("Icmp code should be within 0-16 range");
             return false;
         }