From bab6839f38262a228da6c9a193972cf6def2616a Mon Sep 17 00:00:00 2001 From: Min Chen Date: Mon, 11 Mar 2013 11:48:23 -0700 Subject: [PATCH 01/34] CLOUDSTACK-1621: listProjectInvitations fails with NPE for valid requests. --- .../com/cloud/api/query/dao/ProjectInvitationJoinDaoImpl.java | 4 +++- .../src/com/cloud/api/query/vo/ProjectInvitationJoinVO.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/com/cloud/api/query/dao/ProjectInvitationJoinDaoImpl.java b/server/src/com/cloud/api/query/dao/ProjectInvitationJoinDaoImpl.java index ca0f171dd1a..1dfed3421bb 100644 --- a/server/src/com/cloud/api/query/dao/ProjectInvitationJoinDaoImpl.java +++ b/server/src/com/cloud/api/query/dao/ProjectInvitationJoinDaoImpl.java @@ -56,7 +56,9 @@ public class ProjectInvitationJoinDaoImpl extends GenericDaoBase Date: Fri, 8 Mar 2013 17:40:02 -0800 Subject: [PATCH 02/34] CLOUDSTACK-1615: instead of returning empty host firewall ruleset, ESXi 5.0 Update 1 may return null to indicate empty host firewall ruleset, Cloudstack needs to handle this changed behave --- .../com/cloud/hypervisor/vmware/resource/VmwareResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 155d6d9718f..634827b6468 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -5034,7 +5034,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa boolean bRefresh = false; if(firewallMo != null) { HostFirewallInfo firewallInfo = firewallMo.getFirewallInfo(); - if(firewallInfo != null) { + if(firewallInfo != null && firewallInfo.getRuleset() != null) { for(HostFirewallRuleset rule : firewallInfo.getRuleset()) { if("vncServer".equalsIgnoreCase(rule.getKey())) { bRefresh = true; From 797eb009e239d904c52de8c5be4cb0c82a63fea4 Mon Sep 17 00:00:00 2001 From: David Nalley Date: Mon, 11 Mar 2013 20:08:00 -0400 Subject: [PATCH 03/34] CLOUDSTACK-1630 - awsapi package not obsoleted, which blocked upgrades --- packaging/centos63/cloud.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index e340dd08c60..3af01b2f33f 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -146,6 +146,8 @@ Apache CloudStack command line interface Summary: Apache CloudStack AWS API compatibility wrapper Requires: %{name}-management = %{_ver} %description awsapi +Obsoletes: cloud-aws-api < 4.1.0 +Provides: cloud-aws-api Apache Cloudstack AWS API compatibility wrapper %package docs From 7a0f1bdf813ddb3a61a57d27b98f1ff7753d1be4 Mon Sep 17 00:00:00 2001 From: David Nalley Date: Mon, 11 Mar 2013 20:18:43 -0400 Subject: [PATCH 04/34] CLOUDSTACK-1631 - removing duplicate %doc declarations in -management --- packaging/centos63/cloud.spec | 2 -- 1 file changed, 2 deletions(-) diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index 3af01b2f33f..a79ef175948 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -338,8 +338,6 @@ fi #No default permission as the permission setup is complex %files management %defattr(-,root,root,-) -%doc LICENSE -%doc NOTICE %dir %attr(0770,root,cloud) %{_sysconfdir}/%{name}/management/Catalina %dir %attr(0770,root,cloud) %{_sysconfdir}/%{name}/management/Catalina/localhost %dir %attr(0770,root,cloud) %{_sysconfdir}/%{name}/management/Catalina/localhost/client From 1af8710da858d86c3117cef47e69bdc0b8f0d78d Mon Sep 17 00:00:00 2001 From: Min Chen Date: Mon, 11 Mar 2013 16:55:57 -0700 Subject: [PATCH 05/34] updateTemplatePermissions should allow user to be able to see templates granted permissions by another user. --- .../template/VirtualMachineTemplate.java | 1 + .../cloud/storage/dao/VMTemplateDaoImpl.java | 161 +++++++++--------- 2 files changed, 82 insertions(+), 80 deletions(-) diff --git a/api/src/com/cloud/template/VirtualMachineTemplate.java b/api/src/com/cloud/template/VirtualMachineTemplate.java index cdfe8d38dc5..cedc793c197 100755 --- a/api/src/com/cloud/template/VirtualMachineTemplate.java +++ b/api/src/com/cloud/template/VirtualMachineTemplate.java @@ -37,6 +37,7 @@ public interface VirtualMachineTemplate extends ControlledEntity, Identity, Inte featured, // returns templates that have been marked as featured and public self, // returns templates that have been registered or created by the calling user selfexecutable, // same as self, but only returns templates that are ready to be deployed with + shared, // including templates that have been granted to the calling user by another user sharedexecutable, // ready templates that have been granted to the calling user by another user executable, // templates that are owned by the calling user, or public templates, that can be used to deploy a community, // returns templates that have been marked as public but not featured diff --git a/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java b/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java index c4928be8fee..b51b216717b 100755 --- a/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -78,12 +78,12 @@ import com.cloud.utils.exception.CloudRuntimeException; @Local(value={VMTemplateDao.class}) public class VMTemplateDaoImpl extends GenericDaoBase implements VMTemplateDao { private static final Logger s_logger = Logger.getLogger(VMTemplateDaoImpl.class); - + @Inject VMTemplateZoneDao _templateZoneDao; @Inject VMTemplateDetailsDao _templateDetailsDao; - + @Inject ConfigurationDao _configDao; @Inject @@ -94,10 +94,10 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem DataCenterDao _dcDao; private final String SELECT_TEMPLATE_HOST_REF = "SELECT t.id, h.data_center_id, t.unique_name, t.name, t.public, t.featured, t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, " + "t.checksum, t.display_text, t.enable_password, t.guest_os_id, t.bootable, t.prepopulate, t.cross_zones, t.hypervisor_type FROM vm_template t"; - + private final String SELECT_TEMPLATE_ZONE_REF = "SELECT t.id, tzr.zone_id, t.unique_name, t.name, t.public, t.featured, t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, " + "t.checksum, t.display_text, t.enable_password, t.guest_os_id, t.bootable, t.prepopulate, t.cross_zones, t.hypervisor_type FROM vm_template t INNER JOIN template_zone_ref tzr on (t.id = tzr.template_id) "; - + private final String SELECT_TEMPLATE_SWIFT_REF = "SELECT t.id, t.unique_name, t.name, t.public, t.featured, t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, " + "t.checksum, t.display_text, t.enable_password, t.guest_os_id, t.bootable, t.prepopulate, t.cross_zones, t.hypervisor_type FROM vm_template t"; @@ -132,17 +132,17 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem private String routerTmpltName; private String consoleProxyTmpltName; - + public VMTemplateDaoImpl() { } - + @Override public List listByPublic() { SearchCriteria sc = PublicSearch.create(); sc.setParameters("public", 1); return listBy(sc); } - + @Override public VMTemplateVO findByName(String templateName) { SearchCriteria sc = UniqueNameSearch.create(); @@ -159,7 +159,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem @Override public List publicIsoSearch(Boolean bootable, boolean listRemoved, Map tags){ - + SearchBuilder sb = null; if (tags == null || tags.isEmpty()) { sb = PublicIsoSearch; @@ -170,7 +170,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem sb.and("type", sb.entity().getTemplateType(), SearchCriteria.Op.EQ); sb.and("bootable", sb.entity().isBootable(), SearchCriteria.Op.EQ); sb.and("removed", sb.entity().getRemoved(), SearchCriteria.Op.EQ); - + SearchBuilder tagSearch = _tagsDao.createSearchBuilder(); for (int count=0; count < tags.size(); count++) { tagSearch.or().op("key" + String.valueOf(count), tagSearch.entity().getKey(), SearchCriteria.Op.EQ); @@ -181,20 +181,20 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem sb.groupBy(sb.entity().getId()); sb.join("tagSearch", tagSearch, sb.entity().getId(), tagSearch.entity().getResourceId(), JoinBuilder.JoinType.INNER); } - + SearchCriteria sc = sb.create(); - + sc.setParameters("public", 1); sc.setParameters("format", "ISO"); sc.setParameters("type", TemplateType.PERHOST.toString()); if (bootable != null) { sc.setParameters("bootable", bootable); } - + if (!listRemoved) { sc.setParameters("removed", (Object)null); } - + if (tags != null && !tags.isEmpty()) { int count = 0; sc.setJoinParameters("tagSearch", "resourceType", TaggedResourceType.ISO.toString()); @@ -204,10 +204,10 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem count++; } } - + return listBy(sc); } - + @Override public List userIsoSearch(boolean listRemoved){ @@ -228,7 +228,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem public List listAllSystemVMTemplates() { SearchCriteria sc = tmpltTypeSearch.create(); sc.setParameters("templateType", Storage.TemplateType.SYSTEM); - + Filter filter = new Filter(VMTemplateVO.class, "id", false, null, null); return listBy(sc, filter); } @@ -236,7 +236,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem @Override public List listPrivateTemplatesByHost(Long hostId) { - String sql = "select * from template_host_ref as thr INNER JOIN vm_template as t ON t.id=thr.template_id " + String sql = "select * from template_host_ref as thr INNER JOIN vm_template as t ON t.id=thr.template_id " + "where thr.host_id=? and t.public=0 and t.featured=0 and t.type='USER' and t.removed is NULL"; List l = new ArrayList(); @@ -256,7 +256,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem } return l; } - + @Override public List listReadyTemplates() { SearchCriteria sc = createSearchCriteria(); @@ -264,7 +264,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem sc.addAnd("format", SearchCriteria.Op.NEQ, Storage.ImageFormat.ISO); return listIncludingRemovedBy(sc); } - + @Override public List findIsosByIdAndPath(Long domainId, Long accountId, String path) { SearchCriteria sc = createSearchCriteria(); @@ -287,7 +287,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem sc.setParameters("accountId", accountId); return listBy(sc); } - + @Override public List listByHypervisorType(List hyperTypes) { SearchCriteria sc = createSearchCriteria(); @@ -299,17 +299,17 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem @Override public boolean configure(String name, Map params) throws ConfigurationException { boolean result = super.configure(name, params); - + PublicSearch = createSearchBuilder(); PublicSearch.and("public", PublicSearch.entity().isPublicTemplate(), SearchCriteria.Op.EQ); routerTmpltName = (String)params.get("routing.uniquename"); - + s_logger.debug("Found parameter routing unique name " + routerTmpltName); if (routerTmpltName==null) { routerTmpltName="routing"; } - + consoleProxyTmpltName = (String)params.get("consoleproxy.uniquename"); if(consoleProxyTmpltName == null) { consoleProxyTmpltName = "routing"; @@ -345,16 +345,16 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem hostHyperSearch.and("type", hostHyperSearch.entity().getType(), SearchCriteria.Op.EQ); hostHyperSearch.and("zoneId", hostHyperSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ); hostHyperSearch.groupBy(hostHyperSearch.entity().getHypervisorType()); - + tmpltTypeHyperSearch.join("tmplHyper", hostHyperSearch, hostHyperSearch.entity().getHypervisorType(), tmpltTypeHyperSearch.entity().getHypervisorType(), JoinBuilder.JoinType.INNER); hostHyperSearch.done(); tmpltTypeHyperSearch.done(); - + tmpltTypeHyperSearch2 = createSearchBuilder(); tmpltTypeHyperSearch2.and("templateType", tmpltTypeHyperSearch2.entity().getTemplateType(), SearchCriteria.Op.EQ); tmpltTypeHyperSearch2.and("hypervisorType", tmpltTypeHyperSearch2.entity().getHypervisorType(), SearchCriteria.Op.EQ); - + tmpltTypeSearch = createSearchBuilder(); tmpltTypeSearch.and("removed", tmpltTypeSearch.entity().getRemoved(), SearchCriteria.Op.NULL); tmpltTypeSearch.and("templateType", tmpltTypeSearch.entity().getTemplateType(), SearchCriteria.Op.EQ); @@ -363,11 +363,11 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem AccountIdSearch.and("accountId", AccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ); AccountIdSearch.and("publicTemplate", AccountIdSearch.entity().isPublicTemplate(), SearchCriteria.Op.EQ); AccountIdSearch.done(); - + SearchBuilder tmpltZoneSearch = _templateZoneDao.createSearchBuilder(); tmpltZoneSearch.and("removed", tmpltZoneSearch.entity().getRemoved(), SearchCriteria.Op.NULL); tmpltZoneSearch.and("zoneId", tmpltZoneSearch.entity().getZoneId(), SearchCriteria.Op.EQ); - + TmpltsInZoneSearch = createSearchBuilder(); TmpltsInZoneSearch.and("removed", TmpltsInZoneSearch.entity().getRemoved(), SearchCriteria.Op.NULL); TmpltsInZoneSearch.and().op("avoidtype", TmpltsInZoneSearch.entity().getTemplateType(), SearchCriteria.Op.NEQ); @@ -378,11 +378,11 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem TmpltsInZoneSearch.done(); CountTemplatesByAccount = createSearchBuilder(Long.class); - CountTemplatesByAccount.select(null, Func.COUNT, null); + CountTemplatesByAccount.select(null, Func.COUNT, null); CountTemplatesByAccount.and("account", CountTemplatesByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); CountTemplatesByAccount.and("removed", CountTemplatesByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); CountTemplatesByAccount.done(); - + updateStateSearch = this.createSearchBuilder(); updateStateSearch.and("id", updateStateSearch.entity().getId(), Op.EQ); updateStateSearch.and("state", updateStateSearch.entity().getState(), Op.EQ); @@ -472,7 +472,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem } else { whereClause += " AND t.account_id IN (" + permittedAccountsStr + ")"; } - } else if (templateFilter == TemplateFilter.sharedexecutable && caller.getType() != Account.ACCOUNT_TYPE_ADMIN) { + } else if ((templateFilter == TemplateFilter.shared || templateFilter == TemplateFilter.sharedexecutable) && caller.getType() != Account.ACCOUNT_TYPE_ADMIN) { if (caller.getType() == Account.ACCOUNT_TYPE_NORMAL) { joinClause += " LEFT JOIN launch_permission lp ON t.id = lp.template_id WHERE" + " (t.account_id IN (" + permittedAccountsStr + ") OR" + " lp.account_id IN (" + permittedAccountsStr + "))"; @@ -517,7 +517,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem @Override - public Set> searchTemplates(String name, String keyword, TemplateFilter templateFilter, + public Set> searchTemplates(String name, String keyword, TemplateFilter templateFilter, boolean isIso, List hypers, Boolean bootable, DomainVO domain, Long pageSize, Long startIndex, Long zoneId, HypervisorType hyperType, boolean onlyReady, boolean showDomr,List permittedAccounts, Account caller, ListProjectResourcesCriteria listProjectResourcesCriteria, Map tags) { @@ -527,17 +527,17 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem builder.append(permittedAccount.getAccountId() + ","); } } - + String permittedAccountsStr = builder.toString(); - + if (permittedAccountsStr.length() > 0) { //chop the "," off permittedAccountsStr = permittedAccountsStr.substring(0, permittedAccountsStr.length()-1); } - + Transaction txn = Transaction.currentTxn(); txn.start(); - + /* Use LinkedHashSet here to guarantee iteration order */ Set> templateZonePairList = new LinkedHashSet>(); PreparedStatement pstmt = null; @@ -545,15 +545,15 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem StringBuilder relatedDomainIds = new StringBuilder(); String sql = SELECT_TEMPLATE_ZONE_REF; String groupByClause = ""; - try { + try { //short accountType; //String accountId = null; - String guestOSJoin = ""; + String guestOSJoin = ""; StringBuilder templateHostRefJoin = new StringBuilder(); String dataCenterJoin = "", lpjoin = ""; String tagsJoin = ""; - if (isIso && !hyperType.equals(HypervisorType.None)) { + if (isIso && !hyperType.equals(HypervisorType.None)) { guestOSJoin = " INNER JOIN guest_os guestOS on (guestOS.id = t.guest_os_id) INNER JOIN guest_os_hypervisor goh on ( goh.guest_os_id = guestOS.id) "; } if (onlyReady){ @@ -564,34 +564,34 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem if ((templateFilter == TemplateFilter.featured) || (templateFilter == TemplateFilter.community)) { dataCenterJoin = " INNER JOIN data_center dc on (h.data_center_id = dc.id)"; } - - if (templateFilter == TemplateFilter.sharedexecutable){ + + if (templateFilter == TemplateFilter.sharedexecutable || templateFilter == TemplateFilter.shared ){ lpjoin = " INNER JOIN launch_permission lp ON t.id = lp.template_id "; } - + if (tags != null && !tags.isEmpty()) { tagsJoin = " INNER JOIN resource_tags r ON t.id = r.resource_id "; } - + sql += guestOSJoin + templateHostRefJoin + dataCenterJoin + lpjoin + tagsJoin; String whereClause = ""; - + //All joins have to be made before we start setting the condition settings - if ((listProjectResourcesCriteria == ListProjectResourcesCriteria.SkipProjectResources - || (!permittedAccounts.isEmpty() && !(templateFilter == TemplateFilter.community || templateFilter == TemplateFilter.featured))) && + if ((listProjectResourcesCriteria == ListProjectResourcesCriteria.SkipProjectResources + || (!permittedAccounts.isEmpty() && !(templateFilter == TemplateFilter.community || templateFilter == TemplateFilter.featured))) && !(caller.getType() != Account.ACCOUNT_TYPE_NORMAL && templateFilter == TemplateFilter.all)) { whereClause += " INNER JOIN account a on (t.account_id = a.id)"; - if ((templateFilter == TemplateFilter.self || templateFilter == TemplateFilter.selfexecutable) && (caller.getType() == Account.ACCOUNT_TYPE_DOMAIN_ADMIN || caller.getType() == Account.ACCOUNT_TYPE_RESOURCE_DOMAIN_ADMIN)) { - whereClause += " INNER JOIN domain d on (a.domain_id = d.id) WHERE d.path LIKE '" + domain.getPath() + "%'"; + if ((templateFilter == TemplateFilter.self || templateFilter == TemplateFilter.selfexecutable) && (caller.getType() == Account.ACCOUNT_TYPE_DOMAIN_ADMIN || caller.getType() == Account.ACCOUNT_TYPE_RESOURCE_DOMAIN_ADMIN)) { + whereClause += " INNER JOIN domain d on (a.domain_id = d.id) WHERE d.path LIKE '" + domain.getPath() + "%'"; if (listProjectResourcesCriteria == ListProjectResourcesCriteria.SkipProjectResources) { whereClause += " AND a.type != " + Account.ACCOUNT_TYPE_PROJECT; } - } else + } else if (listProjectResourcesCriteria == ListProjectResourcesCriteria.SkipProjectResources) { whereClause += " WHERE a.type != " + Account.ACCOUNT_TYPE_PROJECT; } } - + if (!permittedAccounts.isEmpty()) { for (Account account : permittedAccounts) { //accountType = account.getType(); @@ -621,12 +621,12 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem relatedDomainIds.setLength(relatedDomainIds.length()-1); } } - + String attr = " AND "; if (whereClause.endsWith(" WHERE ")) { attr += " WHERE "; } - + if (!isIso) { if ( hypers.isEmpty() ) { return templateZonePairList; @@ -642,9 +642,10 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem whereClause += attr + " t.hypervisor_type IN (" + relatedHypers + ")"; } } - - if (!permittedAccounts.isEmpty() && !(templateFilter == TemplateFilter.featured || - templateFilter == TemplateFilter.community || templateFilter == TemplateFilter.executable) && !isAdmin(caller.getType()) ) { + + if (!permittedAccounts.isEmpty() && !(templateFilter == TemplateFilter.featured || + templateFilter == TemplateFilter.community || templateFilter == TemplateFilter.executable + || templateFilter == TemplateFilter.shared || templateFilter == TemplateFilter.sharedexecutable) && !isAdmin(caller.getType()) ) { whereClause += attr + "t.account_id IN (" + permittedAccountsStr + ")"; } @@ -652,13 +653,13 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem whereClause += attr + "t.public = 1 AND t.featured = 1"; if (!permittedAccounts.isEmpty()) { whereClause += attr + "(dc.domain_id IN (" + relatedDomainIds + ") OR dc.domain_id is NULL)"; - } + } } else if (templateFilter == TemplateFilter.self || templateFilter == TemplateFilter.selfexecutable) { whereClause += " AND t.account_id IN (" + permittedAccountsStr + ")"; - } else if (templateFilter == TemplateFilter.sharedexecutable) { + } else if (templateFilter == TemplateFilter.sharedexecutable || templateFilter == TemplateFilter.shared ) { whereClause += " AND " + " (t.account_id IN (" + permittedAccountsStr + ") OR" + - " lp.account_id IN (" + permittedAccountsStr + "))"; + " lp.account_id IN (" + permittedAccountsStr + "))"; } else if (templateFilter == TemplateFilter.executable && !permittedAccounts.isEmpty()) { whereClause += attr + "(t.public = 1 OR t.account_id IN (" + permittedAccountsStr + "))"; } else if (templateFilter == TemplateFilter.community) { @@ -669,7 +670,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem } else if (caller.getType() != Account.ACCOUNT_TYPE_ADMIN && !isIso) { return templateZonePairList; } - + if (tags != null && !tags.isEmpty()) { whereClause += " AND ("; boolean first = true; @@ -682,13 +683,13 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem } whereClause += ")"; } - + if (whereClause.equals("")) { whereClause += " WHERE "; } else if (!whereClause.equals(" WHERE ")) { whereClause += " AND "; } - + sql += whereClause + getExtrasWhere(templateFilter, name, keyword, isIso, bootable, hyperType, zoneId, onlyReady, showDomr) + groupByClause + getOrderByLimit(pageSize, startIndex); @@ -699,8 +700,8 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem Pair templateZonePair = new Pair(rs.getLong(1), rs.getLong(2)); templateZonePairList.add(templateZonePair); } - //for now, defaulting pageSize to a large val if null; may need to revisit post 2.2RC2 - if(isIso && templateZonePairList.size() < (pageSize != null ? pageSize : 500) + //for now, defaulting pageSize to a large val if null; may need to revisit post 2.2RC2 + if(isIso && templateZonePairList.size() < (pageSize != null ? pageSize : 500) && templateFilter != TemplateFilter.community && !(templateFilter == TemplateFilter.self && !BaseCmd.isRootAdmin(caller.getType())) ){ //evaluates to true If root admin and filter=self @@ -747,7 +748,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem s_logger.warn("Error in cleaning up", sqle); } } - + return templateZonePairList; } @@ -770,7 +771,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem sql += " AND t.hypervisor_type = '" + hyperType.toString() + "'"; } } - + if (bootable != null) { sql += " AND t.bootable = " + bootable; } @@ -787,7 +788,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem } if (!showDomr){ sql += " AND t.type != '" +Storage.TemplateType.SYSTEM.toString() + "'"; - } + } sql += " AND t.removed IS NULL"; @@ -797,14 +798,14 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem private String getOrderByLimit(Long pageSize, Long startIndex) { Boolean isAscending = Boolean.parseBoolean(_configDao.getValue("sortkey.algorithm")); isAscending = (isAscending == null ? true : isAscending); - + String sql; if (isAscending) { sql = " ORDER BY t.sort_key ASC"; } else { sql = " ORDER BY t.sort_key DESC"; } - + if ((pageSize != null) && (startIndex != null)) { sql += " LIMIT " + startIndex.toString() + "," + pageSize.toString(); } @@ -835,7 +836,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem _templateZoneDao.update(tmpltZoneVO.getId(), tmpltZoneVO); } txn.commit(); - + return tmplt.getId(); } @@ -854,7 +855,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem sc.setParameters("templateType", Storage.TemplateType.BUILTIN); return listBy(sc); } - + @Override public VMTemplateVO findSystemVMTemplate(long zoneId) { SearchCriteria sc = tmpltTypeHyperSearch.create(); @@ -864,14 +865,14 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem //order by descending order of id and select the first (this is going to be the latest) List tmplts = listBy(sc, new Filter(VMTemplateVO.class, "id", false, null, 1l)); - + if (tmplts.size() > 0) { return tmplts.get(0); } else { return null; } } - + public VMTemplateVO findSystemVMTemplate(long zoneId, HypervisorType hType) { SearchCriteria sc = tmpltTypeHyperSearch.create(); sc.setParameters("templateType", Storage.TemplateType.SYSTEM); @@ -900,7 +901,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem //order by descending order of id and select the first (this is going to be the latest) List tmplts = listBy(sc, new Filter(VMTemplateVO.class, "id", false, null, 1l)); - + if (tmplts.size() > 0) { return tmplts.get(0); } else { @@ -914,7 +915,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem sc.setParameters("account", accountId); return customSearch(sc, null).get(0); } - + @Override @DB public boolean remove(Long id) { @@ -922,7 +923,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem txn.start(); VMTemplateVO template = createForUpdate(); template.setRemoved(new Date()); - + VMTemplateVO vo = findById(id); if (vo != null) { if (vo.getFormat() == ImageFormat.ISO) { @@ -936,14 +937,14 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem txn.commit(); return result; } - + private boolean isAdmin(short accountType) { return ((accountType == Account.ACCOUNT_TYPE_ADMIN) || (accountType == Account.ACCOUNT_TYPE_RESOURCE_DOMAIN_ADMIN) || (accountType == Account.ACCOUNT_TYPE_DOMAIN_ADMIN) || (accountType == Account.ACCOUNT_TYPE_READ_ONLY_ADMIN)); } - + @Override public List findTemplatesToSyncToS3() { return executeList(SELECT_S3_CANDIDATE_TEMPLATES, new Object[] {}); @@ -1037,7 +1038,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem joinClause.append(" LEFT JOIN launch_permission lp ON t.id = lp.template_id WHERE (t.account_id IN ("); joinClause.append(permittedAccountsStr); joinClause.append(") OR lp.account_id IN ("); - joinClause.append(permittedAccountsStr); + joinClause.append(permittedAccountsStr); joinClause.append("))"); } else { joinClause.append(" INNER JOIN account a on (t.account_id = a.id) "); @@ -1088,8 +1089,8 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem TemplateState nextState, VMTemplateVO vo, Object data) { Long oldUpdated = vo.getUpdatedCount(); Date oldUpdatedTime = vo.getUpdated(); - - + + SearchCriteria sc = updateStateSearch.create(); sc.setParameters("id", vo.getId()); sc.setParameters("state", currentState); From 9f7b82281d5768e0b8a1ca67e5d9d7014c7ed3b5 Mon Sep 17 00:00:00 2001 From: David Nalley Date: Mon, 11 Mar 2013 21:58:07 -0400 Subject: [PATCH 06/34] commenting out docs --- packaging/centos63/cloud.spec | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index a79ef175948..2fc744dbaff 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -150,10 +150,10 @@ Obsoletes: cloud-aws-api < 4.1.0 Provides: cloud-aws-api Apache Cloudstack AWS API compatibility wrapper -%package docs -Summary: Apache CloudStack documentation -%description docs -Apache CloudStack documentations +#%package docs +#Summary: Apache CloudStack documentation +#%description docs +#Apache CloudStack documentations %prep echo Doing CloudStack build @@ -432,9 +432,9 @@ fi %doc LICENSE %doc NOTICE -%files docs -%doc LICENSE -%doc NOTICE +#%files docs +#%doc LICENSE +#%doc NOTICE %files awsapi %defattr(0644,cloud,cloud,0755) From c235d029ce1ae22f7e7b041bc2b36ac63368f38e Mon Sep 17 00:00:00 2001 From: Kelven Yang Date: Mon, 11 Mar 2013 19:06:12 -0700 Subject: [PATCH 07/34] CLOUDSTACK-1267: Configure log4j in proper place so that it can be initialized correctly in KVM and system VM environment --- agent/src/com/cloud/agent/AgentShell.java | 15 +++++++++++++++ utils/src/com/cloud/utils/LogUtils.java | 3 +-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/agent/src/com/cloud/agent/AgentShell.java b/agent/src/com/cloud/agent/AgentShell.java index 9cb3c3db3ff..2af08e9bf21 100644 --- a/agent/src/com/cloud/agent/AgentShell.java +++ b/agent/src/com/cloud/agent/AgentShell.java @@ -38,10 +38,13 @@ import java.util.UUID; import javax.naming.ConfigurationException; +import org.apache.commons.beanutils.PropertyUtils; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.log4j.Logger; +import org.apache.log4j.PropertyConfigurator; +import org.apache.log4j.xml.DOMConfigurator; import com.cloud.agent.Agent.ExitStatus; import com.cloud.agent.dao.StorageComponent; @@ -377,6 +380,18 @@ public class AgentShell implements IAgentShell { public void init(String[] args) throws ConfigurationException { + // PropertiesUtil is used both in management server and agent packages, + // it searches path under class path and common J2EE containers + // For KVM agent, do it specially here + + File file = new File("/etc/cloudstack/agent/log4j-cloud.xml"); + if(file == null || !file.exists()) { + file = PropertiesUtil.findConfigFile("log4j-cloud.xml"); + } + DOMConfigurator.configureAndWatch(file.getAbsolutePath()); + + s_logger.info("Agent started"); + final Class c = this.getClass(); _version = c.getPackage().getImplementationVersion(); if (_version == null) { diff --git a/utils/src/com/cloud/utils/LogUtils.java b/utils/src/com/cloud/utils/LogUtils.java index 975de831622..19c4c0e3e2e 100644 --- a/utils/src/com/cloud/utils/LogUtils.java +++ b/utils/src/com/cloud/utils/LogUtils.java @@ -19,7 +19,6 @@ package com.cloud.utils; import java.io.File; import org.apache.log4j.Logger; -import org.apache.log4j.PropertyConfigurator; import org.apache.log4j.xml.DOMConfigurator; public class LogUtils { @@ -36,7 +35,7 @@ public class LogUtils { file = PropertiesUtil.findConfigFile(nameWithoutExtension + ".properties"); if (file != null) { s_logger.info("log4j configuration found at " + file.getAbsolutePath()); - PropertyConfigurator.configureAndWatch(file.getAbsolutePath()); + DOMConfigurator.configureAndWatch(file.getAbsolutePath()); } } } From da89946ca93a872d0a4bf907d4545c392b2055f1 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Tue, 12 Mar 2013 11:56:21 +0530 Subject: [PATCH 08/34] CLOUDSTACK-1625. NPE with updateResourceCount when && is passed thru API. If any API contains '&' i.e. no key value pair or '&' i.e. a parameter without a value, then we get an NPE as owasp.esapi.StringUtilities.stripControls deosn't handle NPE. --- server/src/com/cloud/api/ApiServer.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/src/com/cloud/api/ApiServer.java b/server/src/com/cloud/api/ApiServer.java index deb5e12f9fc..0439c6e2cc9 100755 --- a/server/src/com/cloud/api/ApiServer.java +++ b/server/src/com/cloud/api/ApiServer.java @@ -327,10 +327,12 @@ public class ApiServer implements HttpRequestHandler, ApiServerService { } String[] value = (String[]) params.get(key); // fail if parameter value contains ASCII control (non-printable) characters - String newValue = StringUtils.stripControlCharacters(value[0]); - if ( !newValue.equals(value[0]) ) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Received value " + value[0] + " for parameter " - + key + " is invalid, contains illegal ASCII non-printable characters"); + if (value[0] != null) { + String newValue = StringUtils.stripControlCharacters(value[0]); + if ( !newValue.equals(value[0]) ) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Received value " + value[0] + " for parameter " + + key + " is invalid, contains illegal ASCII non-printable characters"); + } } paramMap.put(key, value[0]); } From 9bcfaffc3195f29997790fa9386d814c1de05b37 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Tue, 12 Mar 2013 14:43:11 +0530 Subject: [PATCH 09/34] plugins: Add it's to cmdClass in ApiDiscovery, spring fails to self inject Signed-off-by: Rohit Yadav --- .../org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/api/discovery/src/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java b/plugins/api/discovery/src/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java index 293a1a47c19..b3714883964 100755 --- a/plugins/api/discovery/src/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java +++ b/plugins/api/discovery/src/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java @@ -73,6 +73,7 @@ public class ApiDiscoveryServiceImpl implements ApiDiscoveryService { s_logger.debug(String.format("getting api commands of service: %s", service.getClass().getName())); cmdClasses.addAll(service.getCommands()); } + cmdClasses.addAll(this.getCommands()); cacheResponseMap(cmdClasses); long endTime = System.nanoTime(); s_logger.info("Api Discovery Service: Annotation, docstrings, api relation graph processed in " + (endTime - startTime) / 1000000.0 + " ms"); From 162de5b17f2caab664e20ae9c966fcaf9f86d2cf Mon Sep 17 00:00:00 2001 From: Kelven Yang Date: Fri, 8 Mar 2013 16:54:46 -0800 Subject: [PATCH 10/34] CLOUDSTACK-1323: retire some of no-longer used files Conflicts: client/tomcatconf/components.xml.in Signed-off-by: Prasanna Santhanam --- .../tomcatconf/components-cloudzones.xml.in | 33 -- client/tomcatconf/components-nonoss.xml.in | 97 ------ client/tomcatconf/components.xml.in | 306 ------------------ 3 files changed, 436 deletions(-) delete mode 100755 client/tomcatconf/components-cloudzones.xml.in delete mode 100755 client/tomcatconf/components-nonoss.xml.in delete mode 100755 client/tomcatconf/components.xml.in diff --git a/client/tomcatconf/components-cloudzones.xml.in b/client/tomcatconf/components-cloudzones.xml.in deleted file mode 100755 index 4bd39108e1f..00000000000 --- a/client/tomcatconf/components-cloudzones.xml.in +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - - - - - - true - - - diff --git a/client/tomcatconf/components-nonoss.xml.in b/client/tomcatconf/components-nonoss.xml.in deleted file mode 100755 index fbfc5cc2726..00000000000 --- a/client/tomcatconf/components-nonoss.xml.in +++ /dev/null @@ -1,97 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - true - - - - - - diff --git a/client/tomcatconf/components.xml.in b/client/tomcatconf/components.xml.in deleted file mode 100755 index 1d3faf384dc..00000000000 --- a/client/tomcatconf/components.xml.in +++ /dev/null @@ -1,306 +0,0 @@ - - - - - - - - - - - - - - - true - - - - 1 - 25 - 50000 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From b5f367e75a50c70a18dcc246b80a08e3167c37b6 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Tue, 12 Mar 2013 16:34:50 +0530 Subject: [PATCH 11/34] CLOUDSTACK-1643: Check and enable Intel AES NI if available for VR This is an encryption optimization for VPN/SSL, with upto 10x advertised speed. The patch check for the aesni_intel module if aes is available on the cpu, this will be true for HVMs. Signed-off-by: Rohit Yadav --- .../debian/config/etc/init.d/cloud-early-config | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/patches/systemvm/debian/config/etc/init.d/cloud-early-config b/patches/systemvm/debian/config/etc/init.d/cloud-early-config index 877ab06009d..408264b478a 100755 --- a/patches/systemvm/debian/config/etc/init.d/cloud-early-config +++ b/patches/systemvm/debian/config/etc/init.d/cloud-early-config @@ -593,9 +593,20 @@ setup_redundant_router() { fi } +setup_aesni() { + if [ `grep aes /proc/cpuinfo | wc -l` -gt 0 ] + then + modprobe aesni_intel + if [ `lsmod | grep aesni_intel | wc -l` -gt 0 ] + then + echo aesni_intel >> /etc/modules + fi + fi +} + setup_router() { log_it "Setting up virtual router system vm" - + oldmd5= [ -f "/etc/udev/rules.d/70-persistent-net.rules" ] && oldmd5=$(md5sum "/etc/udev/rules.d/70-persistent-net.rules" | awk '{print $1}') @@ -643,10 +654,8 @@ setup_router() { fi fi - - + setup_aesni setup_dnsmasq - setup_apache2 $ETH0_IP sed -i /gateway/d /etc/hosts From 6eade59209233f37dd8ad6768dd8b814205d7daf Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Tue, 12 Mar 2013 16:32:46 +0530 Subject: [PATCH 12/34] CLOUDSTACK-1644: INFRA-5977 ASFBot test This is a fake patch for a test Signed-off-by: Prasanna Santhanam --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 7b4d973666e..cdc9fb6175b 100644 --- a/README.md +++ b/README.md @@ -78,3 +78,6 @@ The following provides more details on the included cryptographic software: CloudStack has a dependency on Apache WSS4J as part of the AWSAPI implementation. CloudStack has a dependency on and makes use of JSch - a java SSH2 implementation. + +# ASFBot test +This is for an ASFBot test. Will be removed soon From bc366124829337ddae3e6c1e10109f6e11b661ac Mon Sep 17 00:00:00 2001 From: Prasanna Santhanam Date: Tue, 12 Mar 2013 17:08:35 +0530 Subject: [PATCH 13/34] CLOUDSTACK-1644: INFRA-5977 ASFBot test This reverts commit 6eade59209233f37dd8ad6768dd8b814205d7daf. This is a second test. --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index cdc9fb6175b..7b4d973666e 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,3 @@ The following provides more details on the included cryptographic software: CloudStack has a dependency on Apache WSS4J as part of the AWSAPI implementation. CloudStack has a dependency on and makes use of JSch - a java SSH2 implementation. - -# ASFBot test -This is for an ASFBot test. Will be removed soon From 9af338c651f2dc21875dc748233aba5b93dc3989 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Tue, 12 Mar 2013 17:56:43 +0530 Subject: [PATCH 14/34] appliance: While building don't just check empty arg, check if it was passed The fix makes systemvmtemplate as the default veewee vbox appliance if not other appliance is provided to be built Signed-off-by: Rohit Yadav --- tools/appliance/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index d18faaf7ce6..f1ee4a64b40 100644 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -18,7 +18,7 @@ set -x -if [ "$1"!="" ] +if [ ! -z "$1" ] then appliance="$1" else From c98244f75d05263a6c305bdd901d81a9306f1f15 Mon Sep 17 00:00:00 2001 From: Mice Xia Date: Tue, 12 Mar 2013 20:34:30 +0800 Subject: [PATCH 15/34] CLOUDSTACK-1634 VMSnapshot: CreateVMSnapshotCmd fails due to due to java.util.ArrayList cannot be cast to [Lcom.vmware.vim25.ManagedObjectReference --- .../hypervisor/vmware/manager/VmwareStorageManagerImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java index 8be76e8fff2..e11dd53f3c9 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.OutputStreamWriter; import java.rmi.RemoteException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -915,7 +916,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { // wait if there are already VM snapshot task running ManagedObjectReference taskmgr = context.getServiceContent().getTaskManager(); - ManagedObjectReference[] tasks = (ManagedObjectReference[]) context.getVimClient().getDynamicProperty(taskmgr, "recentTask"); + List tasks = (ArrayList)context.getVimClient().getDynamicProperty(taskmgr, "recentTask"); for (ManagedObjectReference taskMor : tasks) { TaskInfo info = (TaskInfo) (context.getVimClient().getDynamicProperty(taskMor, "info")); if(info.getEntityName().equals(cmd.getVmName()) && info.getName().equalsIgnoreCase("CreateSnapshot_Task")){ @@ -1048,7 +1049,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { // wait if there are already VM revert task running ManagedObjectReference taskmgr = context.getServiceContent().getTaskManager(); - ManagedObjectReference[] tasks = (ManagedObjectReference[]) context.getVimClient().getDynamicProperty(taskmgr, "recentTask"); + List tasks = (ArrayList)context.getVimClient().getDynamicProperty(taskmgr, "recentTask"); for (ManagedObjectReference taskMor : tasks) { TaskInfo info = (TaskInfo) (context.getVimClient().getDynamicProperty(taskMor, "info")); if(info.getEntityName().equals(cmd.getVmName()) && info.getName().equalsIgnoreCase("RevertToSnapshot_Task")){ From aee6fc3bced4c56d0bf3e969e368f14e20d11d36 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Tue, 12 Mar 2013 10:24:44 +0100 Subject: [PATCH 16/34] debian: Copy the contents of the old ~cloud directory to the new location --- debian/cloudstack-management.postinst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/cloudstack-management.postinst b/debian/cloudstack-management.postinst index 5b6b4b351c3..7b6a1ed6d71 100644 --- a/debian/cloudstack-management.postinst +++ b/debian/cloudstack-management.postinst @@ -20,7 +20,7 @@ if [ "$1" = configure ]; then if ! getent passwd cloud >/dev/null; then adduser --quiet --system --group --no-create-home --home /var/lib/cloudstack/management cloud else - usermod -m /var/lib/cloudstack/management cloud + usermod -m -d /var/lib/cloudstack/management cloud fi chown cloud /var/log/cloudstack/management -fi +fi \ No newline at end of file From 7a794519a2703ee35651123862ffd5445df24063 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Tue, 12 Mar 2013 13:05:44 +0100 Subject: [PATCH 17/34] debian: Fix the usage server packaging --- debian/cloudstack-usage.install | 3 ++- debian/rules | 3 +++ packaging/debian/init/cloud-usage | 18 +++++++----------- usage/pom.xml | 18 ++++++++++++++++++ 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/debian/cloudstack-usage.install b/debian/cloudstack-usage.install index 53f2a56afa3..4722e688dcd 100644 --- a/debian/cloudstack-usage.install +++ b/debian/cloudstack-usage.install @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. -/usr/share/cloudstack-usage/lib/cloudstack-usage.jar +/usr/share/cloudstack-usage/lib/* +/usr/share/cloudstack-usage/plugins /etc/init.d/cloudstack-usage /var/log/cloudstack/usage /etc/cloudstack/usage/* diff --git a/debian/rules b/debian/rules index f0cb67f276a..0967979a1ed 100755 --- a/debian/rules +++ b/debian/rules @@ -124,7 +124,10 @@ install: # cloudstack-usage mkdir $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/usage mkdir $(DESTDIR)/var/log/$(PACKAGE)/usage + mkdir $(DESTDIR)/usr/share/$(PACKAGE)-usage + mkdir $(DESTDIR)/usr/share/$(PACKAGE)-usage/plugins install -D usage/target/cloud-usage-4.2.0-SNAPSHOT.jar $(DESTDIR)/usr/share/$(PACKAGE)-usage/lib/$(PACKAGE)-usage.jar + install -D usage/target/dependencies/* $(DESTDIR)/usr/share/$(PACKAGE)-usage/lib/ cp usage/target/transformed/* $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/usage/ ln -s ../management/db.properties $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/usage/db.properties install -D packaging/debian/init/cloud-usage $(DESTDIR)/$(SYSCONFDIR)/init.d/$(PACKAGE)-usage diff --git a/packaging/debian/init/cloud-usage b/packaging/debian/init/cloud-usage index 3421cd1fbd4..1cc75cc03f8 100755 --- a/packaging/debian/init/cloud-usage +++ b/packaging/debian/init/cloud-usage @@ -1,7 +1,7 @@ #!/bin/bash ### BEGIN INIT INFO -# Provides: cloud usage +# Provides: cloudstack usage # Required-Start: $network $local_fs # Required-Stop: $network $local_fs # Default-Start: 3 4 5 @@ -32,17 +32,15 @@ . /lib/lsb/init-functions -SHORTNAME="cloud-usage" +SHORTNAME="cloudstack-usage" PIDFILE=/var/run/"$SHORTNAME".pid -LOGFILE=/var/log/cloud/usage/usage-server.log PROGNAME="CloudStack Usage Monitor" CLASS="com.cloud.usage.UsageServer" PROG="jsvc" DAEMON="/usr/bin/jsvc" -USER=@MSUSER@ unset OPTIONS -[ -r @SYSCONFDIR@/default/"$SHORTNAME" ] && source @SYSCONFDIR@/default/"$SHORTNAME" +[ -r /etc/default/"$SHORTNAME" ] && source /etc/default/"$SHORTNAME" # The first existing directory is used for JAVA_HOME (if JAVA_HOME is not defined in $DEFAULT) JDK_DIRS="/usr/lib/jvm/java-7-openjdk-amd64 /usr/lib/jvm/java-7-openjdk-i386 /usr/lib/jvm/java-6-openjdk /usr/lib/jvm/java-6-openjdk-i386 /usr/lib/jvm/java-6-openjdk-amd64 /usr/lib/jvm/java-6-sun" @@ -54,14 +52,12 @@ for jdir in $JDK_DIRS; do done export JAVA_HOME -SCP="@SYSTEMCLASSPATH@" -DCP="@DEPSCLASSPATH@" -UCP="@USAGECLASSPATH@" -JCP="/usr/share/java/commons-daemon.jar" +UCP=`ls /usr/share/cloudstack-usage/lib/*.jar | tr '\n' ':' | sed s'/.$//'` +PCP=`ls /usr/share/cloudstack-usage/plugins/*.jar 2>/dev/null | tr '\n' ':' | sed s'/.$//'` # We need to append the JSVC daemon JAR to the classpath # AgentShell implements the JSVC daemon methods -export CLASSPATH="$SCP:$DCP:$UCP:$JCP:@USAGESYSCONFDIR@" +export CLASSPATH="/usr/share/java/commons-daemon.jar:$UCP:$PCP:/etc/cloudstack/usage" start() { if [ -s "$PIDFILE" ] && kill -0 $(cat "$PIDFILE") >/dev/null 2>&1; then @@ -79,7 +75,7 @@ start() { exit 1 fi - if start_daemon -p $PIDFILE $DAEMON -cp "$CLASSPATH" -pidfile "$PIDFILE" -user "$USER" -outfile SYSLOG -errfile SYSLOG -Dpid=$$ $CLASS + if start_daemon -p $PIDFILE $DAEMON -cp "$CLASSPATH" -pidfile "$PIDFILE" -outfile SYSLOG -errfile SYSLOG -Dpid=$$ $CLASS RETVAL=$? then rc=0 diff --git a/usage/pom.xml b/usage/pom.xml index 28ab1a7dfc0..31df358ea4c 100644 --- a/usage/pom.xml +++ b/usage/pom.xml @@ -127,6 +127,24 @@ + + org.apache.maven.plugins + maven-dependency-plugin + 2.5.1 + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.directory}/dependencies + runtime + + + + From 594bb9b5a3391cb41a1122b15abc2ad590b920e4 Mon Sep 17 00:00:00 2001 From: Pradeep Date: Tue, 12 Mar 2013 11:28:51 +0530 Subject: [PATCH 18/34] Adding awsapi in the require section of cloud.spec as it resolves issue with EC2 QUERY API Feature. Moved the post awsapi section to the post management since it is trying to provide symb link before installing management server. Signed-off-by: Pradeep --- packaging/centos63/cloud.spec | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index 2fc744dbaff..8f71f03b811 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -78,7 +78,8 @@ Requires: mkisofs Requires: MySQL-python Requires: python-paramiko Requires: ipmitool -Requires: %{name}-common = %{_ver} +Requires: %{name}-common = %{_ver} +Requires: %{name}-awsapi = %{_ver} Obsoletes: cloud-client < 4.1.0 Obsoletes: cloud-client-ui < 4.1.0 Obsoletes: cloud-daemonize < 4.1.0 @@ -318,6 +319,10 @@ if [ "$1" == "1" ] ; then /sbin/chkconfig --level 345 cloud-management on > /dev/null 2>&1 || true fi +if [ -d "%{_datadir}/%{name}-management" ] ; then + ln -s %{_datadir}/%{name}-bridge/webapps %{_datadir}/%{name}-management/webapps7080 +fi + if [ ! -f %{_datadir}/cloudstack-common/scripts/vm/hypervisor/xenserver/vhd-util ] ; then echo Please download vhd-util from http://download.cloud.com.s3.amazonaws.com/tools/vhd-util and put it in echo %{_datadir}/cloudstack-common/scripts/vm/hypervisor/xenserver/ @@ -330,10 +335,10 @@ if getent passwd cloud | grep -q /var/lib/cloud; then fi -%post awsapi -if [ -d "%{_datadir}/%{name}-management" ] ; then - ln -s %{_datadir}/%{name}-bridge/webapps %{_datadir}/%{name}-management/webapps7080 -fi +#%post awsapi +#if [ -d "%{_datadir}/%{name}-management" ] ; then +# ln -s %{_datadir}/%{name}-bridge/webapps %{_datadir}/%{name}-management/webapps7080 +#fi #No default permission as the permission setup is complex %files management From 9abb4d5f4b9a5afca7be1ffa17c005a9e6ede3a2 Mon Sep 17 00:00:00 2001 From: Pranav Saxena Date: Tue, 12 Mar 2013 19:38:46 +0530 Subject: [PATCH 19/34] CLOUDSTACK-1539:Editing any parameter in the Network UI results in implementation of IP Reservation. --- ui/scripts/network.js | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ui/scripts/network.js b/ui/scripts/network.js index 6fcd46040ef..2353a038f1f 100755 --- a/ui/scripts/network.js +++ b/ui/scripts/network.js @@ -629,8 +629,20 @@ networkdomain: args.data.networkdomain }); } + + var oldcidr; + $.ajax({ + url: createURL("listNetworks&id=" + args.context.networks[0].id ), + dataType: "json", + async: false, + success: function(json) { + oldcidr = json.listnetworksresponse.network[0].cidr; + + } + }); - if(args.data.cidr !="" ){ + + if(args.data.cidr !="" && args.data.cidr != oldcidr ){ $.extend(data, { guestvmcidr: args.data.cidr }); From bfb225bc167fdc4a59125c69dfa5255bb4490109 Mon Sep 17 00:00:00 2001 From: David Nalley Date: Mon, 11 Mar 2013 23:20:14 -0400 Subject: [PATCH 20/34] CLOUDSTACK-1630 - Actually put obsoletes in the right order --- packaging/centos63/cloud.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index 8f71f03b811..16c36020b45 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -146,9 +146,9 @@ Apache CloudStack command line interface %package awsapi Summary: Apache CloudStack AWS API compatibility wrapper Requires: %{name}-management = %{_ver} -%description awsapi Obsoletes: cloud-aws-api < 4.1.0 Provides: cloud-aws-api +%description awsapi Apache Cloudstack AWS API compatibility wrapper #%package docs From a331cfa20f837384d3ec4e1bcecea29b3b65636f Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Tue, 12 Mar 2013 15:12:49 +0100 Subject: [PATCH 21/34] debian: Do not hard code filenames to package --- debian/rules | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debian/rules b/debian/rules index 0967979a1ed..429e93b24da 100755 --- a/debian/rules +++ b/debian/rules @@ -64,8 +64,8 @@ install: mkdir $(DESTDIR)/var/log/$(PACKAGE)/agent mkdir $(DESTDIR)/usr/share/$(PACKAGE)-agent mkdir $(DESTDIR)/usr/share/$(PACKAGE)-agent/plugins - install -D agent/target/cloud-agent-4.2.0-SNAPSHOT.jar $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/$(PACKAGE)-agent.jar - install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-4.2.0-SNAPSHOT.jar $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/ + install -D agent/target/cloud-agent-$(VERSION)-SNAPSHOT.jar $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/$(PACKAGE)-agent.jar + install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-$(VERSION)-SNAPSHOT.jar $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/ install -D plugins/hypervisors/kvm/target/dependencies/* $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/ install -D packaging/debian/init/cloud-agent $(DESTDIR)/$(SYSCONFDIR)/init.d/$(PACKAGE)-agent install -D agent/bindir/cloud-setup-agent.in $(DESTDIR)/usr/bin/cloud-setup-agent @@ -86,14 +86,14 @@ install: mkdir $(DESTDIR)/var/lib/$(PACKAGE)/management mkdir $(DESTDIR)/var/lib/$(PACKAGE)/mnt cp -r client/target/utilities/scripts/db/* $(DESTDIR)/usr/share/$(PACKAGE)-management/setup/ - cp -r client/target/cloud-client-ui-4.2.0-SNAPSHOT/* $(DESTDIR)/usr/share/$(PACKAGE)-management/webapps/client/ + cp -r client/target/cloud-client-ui-$(VERSION)-SNAPSHOT/* $(DESTDIR)/usr/share/$(PACKAGE)-management/webapps/client/ cp server/target/conf/* $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/server/ cp client/target/conf/* $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/management/ ln -s tomcat6-nonssl.conf $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/management/tomcat6.conf mkdir -p $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/management/Catalina/localhost/client install -D packaging/debian/init/cloud-management $(DESTDIR)/$(SYSCONFDIR)/init.d/$(PACKAGE)-management install -D client/bindir/cloud-update-xenserver-licenses.in $(DESTDIR)/usr/bin/cloud-update-xenserver-licenses - install -D server/target/cloud-server-4.2.0-SNAPSHOT.jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/$(PACKAGE)-server.jar + install -D server/target/cloud-server-$(VERSION)-SNAPSHOT.jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/$(PACKAGE)-server.jar ln -s /usr/share/tomcat6/bin $(DESTDIR)/usr/share/$(PACKAGE)-management/bin ln -s ../../..$(SYSCONFDIR)/$(PACKAGE)/management $(DESTDIR)/usr/share/$(PACKAGE)-management/conf ln -s /usr/share/tomcat6/lib $(DESTDIR)/usr/share/$(PACKAGE)-management/lib @@ -126,7 +126,7 @@ install: mkdir $(DESTDIR)/var/log/$(PACKAGE)/usage mkdir $(DESTDIR)/usr/share/$(PACKAGE)-usage mkdir $(DESTDIR)/usr/share/$(PACKAGE)-usage/plugins - install -D usage/target/cloud-usage-4.2.0-SNAPSHOT.jar $(DESTDIR)/usr/share/$(PACKAGE)-usage/lib/$(PACKAGE)-usage.jar + install -D usage/target/cloud-usage-$(VERSION)-SNAPSHOT.jar $(DESTDIR)/usr/share/$(PACKAGE)-usage/lib/$(PACKAGE)-usage.jar install -D usage/target/dependencies/* $(DESTDIR)/usr/share/$(PACKAGE)-usage/lib/ cp usage/target/transformed/* $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/usage/ ln -s ../management/db.properties $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/usage/db.properties From 67f87fe4f2ad07ed51d357ddcee34a18ea63ae00 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Tue, 12 Mar 2013 16:11:55 +0100 Subject: [PATCH 22/34] debian: Change the location of the SystemVM ISO --- debian/rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/rules b/debian/rules index 429e93b24da..7fc6425b34f 100755 --- a/debian/rules +++ b/debian/rules @@ -115,7 +115,7 @@ install: install -D client/target/utilities/bin/cloud-set-guest-sshkey $(DESTDIR)/usr/bin install -D client/target/utilities/bin/cloud-setup-databases $(DESTDIR)/usr/bin install -D client/target/utilities/bin/cloud-setup-management $(DESTDIR)/usr/bin - install -D services/console-proxy/server/dist/systemvm.iso $(DESTDIR)/usr/share/$(PACKAGE)-common/vms/systemvm.iso + install -D client/target/cloud-client-ui-$(VERSION)-SNAPSHOT/WEB-INF/classes/vms/systemvm.iso $(DESTDIR)/usr/share/$(PACKAGE)-common/vms/systemvm.iso # cloudstack-python mkdir -p $(DESTDIR)/usr/lib/python2.7/dist-packages From cd70c8d03f7a7150a29505efb79244da999c3c54 Mon Sep 17 00:00:00 2001 From: David Grizzanti Date: Tue, 12 Mar 2013 11:31:06 -0400 Subject: [PATCH 23/34] CLOUDSTACK-1544 updated API documentation on deleteUser command; fixed API and id parameter descriptions; fixed responseObject doc designation --- .../cloudstack/api/command/admin/user/DeleteUserCmd.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/org/apache/cloudstack/api/command/admin/user/DeleteUserCmd.java b/api/src/org/apache/cloudstack/api/command/admin/user/DeleteUserCmd.java index bb37d085c8e..5683d58ab30 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/user/DeleteUserCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/user/DeleteUserCmd.java @@ -33,7 +33,7 @@ import com.cloud.user.Account; import com.cloud.user.User; import com.cloud.user.UserContext; -@APICommand(name = "deleteUser", description="Creates a user for an account", responseObject=UserResponse.class) +@APICommand(name = "deleteUser", description="Deletes a user for an account", responseObject=SuccessResponse.class) public class DeleteUserCmd extends BaseCmd { public static final Logger s_logger = Logger.getLogger(DeleteUserCmd.class.getName()); @@ -42,7 +42,7 @@ public class DeleteUserCmd extends BaseCmd { ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name=ApiConstants.ID, type=CommandType.UUID, entityType=UserResponse.class, required=true, description="Deletes a user") + @Parameter(name=ApiConstants.ID, type=CommandType.UUID, entityType=UserResponse.class, required=true, description="id of the user to be deleted") private Long id; @Inject RegionService _regionService; From 0ca85659a66bbf722bd02d35ee899e8d074e6fbe Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Tue, 12 Mar 2013 12:20:22 -0600 Subject: [PATCH 24/34] CLOUDSTACK-1651 - point agent scripts to correct logging location, due to move of /var/log/cloud to /var/log/cloudstack Signed-off-by: Marcus Sorensen 1363112422 -0600 --- packaging/centos63/cloud-ipallocator.rc | 2 +- scripts/storage/qcow2/resizevolume.sh | 4 ++-- scripts/vm/network/security_group.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/centos63/cloud-ipallocator.rc b/packaging/centos63/cloud-ipallocator.rc index ffeffa342c8..08cf9c88a0f 100755 --- a/packaging/centos63/cloud-ipallocator.rc +++ b/packaging/centos63/cloud-ipallocator.rc @@ -30,7 +30,7 @@ whatami=cloud-external-ipallocator SHORTNAME="$whatami" PIDFILE=/var/run/"$whatami".pid LOCKFILE=/var/lock/subsys/"$SHORTNAME" -LOGFILE=/var/log/cloud/ipallocator/ipallocator.log +LOGFILE=/var/log/cloudstack/ipallocator/ipallocator.log PROGNAME="External IPAllocator" unset OPTIONS diff --git a/scripts/storage/qcow2/resizevolume.sh b/scripts/storage/qcow2/resizevolume.sh index d15513e250f..6a5c91d9dd6 100755 --- a/scripts/storage/qcow2/resizevolume.sh +++ b/scripts/storage/qcow2/resizevolume.sh @@ -68,7 +68,7 @@ log() { if [ $shouldwelog -eq 1 ] then - echo "$d - $1" >> /var/log/cloud/agent/resizevolume.log + echo "$d - $1" >> /var/log/cloudstack/agent/resizevolume.log fi } @@ -237,7 +237,7 @@ do esac done -shouldwelog=1 #set this to 1 while debugging to get output in /var/log/cloud/agent/resizevolume.log +shouldwelog=1 #set this to 1 while debugging to get output in /var/log/cloudstack/agent/resizevolume.log if [ "$ptype" == "CLVM" ] then diff --git a/scripts/vm/network/security_group.py b/scripts/vm/network/security_group.py index 83d7ad13c86..ed4180a508e 100755 --- a/scripts/vm/network/security_group.py +++ b/scripts/vm/network/security_group.py @@ -797,7 +797,7 @@ def addFWFramework(brname): return False if __name__ == '__main__': - logging.basicConfig(filename="/var/log/cloud/security_group.log", format="%(asctime)s - %(message)s", level=logging.DEBUG) + logging.basicConfig(filename="/var/log/cloudstack/agent/security_group.log", format="%(asctime)s - %(message)s", level=logging.DEBUG) parser = OptionParser() parser.add_option("--vmname", dest="vmName") parser.add_option("--vmip", dest="vmIP") From d55560a6270238aefca3949d000ec53e7d7221a5 Mon Sep 17 00:00:00 2001 From: Kelven Yang Date: Tue, 12 Mar 2013 14:56:35 -0700 Subject: [PATCH 25/34] CLOUDSTACK-1624: Initialize configuration variables properly to avoid influence of any racing conditions --- .../configuration/ConfigurationManagerImpl.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index 7f449dec60e..b1ad6b7fb9d 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -252,17 +252,19 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati // FIXME - why don't we have interface for DataCenterLinkLocalIpAddressDao? @Inject protected DataCenterLinkLocalIpAddressDao _LinkLocalIpAllocDao; - private int _maxVolumeSizeInGb; - private long _defaultPageSize; + private int _maxVolumeSizeInGb = Integer.parseInt(Config.MaxVolumeSize.getDefaultValue()); + private long _defaultPageSize = Long.parseLong(Config.DefaultPageSize.getDefaultValue()); protected Set configValuesForValidation; @Override public boolean configure(final String name, final Map params) throws ConfigurationException { - String maxVolumeSizeInGbString = _configDao.getValue("storage.max.volume.size"); - _maxVolumeSizeInGb = NumbersUtil.parseInt(maxVolumeSizeInGbString, 2000); + String maxVolumeSizeInGbString = _configDao.getValue(Config.MaxVolumeSize.key()); + _maxVolumeSizeInGb = NumbersUtil.parseInt(maxVolumeSizeInGbString, + Integer.parseInt(Config.MaxVolumeSize.getDefaultValue())); - String defaultPageSizeString = _configDao.getValue("default.page.size"); - _defaultPageSize = NumbersUtil.parseLong(defaultPageSizeString, 500L); + String defaultPageSizeString = _configDao.getValue(Config.DefaultPageSize.key()); + _defaultPageSize = NumbersUtil.parseLong(defaultPageSizeString, + Long.parseLong(Config.DefaultPageSize.getDefaultValue())); populateConfigValuesForValidationSet(); return true; From bf15eb0c7fbb8716a9df1fb1cd2209a68eed3d20 Mon Sep 17 00:00:00 2001 From: Kelven Yang Date: Tue, 12 Mar 2013 17:26:10 -0700 Subject: [PATCH 26/34] CLOUDSTACK-1655: A regression in ConsoleProxy caused by deprecating old URL form --- .../com/cloud/consoleproxy/ConsoleProxyHttpHandlerHelper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/console-proxy/server/src/com/cloud/consoleproxy/ConsoleProxyHttpHandlerHelper.java b/services/console-proxy/server/src/com/cloud/consoleproxy/ConsoleProxyHttpHandlerHelper.java index 7f79bae657f..6815b0d43bc 100644 --- a/services/console-proxy/server/src/com/cloud/consoleproxy/ConsoleProxyHttpHandlerHelper.java +++ b/services/console-proxy/server/src/com/cloud/consoleproxy/ConsoleProxyHttpHandlerHelper.java @@ -50,9 +50,10 @@ public class ConsoleProxyHttpHandlerHelper { ConsoleProxyPasswordBasedEncryptor encryptor = new ConsoleProxyPasswordBasedEncryptor( ConsoleProxy.getEncryptorPassword()); + ConsoleProxyClientParam param = encryptor.decryptObject(ConsoleProxyClientParam.class, map.get("token")); + // make sure we get information from token only map.clear(); - ConsoleProxyClientParam param = encryptor.decryptObject(ConsoleProxyClientParam.class, map.get("token")); if(param != null) { if(param.getClientHostAddress() != null) map.put("host", param.getClientHostAddress()); From 630e75596ed6a4cf769b24900d383a05ebb25cdc Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Tue, 12 Mar 2013 18:24:31 -0700 Subject: [PATCH 27/34] CLOUDSTACK-1653: Redundant router: Fix check_heartbeat.sh malfunctional due to delayed cron job The interval between keepalived.ts and keepalived.ts2 should be >= 60 seconds in normal condition, because every 10 seconds keepalived.ts would be updated, and at least every 60 seconds, keepalived.ts would be copy to keepalived.ts2. If the interval is less than 60 seconds, then keepalived process failed to update keepalived.ts every 10 seconds. Take some delay of updating into consideration, check_heartbeat.sh would use 30 seconds as a way to tell keepalived process is alive or not. --- .../config/root/redundant_router/check_heartbeat.sh.templ | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patches/systemvm/debian/config/root/redundant_router/check_heartbeat.sh.templ b/patches/systemvm/debian/config/root/redundant_router/check_heartbeat.sh.templ index 908c0d8f06f..7a980bdfb8c 100755 --- a/patches/systemvm/debian/config/root/redundant_router/check_heartbeat.sh.templ +++ b/patches/systemvm/debian/config/root/redundant_router/check_heartbeat.sh.templ @@ -22,7 +22,7 @@ then lasttime=$(cat [RROUTER_BIN_PATH]/keepalived.ts2) thistime=$(cat [RROUTER_BIN_PATH]/keepalived.ts) diff=$(($thistime - $lasttime)) - if [ $diff -gt 100 ] + if [ $diff -lt 30] then echo Keepalived process is dead! >> [RROUTER_LOG] service keepalived stop >> [RROUTER_LOG] 2>&1 From a49261c3b1f2d52d637f481e0bcff4b0d58d7b56 Mon Sep 17 00:00:00 2001 From: Jayapal Uradi Date: Wed, 6 Mar 2013 18:11:10 +0530 Subject: [PATCH 28/34] CLOUDSTACK-24: mipn feature for basiczone Signed-off-by: Abhinandan Prateek --- .../agent/api/SecurityGroupRulesCmd.java | 37 ++++ api/src/com/cloud/agent/api/to/NicTO.java | 11 + .../security/SecurityGroupService.java | 4 +- .../api/command/user/vm/AddIpToVmNicCmd.java | 29 ++- .../command/user/vm/RemoveIpFromVmNicCmd.java | 48 ++++- .../api/NetworkRulesVmSecondaryIpCommand.java | 71 +++++++ .../xen/resource/CitrixResourceBase.java | 32 ++- scripts/vm/hypervisor/xenserver/vmops | 199 +++++++++++++++++- .../cloud/hypervisor/HypervisorGuruBase.java | 12 +- .../src/com/cloud/network/NetworkManager.java | 5 +- .../com/cloud/network/NetworkManagerImpl.java | 29 ++- .../com/cloud/network/NetworkModelImpl.java | 2 +- .../com/cloud/network/NetworkServiceImpl.java | 94 +++++++-- .../com/cloud/network/dao/IPAddressDao.java | 2 + .../cloud/network/dao/IPAddressDaoImpl.java | 8 + .../cloud/network/guru/DirectNetworkGuru.java | 18 +- .../security/SecurityGroupManagerImpl.java | 93 +++++++- .../security/SecurityGroupManagerImpl2.java | 13 +- .../src/com/cloud/vm/UserVmManagerImpl.java | 7 - server/src/com/cloud/vm/dao/NicDao.java | 2 + server/src/com/cloud/vm/dao/NicDaoImpl.java | 9 + .../cloud/network/MockNetworkManagerImpl.java | 20 +- .../com/cloud/vpc/MockNetworkManagerImpl.java | 33 ++- 23 files changed, 704 insertions(+), 74 deletions(-) create mode 100644 core/src/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java diff --git a/api/src/com/cloud/agent/api/SecurityGroupRulesCmd.java b/api/src/com/cloud/agent/api/SecurityGroupRulesCmd.java index affad1f9b9d..4336b4c32b4 100644 --- a/api/src/com/cloud/agent/api/SecurityGroupRulesCmd.java +++ b/api/src/com/cloud/agent/api/SecurityGroupRulesCmd.java @@ -18,6 +18,7 @@ package com.cloud.agent.api; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.List; import java.util.zip.DeflaterOutputStream; import org.apache.commons.codec.binary.Base64; @@ -80,6 +81,7 @@ public class SecurityGroupRulesCmd extends Command { Long msId; IpPortAndProto [] ingressRuleSet; IpPortAndProto [] egressRuleSet; + private List secIps; public SecurityGroupRulesCmd() { super(); @@ -103,6 +105,23 @@ public class SecurityGroupRulesCmd extends Command { } + public SecurityGroupRulesCmd(String guestIp, String guestMac, String vmName, Long vmId, String signature, Long seqNum, IpPortAndProto[] ingressRuleSet, IpPortAndProto[] egressRuleSet, List secIps) { + super(); + this.guestIp = guestIp; + this.vmName = vmName; + this.ingressRuleSet = ingressRuleSet; + this.egressRuleSet = egressRuleSet; + this.guestMac = guestMac; + this.signature = signature; + this.seqNum = seqNum; + this.vmId = vmId; + if (signature == null) { + String stringified = stringifyRules(); + this.signature = DigestUtils.md5Hex(stringified); + } + this.secIps = secIps; + } + @Override public boolean executeInSequence() { return true; @@ -131,6 +150,10 @@ public class SecurityGroupRulesCmd extends Command { return guestIp; } + public List getSecIps() { + return secIps; + } + public String getVmName() { return vmName; @@ -165,6 +188,20 @@ public class SecurityGroupRulesCmd extends Command { } + public String getSecIpsString() { + StringBuilder sb = new StringBuilder(); + List ips = getSecIps(); + if (ips == null) { + return "0:"; + } else { + for (String ip : ips) { + sb.append(ip).append(":"); + } + } + return sb.toString(); + } + + public String stringifyCompressedRules() { StringBuilder ruleBuilder = new StringBuilder(); for (SecurityGroupRulesCmd.IpPortAndProto ipPandP : getIngressRuleSet()) { diff --git a/api/src/com/cloud/agent/api/to/NicTO.java b/api/src/com/cloud/agent/api/to/NicTO.java index aa2aa19cc19..ccebe115f97 100644 --- a/api/src/com/cloud/agent/api/to/NicTO.java +++ b/api/src/com/cloud/agent/api/to/NicTO.java @@ -16,12 +16,15 @@ // under the License. package com.cloud.agent.api.to; +import java.util.List; + public class NicTO extends NetworkTO { int deviceId; Integer networkRateMbps; Integer networkRateMulticastMbps; boolean defaultNic; String uuid; + List nicSecIps; public NicTO() { super(); @@ -69,4 +72,12 @@ public class NicTO extends NetworkTO { public String toString() { return new StringBuilder("[Nic:").append(type).append("-").append(ip).append("-").append(broadcastUri).append("]").toString(); } + + public void setNicSecIps(List secIps) { + this.nicSecIps = secIps; + } + + public List getNicSecIps() { + return nicSecIps; + } } diff --git a/api/src/com/cloud/network/security/SecurityGroupService.java b/api/src/com/cloud/network/security/SecurityGroupService.java index c6480323780..397de1ccb46 100644 --- a/api/src/com/cloud/network/security/SecurityGroupService.java +++ b/api/src/com/cloud/network/security/SecurityGroupService.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.api.command.user.securitygroup.CreateSecurityGroupC import org.apache.cloudstack.api.command.user.securitygroup.DeleteSecurityGroupCmd; import org.apache.cloudstack.api.command.user.securitygroup.RevokeSecurityGroupEgressCmd; import org.apache.cloudstack.api.command.user.securitygroup.RevokeSecurityGroupIngressCmd; +import org.apache.cloudstack.api.command.user.vm.AddIpToVmNicCmd; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; @@ -45,5 +46,6 @@ public interface SecurityGroupService { public List authorizeSecurityGroupIngress(AuthorizeSecurityGroupIngressCmd cmd); public List authorizeSecurityGroupEgress(AuthorizeSecurityGroupEgressCmd cmd); - + public boolean securityGroupRulesForVmSecIp(Long nicId, Long networkId, + String secondaryIp, boolean ruleAction); } diff --git a/api/src/org/apache/cloudstack/api/command/user/vm/AddIpToVmNicCmd.java b/api/src/org/apache/cloudstack/api/command/user/vm/AddIpToVmNicCmd.java index 0f992743f6d..cc34b7b2b44 100644 --- a/api/src/org/apache/cloudstack/api/command/user/vm/AddIpToVmNicCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/vm/AddIpToVmNicCmd.java @@ -28,6 +28,8 @@ import org.apache.cloudstack.api.response.NicResponse; import org.apache.cloudstack.api.response.NicSecondaryIpResponse; import com.cloud.async.AsyncJob; +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenter.NetworkType; import com.cloud.event.EventTypes; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientAddressCapacityException; @@ -83,6 +85,9 @@ public class AddIpToVmNicCmd extends BaseAsyncCmd { public Long getNetworkId() { Nic nic = _entityMgr.findById(Nic.class, nicId); + if (nic == null) { + throw new InvalidParameterValueException("Can't find network id for specified nic"); + } Long networkId = nic.getNetworkId(); return networkId; } @@ -98,6 +103,13 @@ public class AddIpToVmNicCmd extends BaseAsyncCmd { return null; } } + + public NetworkType getNetworkType() { + Network ntwk = _entityMgr.findById(Network.class, getNetworkId()); + DataCenter dc = _entityMgr.findById(DataCenter.class, ntwk.getDataCenterId()); + return dc.getNetworkType(); + } + @Override public long getEntityOwnerId() { Account caller = UserContext.current().getCaller(); @@ -134,7 +146,7 @@ public class AddIpToVmNicCmd extends BaseAsyncCmd { UserContext.current().setEventDetails("Nic Id: " + getNicId() ); String ip; - String SecondaryIp = null; + String secondaryIp = null; if ((ip = getIpaddress()) != null) { if (!NetUtils.isValidIp(ip)) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Invalid ip address " + ip); @@ -142,13 +154,22 @@ public class AddIpToVmNicCmd extends BaseAsyncCmd { } try { - SecondaryIp = _networkService.allocateSecondaryGuestIP(_accountService.getAccount(getEntityOwnerId()), getZoneId(), getNicId(), getNetworkId(), getIpaddress()); + secondaryIp = _networkService.allocateSecondaryGuestIP(_accountService.getAccount(getEntityOwnerId()), getZoneId(), getNicId(), getNetworkId(), getIpaddress()); } catch (InsufficientAddressCapacityException e) { throw new InvalidParameterValueException("Allocating guest ip for nic failed"); } - if (SecondaryIp != null) { - s_logger.info("Associated ip address to NIC : " + SecondaryIp); + if (secondaryIp != null) { + if (getNetworkType() == NetworkType.Basic) { + // add security group rules for the secondary ip addresses + boolean success = false; + success = _securityGroupService.securityGroupRulesForVmSecIp(getNicId(), getNetworkId(), secondaryIp, (boolean) true); + if (success == false) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to set security group rules for the secondary ip"); + } + } + + s_logger.info("Associated ip address to NIC : " + secondaryIp); NicSecondaryIpResponse response = new NicSecondaryIpResponse(); response = _responseGenerator.createSecondaryIPToNicResponse(ip, getNicId(), getNetworkId()); response.setResponseName(getCommandName()); diff --git a/api/src/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java b/api/src/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java index cb5e0855f64..d9ab7e03080 100644 --- a/api/src/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java @@ -27,10 +27,15 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.NicSecondaryIpResponse; import org.apache.cloudstack.api.response.SuccessResponse; import com.cloud.async.AsyncJob; +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenter.NetworkType; import com.cloud.event.EventTypes; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.network.Network; import com.cloud.user.Account; import com.cloud.user.UserContext; +import com.cloud.vm.Nic; +import com.cloud.vm.NicSecondaryIp; @APICommand(name = "removeIpFromNic", description="Assigns secondary IP to NIC.", responseObject=SuccessResponse.class) public class RemoveIpFromVmNicCmd extends BaseAsyncCmd { @@ -43,7 +48,7 @@ public class RemoveIpFromVmNicCmd extends BaseAsyncCmd { @Parameter(name=ApiConstants.ID, type=CommandType.UUID, required = true, entityType = NicSecondaryIpResponse.class, description="the ID of the secondary ip address to nic") - private long id; + private Long id; // unexposed parameter needed for events logging @Parameter(name=ApiConstants.ACCOUNT_ID, type=CommandType.UUID, expose=false) @@ -57,7 +62,7 @@ public class RemoveIpFromVmNicCmd extends BaseAsyncCmd { return "nic_secondary_ips"; } - public long getIpAddressId() { + public Long getIpAddressId() { return id; } @@ -80,6 +85,11 @@ public class RemoveIpFromVmNicCmd extends BaseAsyncCmd { return EventTypes.EVENT_NET_IP_ASSIGN; } + public NicSecondaryIp getIpEntry() { + NicSecondaryIp nicSecIp = _entityMgr.findById(NicSecondaryIp.class, getIpAddressId()); + return nicSecIp; + } + @Override public String getEventDescription() { return ("Disassociating ip address with id=" + id); @@ -98,9 +108,43 @@ public class RemoveIpFromVmNicCmd extends BaseAsyncCmd { return "addressinfo"; } + public Long getNetworkId() { + NicSecondaryIp nicSecIp = _entityMgr.findById(NicSecondaryIp.class, getIpAddressId()); + if (nicSecIp != null) { + Long networkId = nicSecIp.getNetworkId(); + return networkId; + } else { + return null; + } + } + + public NetworkType getNetworkType() { + Network ntwk = _entityMgr.findById(Network.class, getNetworkId()); + if (ntwk != null) { + DataCenter dc = _entityMgr.findById(DataCenter.class, ntwk.getDataCenterId()); + return dc.getNetworkType(); + } + return null; + } + @Override public void execute() throws InvalidParameterValueException { UserContext.current().setEventDetails("Ip Id: " + getIpAddressId()); + NicSecondaryIp nicSecIp = getIpEntry(); + + if (nicSecIp == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Invalid IP id is passed"); + } + + if (getNetworkType() == NetworkType.Basic) { + //remove the security group rules for this secondary ip + boolean success = false; + success = _securityGroupService.securityGroupRulesForVmSecIp(nicSecIp.getNicId(), nicSecIp.getNetworkId(),nicSecIp.getIp4Address(), false); + if (success == false) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to set security group rules for the secondary ip"); + } + } + boolean result = _networkService.releaseSecondaryIpFromNic(getIpAddressId()); if (result) { SuccessResponse response = new SuccessResponse(getCommandName()); diff --git a/core/src/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java b/core/src/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java new file mode 100644 index 00000000000..ce4080878a1 --- /dev/null +++ b/core/src/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +import com.cloud.vm.VirtualMachine; + +public class NetworkRulesVmSecondaryIpCommand extends Command { + + private String vmName; + private VirtualMachine.Type type; + private String vmSecIp; + private String vmMac; + private String action; + + public NetworkRulesVmSecondaryIpCommand(String vmName, VirtualMachine.Type type) { + this.vmName = vmName; + this.type = type; + } + + + public NetworkRulesVmSecondaryIpCommand(String vmName, String vmMac, + String secondaryIp, boolean action) { + this.vmName = vmName; + this.vmMac = vmMac; + this.vmSecIp = secondaryIp; + if (action) { + this.action = "-A"; + } else { + this.action = "-D"; + } + } + + public String getVmName() { + return vmName; + } + + public VirtualMachine.Type getType() { + return type; + } + + public String getVmSecIp() { + return vmSecIp; + } + + public String getVmMac() { + return vmMac; + } + + public String getAction() { + return action; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java index 886fc6f04fa..f0cf2f05bf0 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java @@ -118,6 +118,7 @@ import com.cloud.agent.api.ModifySshKeysCommand; import com.cloud.agent.api.ModifyStoragePoolAnswer; import com.cloud.agent.api.ModifyStoragePoolCommand; import com.cloud.agent.api.NetworkRulesSystemVmCommand; +import com.cloud.agent.api.NetworkRulesVmSecondaryIpCommand; import com.cloud.agent.api.PingCommand; import com.cloud.agent.api.PingRoutingCommand; import com.cloud.agent.api.PingRoutingWithNwGroupsCommand; @@ -597,6 +598,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe return execute((DeleteVMSnapshotCommand)cmd); } else if (clazz == RevertToVMSnapshotCommand.class) { return execute((RevertToVMSnapshotCommand)cmd); + } else if (clazz == NetworkRulesVmSecondaryIpCommand.class) { + return execute((NetworkRulesVmSecondaryIpCommand)cmd); } else { return Answer.createUnsupportedCommandAnswer(cmd); } @@ -1468,7 +1471,18 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe for (NicTO nic : nics) { if ( nic.isSecurityGroupEnabled() || nic.getIsolationUri() != null && nic.getIsolationUri().getScheme().equalsIgnoreCase(IsolationType.Ec2.toString())) { - result = callHostPlugin(conn, "vmops", "default_network_rules", "vmName", vmName, "vmIP", nic.getIp(), "vmMAC", nic.getMac(), "vmID", Long.toString(vmSpec.getId())); + List nicSecIps = nic.getNicSecIps(); + String secIpsStr; + StringBuilder sb = new StringBuilder(); + if (nicSecIps != null) { + for (String ip : nicSecIps) { + sb.append(ip).append(":"); + } + secIpsStr = sb.toString(); + } else { + secIpsStr = "0:"; + } + result = callHostPlugin(conn, "vmops", "default_network_rules", "vmName", vmName, "vmIP", nic.getIp(), "vmMAC", nic.getMac(), "vmID", Long.toString(vmSpec.getId()), "secIps", secIpsStr); if (result == null || result.isEmpty() || !Boolean.parseBoolean(result)) { s_logger.warn("Failed to program default network rules for " + vmName+" on nic with ip:"+nic.getIp()+" mac:"+nic.getMac()); @@ -5454,7 +5468,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe "signature", cmd.getSignature(), "seqno", Long.toString(cmd.getSeqNum()), "deflated", "true", - "rules", cmd.compressStringifiedRules()); + "rules", cmd.compressStringifiedRules(), + "secIps", cmd.getSecIpsString()); if (result == null || result.isEmpty() || !Boolean.parseBoolean(result)) { s_logger.warn("Failed to program network rules for vm " + cmd.getVmName()); @@ -7506,6 +7521,19 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe return new Answer(cmd, success, ""); } + private Answer execute(NetworkRulesVmSecondaryIpCommand cmd) { + boolean success = true; + Connection conn = getConnection(); + + String result = callHostPlugin(conn, "vmops", "network_rules_vmSecondaryIp", "vmName", cmd.getVmName(), "vmMac", cmd.getVmMac(), "vmSecIp", cmd.getVmSecIp(), "action", + cmd.getAction()); + if (result == null || result.isEmpty() || !Boolean.parseBoolean(result)) { + success = false; + } + + return new Answer(cmd, success, ""); + } + protected SetFirewallRulesAnswer execute(SetFirewallRulesCommand cmd) { String[] results = new String[cmd.getRules().length]; String callResult; diff --git a/scripts/vm/hypervisor/xenserver/vmops b/scripts/vm/hypervisor/xenserver/vmops index db6f6d63ac9..31c9a59663e 100755 --- a/scripts/vm/hypervisor/xenserver/vmops +++ b/scripts/vm/hypervisor/xenserver/vmops @@ -610,6 +610,7 @@ def destroy_network_rules_for_vm(session, args): util.SMlog("Ignoring failure to delete egress chain " + vmchain_egress) remove_rule_log_for_vm(vm_name) + remove_secip_log_for_vm(vm_name) if 1 in [ vm_name.startswith(c) for c in ['r-', 's-', 'v-', 'l-'] ]: return 'true' @@ -749,6 +750,43 @@ def default_arp_antispoof(vm_chain, vifs, vm_ip, vm_mac): return 'true' + +@echo +def network_rules_vmSecondaryIp(session, args): + vm_name = args.pop('vmName') + vm_mac = args.pop('vmMac') + ip_secondary = args.pop('vmSecIp') + action = args.pop('action') + util.SMlog("vmMac = "+ vm_mac) + util.SMlog("vmName = "+ vm_name) + #action = "-A" + util.SMlog("action = "+ action) + try: + vm = session.xenapi.VM.get_by_name_label(vm_name) + if len(vm) != 1: + return 'false' + vm_rec = session.xenapi.VM.get_record(vm[0]) + vm_vifs = vm_rec.get('VIFs') + vifnums = [session.xenapi.VIF.get_record(vif).get('device') for vif in vm_vifs] + domid = vm_rec.get('domid') + except: + util.SMlog("### Failed to get domid or vif list for vm ##" + vm_name) + return 'false' + + if domid == '-1': + util.SMlog("### Failed to get domid for vm (-1): " + vm_name) + return 'false' + + vifs = ["vif" + domid + "." + v for v in vifnums] + #vm_name = '-'.join(vm_name.split('-')[:-1]) + vmchain = chain_name(vm_name) + add_to_ipset(vmchain, [ip_secondary], action) + + #add arptables rules for the secondary ip + arp_rules_vmip(vmchain, vifs, [ip_secondary], vm_mac, action) + + return 'true' + @echo def default_network_rules_systemvm(session, args): vm_name = args.pop('vmName') @@ -798,6 +836,55 @@ def default_network_rules_systemvm(session, args): util.SMlog("Failed to log default network rules for systemvm, ignoring") return 'true' +@echo +def create_ipset_forvm (ipsetname): + result = True + try: + util.SMlog("Creating ipset chain .... " + ipsetname) + util.pread2(['ipset', '-F', ipsetname]) + util.pread2(['ipset', '-X', ipsetname]) + util.pread2(['ipset', '-N', ipsetname, 'iphash']) + except: + util.SMlog("ipset chain not exists creating.... " + ipsetname) + util.pread2(['ipset', '-N', ipsetname, 'iphash']) + + return result + +@echo +def add_to_ipset(ipsetname, ips, action): + result = True + for ip in ips: + try: + util.SMlog("vm ip " + ip) + util.pread2(['ipset', action, ipsetname, ip]) + except: + util.SMlog("vm ip alreday in ip set" + ip) + continue + + return result + +@echo +def arp_rules_vmip (vm_chain, vifs, ips, vm_mac, action): + try: + if action == "-A": + action = "-I" + for vif in vifs: + for vm_ip in ips: + #accept any arp requests to this vm as long as the request is for this vm's ip + util.pread2(['arptables', action, vm_chain, '-o', vif, '--opcode', 'Request', '--destination-ip', vm_ip, '-j', 'ACCEPT']) + #accept any arp replies to this vm as long as the mac and ip matches + util.pread2(['arptables', action, vm_chain, '-o', vif, '--opcode', 'Reply', '--destination-mac', vm_mac, '--destination-ip', vm_ip, '-j', 'ACCEPT']) + #accept arp replies into the bridge as long as the source mac and ips match the vm + util.pread2(['arptables', action, vm_chain, '-i', vif, '--opcode', 'Reply', '--source-mac', vm_mac, '--source-ip', vm_ip, '-j', 'ACCEPT']) + #accept any arp requests from this vm. In the future this can be restricted to deny attacks on hosts + #also important to restrict source ip and src mac in these requests as they can be used to update arp tables on destination + util.pread2(['arptables', action, vm_chain, '-i', vif, '--opcode', 'Request', '--source-mac', vm_mac, '--source-ip', vm_ip, '-j', 'RETURN']) + except: + util.SMlog("Failed to program arptables rules for ip") + return 'false' + + return 'true' + @echo def default_network_rules(session, args): @@ -805,6 +892,8 @@ def default_network_rules(session, args): vm_ip = args.pop('vmIP') vm_id = args.pop('vmID') vm_mac = args.pop('vmMAC') + sec_ips = args.pop("secIps") + action = "-A" try: vm = session.xenapi.VM.get_by_name_label(vm_name) @@ -854,6 +943,32 @@ def default_network_rules(session, args): except: util.pread2(['iptables', '-F', vmchain_default]) + vmipset = vm_name + #create ipset and add vm ips to that ip set + if create_ipset_forvm(vmipset) == False: + util.SMlog(" failed to create ipset for rule " + str(tokens)) + return 'false' + + #add primary nic ip to ipset + if add_to_ipset(vmipset, [vm_ip], action ) == False: + util.SMlog(" failed to add vm " + vm_ip + " ip to set ") + return 'false' + + #add secodnary nic ips to ipset + secIpSet = "1" + ips = sec_ips.split(':') + ips.pop() + if ips[0] == "0": + secIpSet = "0"; + + if secIpSet == "1": + util.SMlog("Adding ipset for secondary ips") + add_to_ipset(vmipset, ips, action) + if write_secip_log_for_vm(vm_name, sec_ips, vm_id) == False: + util.SMlog("Failed to log default network rules, ignoring") + + keyword = '--' + get_ipset_keyword() + try: for v in vifs: util.pread2(['iptables', '-A', 'BRIDGE-FIREWALL', '-m', 'physdev', '--physdev-is-bridged', '--physdev-out', v, '-j', vmchain_default]) @@ -861,16 +976,22 @@ def default_network_rules(session, args): #don't let vm spoof its ip address for v in vifs: - util.pread2(['iptables', '-A', vmchain_default, '-m', 'physdev', '--physdev-is-bridged', '--physdev-in', v, '--source', vm_ip,'-p', 'udp', '--dport', '53', '-j', 'RETURN']) - util.pread2(['iptables', '-A', vmchain_default, '-m', 'physdev', '--physdev-is-bridged', '--physdev-in', v, '--source', '!', vm_ip, '-j', 'DROP']) - util.pread2(['iptables', '-A', vmchain_default, '-m', 'physdev', '--physdev-is-bridged', '--physdev-out', v, '--destination', '!', vm_ip, '-j', 'DROP']) - util.pread2(['iptables', '-A', vmchain_default, '-m', 'physdev', '--physdev-is-bridged', '--physdev-in', v, '--source', vm_ip, '-j', vmchain_egress]) + #util.pread2(['iptables', '-A', vmchain_default, '-m', 'physdev', '--physdev-is-bridged', '--physdev-in', v, '--source', vm_ip,'-p', 'udp', '--dport', '53', '-j', 'RETURN']) + util.pread2(['iptables', '-A', vmchain_default, '-m', 'physdev', '--physdev-is-bridged', '--physdev-in', v, '-m', 'set', keyword, vmipset, 'src', '-p', 'udp', '--dport', '53', '-j', 'RETURN']) + util.pread2(['iptables', '-A', vmchain_default, '-m', 'physdev', '--physdev-is-bridged', '--physdev-in', v, '-m', 'set', '!', keyword, vmipset, 'src', '-j', 'DROP']) + util.pread2(['iptables', '-A', vmchain_default, '-m', 'physdev', '--physdev-is-bridged', '--physdev-out', v, '-m', 'set', '!', keyword, vmipset, 'dst', '-j', 'DROP']) + util.pread2(['iptables', '-A', vmchain_default, '-m', 'physdev', '--physdev-is-bridged', '--physdev-in', v, '-m', 'set', keyword, vmipset, 'src', '-j', vmchain_egress]) util.pread2(['iptables', '-A', vmchain_default, '-m', 'physdev', '--physdev-is-bridged', '--physdev-out', v, '-j', vmchain]) except: util.SMlog("Failed to program default rules for vm " + vm_name) return 'false' default_arp_antispoof(vmchain, vifs, vm_ip, vm_mac) + #add default arp rules for secondary ips; + if secIpSet == "1": + util.SMlog("Adding arp rules for sec ip") + arp_rules_vmip(vmchain, vifs, ips, vm_mac, action) + default_ebtables_antispoof_rules(vmchain, vifs, vm_ip, vm_mac) if write_rule_log_for_vm(vm_name, vm_id, vm_ip, domid, '_initial_', '-1', vm_mac) == False: @@ -994,10 +1115,45 @@ def network_rules_for_rebooted_vm(session, vmName): destroy_arptables_rules(vmchain) [vm_ip, vm_mac] = get_vm_mac_ip_from_log(vmchain) default_arp_antispoof(vmchain, vifs, vm_ip, vm_mac) + + #check wether the vm has secondary ips + if is_secondary_ips_set(vm_name) == True: + vmips = get_vm_sec_ips(vm_name) + #add arp rules for the secondaryp ip + for ip in vmips: + arp_rules_vmip(vmchain, vifs, [ip], vm_mac, "-A") + + default_ebtables_antispoof_rules(vmchain, vifs, vm_ip, vm_mac) rewrite_rule_log_for_vm(vm_name, curr_domid) return True + + +@echo +def get_vm_sec_ips(vm_name): + logfilename = "/var/run/cloud/" + vm_name +".ip" + + lines = (line.rstrip() for line in open(logfilename)) + for line in lines: + try: + [_vmName,_vmIP,_vmID] = line.split(',') + break + except ValueError,v: + [_vmName,_vmIP,_vmID] = line.split(',') + + _vmIPS = _vmIP.split(":")[:-1] + return _vmIPS + +@echo +def is_secondary_ips_set(vm_name): + logfilename = "/var/run/cloud/" + vm_name +".ip" + if not os.path.exists(logfilename): + return False + + return True + +@echo def rewrite_rule_log_for_vm(vm_name, new_domid): logfilename = "/var/run/cloud/" + vm_name +".log" if not os.path.exists(logfilename): @@ -1194,6 +1350,39 @@ def check_rule_log_for_vm(vmName, vmID, vmIP, domID, signature, seqno): return [reprogramDefault, reprogramChain, rewriteLog] +@echo +def write_secip_log_for_vm (vmName, secIps, vmId): + vm_name = vmName + logfilename = "/var/run/cloud/"+vm_name+".ip" + util.SMlog("Writing log to " + logfilename) + logf = open(logfilename, 'w') + output = ','.join([vmName, secIps, vmId]) + result = True + + try: + logf.write(output) + logf.write('\n') + except: + util.SMlog("Failed to write to rule log file " + logfilename) + result = False + + logf.close() + + return result + +@echo +def remove_secip_log_for_vm(vmName): + vm_name = vmName + logfilename = "/var/run/cloud/"+vm_name+".ip" + + result = True + try: + os.remove(logfilename) + except: + util.SMlog("Failed to delete rule log file " + logfilename) + result = False + + return result @echo def write_rule_log_for_vm(vmName, vmID, vmIP, domID, signature, seqno, vmMac='ff:ff:ff:ff:ff:ff'): @@ -1289,6 +1478,7 @@ def network_rules(session, args): vm_mac = args.get('vmMAC') signature = args.pop('signature') seqno = args.pop('seqno') + sec_ips = args.pop("secIps") deflated = 'false' if 'deflated' in args: deflated = args.pop('deflated') @@ -1469,6 +1659,7 @@ if __name__ == "__main__": "can_bridge_firewall":can_bridge_firewall, "default_network_rules":default_network_rules, "destroy_network_rules_for_vm":destroy_network_rules_for_vm, "default_network_rules_systemvm":default_network_rules_systemvm, + "network_rules_vmSecondaryIp":network_rules_vmSecondaryIp, "get_rule_logs_for_vms":get_rule_logs_for_vms, "setLinkLocalIP":setLinkLocalIP, "cleanup_rules":cleanup_rules, diff --git a/server/src/com/cloud/hypervisor/HypervisorGuruBase.java b/server/src/com/cloud/hypervisor/HypervisorGuruBase.java index efe93966fe9..d77796da300 100644 --- a/server/src/com/cloud/hypervisor/HypervisorGuruBase.java +++ b/server/src/com/cloud/hypervisor/HypervisorGuruBase.java @@ -34,6 +34,7 @@ import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.NicSecondaryIpDao; import com.cloud.vm.dao.VMInstanceDao; public abstract class HypervisorGuruBase extends AdapterBase implements HypervisorGuru { @@ -41,7 +42,8 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis @Inject VMTemplateDetailsDao _templateDetailsDao; @Inject NicDao _nicDao; @Inject VMInstanceDao _virtualMachineDao; - + @Inject NicSecondaryIpDao _nicSecIpDao; + protected HypervisorGuruBase() { super(); } @@ -68,6 +70,14 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis // Workaround to make sure the TO has the UUID we need for Niciri integration NicVO nicVO = _nicDao.findById(profile.getId()); to.setUuid(nicVO.getUuid()); + //check whether the this nic has secondary ip addresses set + //set nic secondary ip address in NicTO which are used for security group + // configuration. Use full when vm stop/start + List secIps = null; + if (nicVO.getSecondaryIp()) { + secIps = _nicSecIpDao.getSecondaryIpAddressesForNic(nicVO.getId()); + } + to.setNicSecIps(secIps); return to; } diff --git a/server/src/com/cloud/network/NetworkManager.java b/server/src/com/cloud/network/NetworkManager.java index 8b6bf9a7402..48e017edabd 100755 --- a/server/src/com/cloud/network/NetworkManager.java +++ b/server/src/com/cloud/network/NetworkManager.java @@ -21,6 +21,8 @@ import java.util.Map; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.Pod; import com.cloud.dc.Vlan.VlanType; import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeployDestination; @@ -339,8 +341,9 @@ public interface NetworkManager { public String allocateGuestIP(Account ipOwner, boolean isSystem, long zoneId, Long networkId, String requestedIp) throws InsufficientAddressCapacityException; - boolean removeVmSecondaryIps(long vmId); List listVmNics(Long vmId, Long nicId); + String allocatePublicIpForGuestNic(Long networkId, DataCenter dc, Pod pod, Account caller, String requestedIp) throws InsufficientAddressCapacityException; + boolean removeVmSecondaryIpsOfNic(long nicId); } diff --git a/server/src/com/cloud/network/NetworkManagerImpl.java b/server/src/com/cloud/network/NetworkManagerImpl.java index ba5ab5d378b..3220c9174eb 100755 --- a/server/src/com/cloud/network/NetworkManagerImpl.java +++ b/server/src/com/cloud/network/NetworkManagerImpl.java @@ -1765,12 +1765,8 @@ public class NetworkManagerImpl extends ManagerBase implements NetworkManager, L _nicDao.remove(nic.getId()); s_logger.debug("Removed nic id=" + nic.getId()); //remove the secondary ip addresses corresponding to to this nic - List secondaryIps = _nicSecondaryIpDao.listByNicId(nic.getId()); - if (secondaryIps != null) { - for (NicSecondaryIpVO ip : secondaryIps) { - _nicSecondaryIpDao.remove(ip.getId()); - } - s_logger.debug("Removed nic " + nic.getId() + " secondary ip addreses"); + if (!removeVmSecondaryIpsOfNic(nic.getId())) { + s_logger.debug("Removing nic " + nic.getId() + " secondary ip addreses failed"); } } @@ -2835,7 +2831,6 @@ public class NetworkManagerImpl extends ManagerBase implements NetworkManager, L _accountMgr.checkAccess(caller, null, false, network); - //return acquireGuestIpAddress(network, requestedIp); ipaddr = acquireGuestIpAddress(network, requestedIp); return ipaddr; } @@ -3654,11 +3649,11 @@ public class NetworkManagerImpl extends ManagerBase implements NetworkManager, L return nic.getSecondaryIp(); } - @Override - public boolean removeVmSecondaryIps(long vmId) { + @Override + public boolean removeVmSecondaryIpsOfNic(long nicId) { Transaction txn = Transaction.currentTxn(); txn.start(); - List ipList = _nicSecondaryIpDao.listByVmId(vmId); + List ipList = _nicSecondaryIpDao.listByNicId(nicId); if (ipList != null) { for (NicSecondaryIpVO ip: ipList) { _nicSecondaryIpDao.remove(ip.getId()); @@ -3669,4 +3664,16 @@ public class NetworkManagerImpl extends ManagerBase implements NetworkManager, L return true; } -} + @Override + public String allocatePublicIpForGuestNic(Long networkId, DataCenter dc, Pod pod,Account owner, + String requestedIp) throws InsufficientAddressCapacityException { + PublicIp ip = assignPublicIpAddress(dc.getId(), null, owner, VlanType.DirectAttached, networkId, requestedIp, false); + if (ip == null) { + s_logger.debug("There is no free public ip address"); + return null; + } + Ip ipAddr = ip.getAddress(); + return ipAddr.addr(); + } + + } diff --git a/server/src/com/cloud/network/NetworkModelImpl.java b/server/src/com/cloud/network/NetworkModelImpl.java index e83c0d28ad4..46790b3ebdc 100644 --- a/server/src/com/cloud/network/NetworkModelImpl.java +++ b/server/src/com/cloud/network/NetworkModelImpl.java @@ -742,7 +742,7 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel { @Override public Nic getNicInNetwork(long vmId, long networkId) { - return _nicDao.findByInstanceIdAndNetworkId(networkId, vmId); + return _nicDao.findByInstanceIdAndNetworkIdIncludingRemoved(networkId, vmId); } @Override diff --git a/server/src/com/cloud/network/NetworkServiceImpl.java b/server/src/com/cloud/network/NetworkServiceImpl.java index 821aa6db26f..ba55ff853b9 100755 --- a/server/src/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/com/cloud/network/NetworkServiceImpl.java @@ -517,8 +517,45 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { } catch (InsufficientAddressCapacityException e) { throw new InvalidParameterValueException("Allocating guest ip for nic failed"); } + } else if (dc.getNetworkType() == NetworkType.Basic) { + Account caller = UserContext.current().getCaller(); + long callerUserId = UserContext.current().getCallerUserId(); + _accountMgr.checkAccess(caller, AccessType.UseNetwork, false, network); + //handle the basic networks here + VirtualMachine vm = _userVmDao.findById(nicVO.getInstanceId()); + if (vm == null) { + throw new InvalidParameterValueException("There is no vm with the nic"); + } + VMInstanceVO vmi = (VMInstanceVO)vm; + Long podId = vmi.getPodIdToDeployIn(); + if (podId == null) { + throw new InvalidParameterValueException("vm pod id is null"); + } + Pod pod = _hostPodDao.findById(podId); + if (pod == null) { + throw new InvalidParameterValueException("vm pod is null"); + } + + try { + ipaddr = _networkMgr.allocatePublicIpForGuestNic(networkId, dc, pod, caller, requestedIp); + if (ipaddr == null) { + throw new InvalidParameterValueException("Allocating ip to guest nic " + nicId + " failed"); + } + } catch (InsufficientAddressCapacityException e) { + s_logger.error("Allocating ip to guest nic " + nicId + " failed"); + return null; + } + } else if (isSharedNetworkOfferingWithServices(network.getNetworkOfferingId()) && zone.getNetworkType() == NetworkType.Advanced) { + // if shared network in the advanced zone, then check the caller against the network for 'AccessType.UseNetwork' + Account caller = UserContext.current().getCaller(); + long callerUserId = UserContext.current().getCallerUserId(); + _accountMgr.checkAccess(caller, AccessType.UseNetwork, false, network); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Associate IP address called by the user " + callerUserId + " account " + ipOwner.getId()); + } } else { - throw new InvalidParameterValueException("AddIpToVMNic is not supported in this network..."); + s_logger.error("AddIpToVMNic is not supported in this network..."); + return null; } if (ipaddr != null) { @@ -549,18 +586,18 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { boolean success = false; // Verify input parameters - NicSecondaryIpVO ipVO= _nicSecondaryIpDao.findById(ipAddressId); - if (ipVO == null) { + NicSecondaryIpVO secIpVO= _nicSecondaryIpDao.findById(ipAddressId); + if (secIpVO == null) { throw new InvalidParameterValueException("Unable to find ip address by id"); } - Network network = _networksDao.findById(ipVO.getNetworkId()); + Network network = _networksDao.findById(secIpVO.getNetworkId()); // verify permissions _accountMgr.checkAccess(caller, null, true, network); - Long nicId = ipVO.getNicId(); - s_logger.debug("ip id and nic id" + ipAddressId + "..." + nicId); + Long nicId = secIpVO.getNicId(); + s_logger.debug("ip id and nic id " + ipAddressId + "..." + nicId); //check is this the last secondary ip for NIC List ipList = _nicSecondaryIpDao.listByNicId(nicId); boolean lastIp = false; @@ -568,20 +605,41 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService { // this is the last secondary ip to nic lastIp = true; } - //check PF or static NAT is configured on this ip address - String secondaryIp = ipVO.getIp4Address(); - List pfRuleList = _portForwardingDao.listByDestIpAddr(secondaryIp); - if (pfRuleList.size() != 0) { - s_logger.debug("VM nic IP " + secondaryIp + " is associated with the port forwarding rule"); - throw new InvalidParameterValueException("Can't remove the secondary ip " + secondaryIp + " is associate with the port forwarding rule"); + + DataCenter dc = _dcDao.findById(network.getDataCenterId()); + if (dc == null) { + throw new InvalidParameterValueException("Invalid zone Id is given"); } - //check if the secondary ip associated with any static nat rule - IPAddressVO publicIpVO = _ipAddressDao.findByVmIp(secondaryIp); - if (publicIpVO != null) { - s_logger.debug("VM nic IP " + secondaryIp + " is associated with the static NAT rule public IP address id " + publicIpVO.getId()); - throw new InvalidParameterValueException("Can' remove the ip " + secondaryIp + "is associate with static NAT rule public IP address id " + publicIpVO.getId()); + + s_logger.debug("Calling the ip allocation ..."); + if (dc.getNetworkType() == NetworkType.Advanced && network.getGuestType() == Network.GuestType.Isolated) { + //check PF or static NAT is configured on this ip address + String secondaryIp = secIpVO.getIp4Address(); + List pfRuleList = _portForwardingDao.listByDestIpAddr(secondaryIp); + if (pfRuleList.size() != 0) { + s_logger.debug("VM nic IP " + secondaryIp + " is associated with the port forwarding rule"); + throw new InvalidParameterValueException("Can't remove the secondary ip " + secondaryIp + " is associate with the port forwarding rule"); + } + //check if the secondary ip associated with any static nat rule + IPAddressVO publicIpVO = _ipAddressDao.findByVmIp(secondaryIp); + if (publicIpVO != null) { + s_logger.debug("VM nic IP " + secondaryIp + " is associated with the static NAT rule public IP address id " + publicIpVO.getId()); + throw new InvalidParameterValueException("Can' remove the ip " + secondaryIp + "is associate with static NAT rule public IP address id " + publicIpVO.getId()); + } + } else if (dc.getNetworkType() == NetworkType.Basic) { + IPAddressVO ip = _ipAddressDao.findByIpAndNetworkId(secIpVO.getNetworkId(), secIpVO.getIp4Address()); + if (ip != null) { + Transaction txn = Transaction.currentTxn(); + txn.start(); + _networkMgr.markIpAsUnavailable(ip.getId()); + _ipAddressDao.unassignIpAddress(ip.getId()); + txn.commit(); + } + } else if (isSharedNetworkOfferingWithServices(network.getNetworkOfferingId()) && dc.getNetworkType() == NetworkType.Advanced) { + throw new InvalidParameterValueException("Not supported for this network now"); } - success = removeNicSecondaryIP(ipVO, lastIp); + + success = removeNicSecondaryIP(secIpVO, lastIp); return success; } diff --git a/server/src/com/cloud/network/dao/IPAddressDao.java b/server/src/com/cloud/network/dao/IPAddressDao.java index 1052639ebf2..3d588fa9307 100755 --- a/server/src/com/cloud/network/dao/IPAddressDao.java +++ b/server/src/com/cloud/network/dao/IPAddressDao.java @@ -66,4 +66,6 @@ public interface IPAddressDao extends GenericDao { IPAddressVO findByVmIp(String vmIp); IPAddressVO findByAssociatedVmIdAndVmIp(long vmId, String vmIp); + + IPAddressVO findByIpAndNetworkId(long networkId, String ipAddress); } diff --git a/server/src/com/cloud/network/dao/IPAddressDaoImpl.java b/server/src/com/cloud/network/dao/IPAddressDaoImpl.java index b0eef4f3043..73f310fd628 100755 --- a/server/src/com/cloud/network/dao/IPAddressDaoImpl.java +++ b/server/src/com/cloud/network/dao/IPAddressDaoImpl.java @@ -184,6 +184,14 @@ public class IPAddressDaoImpl extends GenericDaoBase implemen return findOneBy(sc); } + @Override + public IPAddressVO findByIpAndNetworkId(long networkId, String ipAddress) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("network", networkId); + sc.setParameters("ipAddress", ipAddress); + return findOneBy(sc); + } + @Override public IPAddressVO findByIpAndDcId(long dcId, String ipAddress) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/server/src/com/cloud/network/guru/DirectNetworkGuru.java b/server/src/com/cloud/network/guru/DirectNetworkGuru.java index 46a525e214a..8707cfd418c 100755 --- a/server/src/com/cloud/network/guru/DirectNetworkGuru.java +++ b/server/src/com/cloud/network/guru/DirectNetworkGuru.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.network.guru; +import java.util.List; + import javax.ejb.Local; import javax.inject.Inject; @@ -54,7 +56,9 @@ import com.cloud.utils.component.AdapterBase; import com.cloud.utils.db.DB; import com.cloud.utils.db.Transaction; import com.cloud.vm.Nic.ReservationStrategy; +import com.cloud.vm.dao.NicSecondaryIpDao; import com.cloud.vm.NicProfile; +import com.cloud.vm.NicSecondaryIp; import com.cloud.vm.ReservationContext; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; @@ -79,7 +83,9 @@ public class DirectNetworkGuru extends AdapterBase implements NetworkGuru { UserIpv6AddressDao _ipv6Dao; @Inject Ipv6AddressManager _ipv6Mgr; - + @Inject + NicSecondaryIpDao _nicSecondaryIpDao; + private static final TrafficType[] _trafficTypes = {TrafficType.Guest}; @Override @@ -230,6 +236,16 @@ public class DirectNetworkGuru extends AdapterBase implements NetworkGuru { txn.start(); _networkMgr.markIpAsUnavailable(ip.getId()); _ipAddressDao.unassignIpAddress(ip.getId()); + //unassign nic secondary ip address + s_logger.debug("remove nic " + nic.getId() + " secondary ip "); + List nicSecIps = null; + nicSecIps = _nicSecondaryIpDao.getSecondaryIpAddressesForNic(nic.getId()); + for (String secIp: nicSecIps) { + IPAddressVO pubIp = _ipAddressDao.findByIpAndSourceNetworkId(nic.getNetworkId(), secIp); + _networkMgr.markIpAsUnavailable(pubIp.getId()); + _ipAddressDao.unassignIpAddress(pubIp.getId()); + } + txn.commit(); } } diff --git a/server/src/com/cloud/network/security/SecurityGroupManagerImpl.java b/server/src/com/cloud/network/security/SecurityGroupManagerImpl.java index eafe88e36a4..1c189c44688 100755 --- a/server/src/com/cloud/network/security/SecurityGroupManagerImpl.java +++ b/server/src/com/cloud/network/security/SecurityGroupManagerImpl.java @@ -46,8 +46,10 @@ import org.apache.cloudstack.api.command.user.securitygroup.RevokeSecurityGroupI import org.apache.commons.codec.digest.DigestUtils; import org.apache.log4j.Logger; +import com.amazonaws.services.identitymanagement.model.User; import com.cloud.agent.AgentManager; import com.cloud.agent.api.NetworkRulesSystemVmCommand; +import com.cloud.agent.api.NetworkRulesVmSecondaryIpCommand; import com.cloud.agent.api.SecurityGroupRulesCmd; import com.cloud.agent.api.SecurityGroupRulesCmd.IpPortAndProto; import com.cloud.agent.manager.Commands; @@ -67,12 +69,6 @@ import com.cloud.network.NetworkManager; import com.cloud.network.NetworkModel; import com.cloud.network.security.SecurityGroupWork.Step; import com.cloud.network.security.SecurityRule.SecurityRuleType; -import com.cloud.network.security.dao.SecurityGroupDao; -import com.cloud.network.security.dao.SecurityGroupRuleDao; -import com.cloud.network.security.dao.SecurityGroupRulesDao; -import com.cloud.network.security.dao.SecurityGroupVMMapDao; -import com.cloud.network.security.dao.SecurityGroupWorkDao; -import com.cloud.network.security.dao.VmRulesetLogDao; import com.cloud.network.security.dao.*; import com.cloud.projects.ProjectManager; import com.cloud.tags.dao.ResourceTagDao; @@ -97,6 +93,8 @@ import com.cloud.utils.net.NetUtils; import com.cloud.vm.*; import com.cloud.vm.VirtualMachine.Event; import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.NicSecondaryIpDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; import edu.emory.mathcs.backport.java.util.Collections; @@ -149,6 +147,10 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro ProjectManager _projectMgr; @Inject ResourceTagDao _resourceTagDao; + @Inject + NicDao _nicDao; + @Inject + NicSecondaryIpDao _nicSecIpDao; ScheduledExecutorService _executorPool; ScheduledExecutorService _cleanupExecutor; @@ -489,7 +491,7 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro return affectedVms; } - protected SecurityGroupRulesCmd generateRulesetCmd(String vmName, String guestIp, String guestMac, Long vmId, String signature, long seqnum, Map> ingressRules, Map> egressRules) { + protected SecurityGroupRulesCmd generateRulesetCmd(String vmName, String guestIp, String guestMac, Long vmId, String signature, long seqnum, Map> ingressRules, Map> egressRules, List secIps) { List ingressResult = new ArrayList(); List egressResult = new ArrayList(); for (PortAndProto pAp : ingressRules.keySet()) { @@ -506,7 +508,7 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro egressResult.add(ipPortAndProto); } } - return new SecurityGroupRulesCmd(guestIp, guestMac, vmName, vmId, signature, seqnum, ingressResult.toArray(new IpPortAndProto[ingressResult.size()]), egressResult.toArray(new IpPortAndProto[egressResult.size()])); + return new SecurityGroupRulesCmd(guestIp, guestMac, vmName, vmId, signature, seqnum, ingressResult.toArray(new IpPortAndProto[ingressResult.size()]), egressResult.toArray(new IpPortAndProto[egressResult.size()]), secIps); } protected void handleVmStopped(VMInstanceVO vm) { @@ -947,8 +949,19 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro Map> egressRules = generateRulesForVM(userVmId, SecurityRuleType.EgressRule); agentId = vm.getHostId(); if (agentId != null) { + // get nic secondary ip address + String privateIp = vm.getPrivateIpAddress(); + NicVO nic = _nicDao.findByIp4AddressAndVmId(privateIp, vm.getId()); + List nicSecIps = null; + if (nic != null) { + if (nic.getSecondaryIp()) { + //get secondary ips of the vm + long networkId = nic.getNetworkId(); + nicSecIps = _nicSecIpDao.getSecondaryIpAddressesForNic(nic.getId()); + } + } SecurityGroupRulesCmd cmd = generateRulesetCmd( vm.getInstanceName(), vm.getPrivateIpAddress(), vm.getPrivateMacAddress(), vm.getId(), generateRulesetSignature(ingressRules, egressRules), seqnum, - ingressRules, egressRules); + ingressRules, egressRules, nicSecIps); Commands cmds = new Commands(cmd); try { _agentMgr.send(agentId, cmds, _answerListener); @@ -1272,4 +1285,66 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro return true; } } + + @Override + public boolean securityGroupRulesForVmSecIp(Long nicId, Long networkId, + String secondaryIp, boolean ruleAction) { + + String vmMac = null; + String vmName = null; + + if (secondaryIp == null || nicId == null || networkId == null) { + throw new InvalidParameterValueException("Vm nicId or networkId or secondaryIp can't be null"); + } + + NicVO nic = _nicDao.findById(nicId); + Long vmId = nic.getInstanceId(); + + // Validate parameters + List vmSgGrps = getSecurityGroupsForVm(vmId); + if (vmSgGrps == null) { + s_logger.debug("Vm is not in any Security group "); + return true; + } + + Account caller = UserContext.current().getCaller(); + + for (SecurityGroupVO securityGroup: vmSgGrps) { + Account owner = _accountMgr.getAccount(securityGroup.getAccountId()); + if (owner == null) { + throw new InvalidParameterValueException("Unable to find security group owner by id=" + securityGroup.getAccountId()); + } + // Verify permissions + _accountMgr.checkAccess(caller, null, true, securityGroup); + } + + UserVm vm = _userVMDao.findById(vmId); + if (vm.getType() != VirtualMachine.Type.User) { + throw new InvalidParameterValueException("Can't configure the SG ipset, arprules rules for the non user vm"); + } + + if (vm != null) { + vmMac = vm.getPrivateMacAddress(); + vmName = vm.getInstanceName(); + if (vmMac == null || vmName == null) { + throw new InvalidParameterValueException("vm name or vm mac can't be null"); + } + } + + //create command for the to add ip in ipset and arptables rules + NetworkRulesVmSecondaryIpCommand cmd = new NetworkRulesVmSecondaryIpCommand(vmName, vmMac, secondaryIp, ruleAction); + s_logger.debug("Asking agent to configure rules for vm secondary ip"); + Commands cmds = null; + + cmds = new Commands(cmd); + try { + _agentMgr.send(vm.getHostId(), cmds); + } catch (AgentUnavailableException e) { + s_logger.debug(e.toString()); + } catch (OperationTimedoutException e) { + s_logger.debug(e.toString()); + } + + return true; + } } diff --git a/server/src/com/cloud/network/security/SecurityGroupManagerImpl2.java b/server/src/com/cloud/network/security/SecurityGroupManagerImpl2.java index a3a0fc300f9..a42881ec905 100644 --- a/server/src/com/cloud/network/security/SecurityGroupManagerImpl2.java +++ b/server/src/com/cloud/network/security/SecurityGroupManagerImpl2.java @@ -40,6 +40,7 @@ import com.cloud.utils.NumbersUtil; import com.cloud.utils.Profiler; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.mgmt.JmxUtil; +import com.cloud.vm.NicVO; import com.cloud.vm.VirtualMachine.State; import com.cloud.network.security.SecurityRule.SecurityRuleType; @@ -169,9 +170,19 @@ public class SecurityGroupManagerImpl2 extends SecurityGroupManagerImpl{ Map> egressRules = generateRulesForVM(userVmId, SecurityRuleType.EgressRule); Long agentId = vm.getHostId(); if (agentId != null) { + String privateIp = vm.getPrivateIpAddress(); + NicVO nic = _nicDao.findByIp4AddressAndVmId(privateIp, vm.getId()); + List nicSecIps = null; + if (nic != null) { + if (nic.getSecondaryIp()) { + //get secondary ips of the vm + long networkId = nic.getNetworkId(); + nicSecIps = _nicSecIpDao.getSecondaryIpAddressesForNic(nic.getId()); + } + } SecurityGroupRulesCmd cmd = generateRulesetCmd(vm.getInstanceName(), vm.getPrivateIpAddress(), vm.getPrivateMacAddress(), vm.getId(), null, - work.getLogsequenceNumber(), ingressRules, egressRules); + work.getLogsequenceNumber(), ingressRules, egressRules, nicSecIps); cmd.setMsId(_serverId); if (s_logger.isDebugEnabled()) { s_logger.debug("SecurityGroupManager v2: sending ruleset update for vm " + vm.getInstanceName() + diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 6b2f762a74d..88086ced461 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -1360,13 +1360,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Use + " as a part of vm id=" + vmId + " expunge because resource is unavailable", e); } - //remove vm secondary ip addresses - if (_networkMgr.removeVmSecondaryIps(vmId)) { - s_logger.debug("Removed vm " + vmId + " secondary ip address of the VM Nics as a part of expunge process"); - } else { - success = false; - s_logger.warn("Fail to remove secondary ip address of vm " + vmId + " Nics as a part of expunge process"); - } return success; } diff --git a/server/src/com/cloud/vm/dao/NicDao.java b/server/src/com/cloud/vm/dao/NicDao.java index 794bacc6e8b..35d719131bb 100644 --- a/server/src/com/cloud/vm/dao/NicDao.java +++ b/server/src/com/cloud/vm/dao/NicDao.java @@ -60,4 +60,6 @@ public interface NicDao extends GenericDao { NicVO findByIp4AddressAndNetworkIdAndInstanceId(long networkId, long instanceId, String ip4Address); List listByVmIdAndNicId(Long vmId, Long nicId); + + NicVO findByIp4AddressAndVmId(String ip4Address, long instance); } diff --git a/server/src/com/cloud/vm/dao/NicDaoImpl.java b/server/src/com/cloud/vm/dao/NicDaoImpl.java index 44911740e96..b9ec72ee7c9 100644 --- a/server/src/com/cloud/vm/dao/NicDaoImpl.java +++ b/server/src/com/cloud/vm/dao/NicDaoImpl.java @@ -212,4 +212,13 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { sc.setParameters("nicid", nicId); return listBy(sc); } + + @Override + public NicVO findByIp4AddressAndVmId(String ip4Address, long instance) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("address", ip4Address); + sc.setParameters("instance", instance); + return findOneBy(sc); + } + } diff --git a/server/test/com/cloud/network/MockNetworkManagerImpl.java b/server/test/com/cloud/network/MockNetworkManagerImpl.java index 80043102648..eb43cce0b9e 100755 --- a/server/test/com/cloud/network/MockNetworkManagerImpl.java +++ b/server/test/com/cloud/network/MockNetworkManagerImpl.java @@ -32,6 +32,7 @@ import org.springframework.stereotype.Component; import org.apache.cloudstack.api.command.user.vm.ListNicsCmd; import com.cloud.dc.DataCenter; +import com.cloud.dc.Pod; import com.cloud.dc.Vlan.VlanType; import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeployDestination; @@ -854,11 +855,6 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkManage return null; } - @Override - public boolean removeVmSecondaryIps(long vmId) { - // TODO Auto-generated method stub - return false; - } @Override public List listVmNics(Long vmId, Long nicId) { @@ -871,4 +867,18 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkManage // TODO Auto-generated method stub return null; } + + @Override + public String allocatePublicIpForGuestNic(Long networkId, DataCenter dc, + Pod pod, Account caller, String requestedIp) + throws InsufficientAddressCapacityException { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean removeVmSecondaryIpsOfNic(long nicId) { + // TODO Auto-generated method stub + return false; + } } diff --git a/server/test/com/cloud/vpc/MockNetworkManagerImpl.java b/server/test/com/cloud/vpc/MockNetworkManagerImpl.java index 63ef8744be8..c798cdf7810 100644 --- a/server/test/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/test/com/cloud/vpc/MockNetworkManagerImpl.java @@ -34,6 +34,7 @@ import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import com.cloud.dc.DataCenter; +import com.cloud.dc.Pod; import com.cloud.dc.Vlan.VlanType; import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeployDestination; @@ -1365,12 +1366,6 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkManage - @Override - public boolean removeVmSecondaryIps(long vmId) { - // TODO Auto-generated method stub - return false; - } - @@ -1390,4 +1385,30 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkManage // TODO Auto-generated method stub return null; } + + + + + + @Override + public String allocatePublicIpForGuestNic(Long networkId, DataCenter dc, + Pod pod, Account caller, String requestedIp) + throws InsufficientAddressCapacityException { + // TODO Auto-generated method stub + return null; + } + + + + + + + + + + @Override + public boolean removeVmSecondaryIpsOfNic(long nicId) { + // TODO Auto-generated method stub + return false; + } } From 27b753a434de087a9cd8ec50bfa1deb2de7a7925 Mon Sep 17 00:00:00 2001 From: Deepti Dohare Date: Tue, 12 Mar 2013 12:13:26 +0530 Subject: [PATCH 29/34] CLOUDSTACK-1575 Failed to transit volume with uploadVolume Signed-off-by: Abhinandan Prateek --- .../org/apache/cloudstack/storage/volume/VolumeServiceImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index c019374d9b9..32e7d274f01 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -567,7 +567,6 @@ public class VolumeServiceImpl implements VolumeService { AsyncCallFuture future = new AsyncCallFuture(); VolumeObject vo = (VolumeObject) volume; - vo.stateTransit(Volume.Event.UploadRequested); CreateVolumeContext context = new CreateVolumeContext(null, vo, future); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); From 6715f98087541440f1737912f15d9e3408793834 Mon Sep 17 00:00:00 2001 From: Bharat Kumar Date: Fri, 8 Mar 2013 15:08:41 +0530 Subject: [PATCH 30/34] CLOUDSTACK-1551: Failed to list clusters due to NPE at createClusterResponse(ApiResponseHelper.java:837) when cluster is added automatically as a part of addHostCommand Signed-off-by: Abhinandan Prateek --- server/src/com/cloud/resource/ResourceManagerImpl.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index 47b1d55ddac..15d32e0640d 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -770,6 +770,13 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } } clusterId = cluster.getId(); + if (_clusterDetailsDao.findDetail(clusterId,"cpuOvercommitRatio") == null) { + ClusterDetailsVO cluster_cpu_detail = new ClusterDetailsVO(clusterId,"cpuOvercommitRatio","1"); + ClusterDetailsVO cluster_memory_detail = new ClusterDetailsVO(clusterId,"memoryOvercommitRatio","1"); + _clusterDetailsDao.persist(cluster_cpu_detail); + _clusterDetailsDao.persist(cluster_memory_detail); + } + } try { From 1f28e644a3e66bed8702fab5f09d6d319d8a8330 Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Tue, 12 Mar 2013 23:55:15 -0600 Subject: [PATCH 31/34] Summary: Remove unused createFileBasedStoragePool from LibvirtStorageAdaptor Signed-off-by: Marcus Sorensen 1363154115 -0600 --- .../kvm/storage/LibvirtStorageAdaptor.java | 52 ------------------- 1 file changed, 52 deletions(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index d350ef994fb..070c1327ba3 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -293,58 +293,6 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { return parser.parseStorageVolumeXML(volDefXML); } - public StoragePool createFileBasedStoragePool(Connect conn, - String localStoragePath, String uuid) { - if (!(_storageLayer.exists(localStoragePath) && _storageLayer - .isDirectory(localStoragePath))) { - return null; - } - - File path = new File(localStoragePath); - if (!(path.canWrite() && path.canRead() && path.canExecute())) { - return null; - } - - StoragePool pool = null; - - try { - pool = conn.storagePoolLookupByUUIDString(uuid); - } catch (LibvirtException e) { - - } - - if (pool == null) { - LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(poolType.DIR, - uuid, uuid, null, null, localStoragePath); - try { - pool = conn.storagePoolDefineXML(spd.toString(), 0); - pool.create(0); - } catch (LibvirtException e) { - if (pool != null) { - try { - pool.destroy(); - pool.undefine(); - } catch (LibvirtException e1) { - } - pool = null; - } - throw new CloudRuntimeException(e.toString()); - } - } - - try { - StoragePoolInfo spi = pool.getInfo(); - if (spi.state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { - pool.create(0); - } - - } catch (LibvirtException e) { - throw new CloudRuntimeException(e.toString()); - } - - return pool; - } - @Override public KVMStoragePool getStoragePool(String uuid) { StoragePool storage = null; From 0f8c4d0ac09d476b4650c92d26bfb5c2ff4fa5e6 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 13 Mar 2013 11:49:25 +0530 Subject: [PATCH 32/34] baremetal: Remove old templateadapter in cloud-server, fix spring xmls The BareMetalTemplateAdapter in cloud-server is old and was introduced by recent branch merges, Frank had separated and moved code to the baremetal plugin so we will use that. Signed-off-by: Rohit Yadav --- client/tomcatconf/componentContext.xml.in | 6 +- .../tomcatconf/nonossComponentContext.xml.in | 6 +- .../baremetal/BareMetalTemplateAdapter.java | 217 ------------------ 3 files changed, 2 insertions(+), 227 deletions(-) delete mode 100755 server/src/com/cloud/baremetal/BareMetalTemplateAdapter.java diff --git a/client/tomcatconf/componentContext.xml.in b/client/tomcatconf/componentContext.xml.in index 23566e204c0..7b64f49ee20 100644 --- a/client/tomcatconf/componentContext.xml.in +++ b/client/tomcatconf/componentContext.xml.in @@ -107,11 +107,7 @@ - - - - - +