From 50ee9810a8ff7f147f3355f0f176bbaae16e7c75 Mon Sep 17 00:00:00 2001 From: Nitin Mehta Date: Fri, 10 Oct 2014 16:28:48 -0700 Subject: [PATCH] CLOUDSTACK-6826: Improving the download url expiration where the expiration work would be handled by the ssvm that has the corresponding symlink created on it. In case it doesnt exist, then would be any one of the ssvm in the zone. Also when the ssvm is destroyed all the download urls are expired to be cleaned up in the next run by the new ssvm. (cherry picked from commit ce908373573757372a3a09e46f835c3e269f3d02) --- .../api/storage/EndPointSelector.java | 2 ++ .../src/com/cloud/host/dao/HostDao.java | 2 ++ .../src/com/cloud/host/dao/HostDaoImpl.java | 13 +++++++++ .../datastore/db/TemplateDataStoreDao.java | 2 ++ .../datastore/db/VolumeDataStoreDao.java | 2 ++ .../endpoint/DefaultEndPointSelector.java | 29 +++++++++++++++++++ .../image/db/TemplateDataStoreDaoImpl.java | 20 +++++++++++++ .../image/db/VolumeDataStoreDaoImpl.java | 21 ++++++++++++++ .../CloudStackImageStoreDriverImpl.java | 7 +++-- .../SecondaryStorageManagerImpl.java | 7 ++++- 10 files changed, 102 insertions(+), 3 deletions(-) diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java index 4657316dd8a..4d6465b8067 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java @@ -36,4 +36,6 @@ public interface EndPointSelector { EndPoint select(Scope scope, Long storeId); EndPoint selectHypervisorHost(Scope scope); + + EndPoint select(DataStore store, String downloadUrl); } diff --git a/engine/schema/src/com/cloud/host/dao/HostDao.java b/engine/schema/src/com/cloud/host/dao/HostDao.java index 4d57bde0ea9..bd484825c81 100755 --- a/engine/schema/src/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/com/cloud/host/dao/HostDao.java @@ -88,4 +88,6 @@ public interface HostDao extends GenericDao, StateDao listByDataCenterId(long id); List listAllHosts(long zoneId); + + HostVO findByPublicIp(String publicIp); } diff --git a/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java index 58d472e9848..f1567ee51ea 100755 --- a/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java @@ -84,6 +84,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao protected SearchBuilder MsStatusSearch; protected SearchBuilder DcPrivateIpAddressSearch; protected SearchBuilder DcStorageIpAddressSearch; + protected SearchBuilder PublicIpAddressSearch; protected SearchBuilder GuidSearch; protected SearchBuilder DcSearch; @@ -207,6 +208,10 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao DcStorageIpAddressSearch.and("dc", DcStorageIpAddressSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ); DcStorageIpAddressSearch.done(); + PublicIpAddressSearch = createSearchBuilder(); + PublicIpAddressSearch.and("publicIpAddress", PublicIpAddressSearch.entity().getPublicIpAddress(), SearchCriteria.Op.EQ); + PublicIpAddressSearch.done(); + GuidSearch = createSearchBuilder(); GuidSearch.and("guid", GuidSearch.entity().getGuid(), SearchCriteria.Op.EQ); GuidSearch.done(); @@ -1068,6 +1073,14 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao return listBy(sc); } + @Override + public HostVO findByPublicIp(String publicIp) { + SearchCriteria sc = PublicIpAddressSearch.create(); + sc.setParameters("publicIpAddress", publicIp); + return findOneBy(sc); + } + + @Override public List findHypervisorHostInCluster(long clusterId) { SearchCriteria sc = TypeClusterStatusSearch.create(); diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java index 13c4d315da8..9f8bccd83bf 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java @@ -78,4 +78,6 @@ public interface TemplateDataStoreDao extends GenericDao listTemplateDownloadUrls(); void removeByTemplateStore(long templateId, long imageStoreId); + + void expireDnldUrlsForZone(Long dcId); } diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java index 0016185b9cb..625cc297274 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java @@ -44,4 +44,6 @@ public interface VolumeDataStoreDao extends GenericDao, void duplicateCacheRecordsOnRegionStore(long storeId); List listVolumeDownloadUrls(); + + void expireDnldUrlsForZone(Long dcId); } diff --git a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index c620c40d981..18c8991e0c2 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -18,6 +18,8 @@ */ package org.apache.cloudstack.storage.endpoint; +import java.net.URI; +import java.net.URISyntaxException; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -261,6 +263,33 @@ public class DefaultEndPointSelector implements EndPointSelector { return null; } + @Override + public EndPoint select(DataStore store, String downloadUrl){ + + HostVO host = null; + try { + URI uri = new URI(downloadUrl); + String scheme = uri.getScheme(); + String publicIp = uri.getHost(); + // If its https then public ip will be of the form xxx-xxx-xxx-xxx.mydomain.com + if(scheme.equalsIgnoreCase("https")){ + publicIp = publicIp.split("\\.")[0]; // We want xxx-xxx-xxx-xxx + publicIp = publicIp.replace("-","."); // We not want the IP - xxx.xxx.xxx.xxx + } + host = hostDao.findByPublicIp(publicIp); + if(host != null){ + return RemoteHostEndPoint.getHypervisorHostEndPoint(host); + } + + } catch (URISyntaxException e) { + s_logger.debug("Received URISyntaxException for url" +downloadUrl); + } + + // If ssvm doesnt exist then find any ssvm in the zone. + s_logger.debug("Coudn't find ssvm for url" +downloadUrl); + return findEndpointForImageStorage(store); + } + @Override public EndPoint select(DataStore store) { if (store.getRole() == DataStoreRole.Primary) { diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java index a5cf6f8e8cb..dad0b6dc9f6 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.storage.image.db; +import java.sql.PreparedStatement; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -64,6 +65,7 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase storeTemplateStateSearch; private SearchBuilder storeTemplateDownloadStatusSearch; private SearchBuilder downloadTemplateSearch; + private static final String EXPIRE_DOWNLOAD_URLS_FOR_ZONE = "update template_store_ref set download_url_created=? where store_id in (select id from image_store where data_center_id=?)"; @Inject private DataStoreManager _storeMgr; @@ -511,4 +513,22 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase cacheSearch; private SearchBuilder storeVolumeSearch; private SearchBuilder downloadVolumeSearch; + private static final String EXPIRE_DOWNLOAD_URLS_FOR_ZONE = "update volume_store_ref set download_url_created=? where store_id in (select id from image_store where data_center_id=?)"; + @Inject DataStoreManager storeMgr; @@ -267,4 +270,22 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase