From 6f5655cb03f159ff67cf570aeaa44179c4eeb43f Mon Sep 17 00:00:00 2001 From: Edison Su Date: Thu, 24 Jan 2013 18:28:22 -0800 Subject: [PATCH] generalize storage orchestration: DataObjectManager will handle all the object life cycle management --- .../storage/image/ImageServiceImpl.java | 12 +- .../DefaultImageDataStoreDriverImpl.java | 7 + .../image/store/HttpDataStoreImpl.java | 5 +- .../storage/image/store/TemplateObject.java | 53 ++- .../motion/DefaultImageMotionStrategy.java | 7 +- .../storage/test/ChildTestConfiguration.java | 23 +- .../storage/test/MockRpcCallBack.java | 8 +- .../storage/test/volumeServiceTest.java | 101 +++-- .../cloudstack/storage/command/CopyCmd.java | 23 +- .../storage/datastore/DataObjectManager.java | 37 ++ .../datastore/DataObjectManagerImpl.java | 402 ++++++++++++++++++ .../datastore/ObjectInDataStoreManager.java | 9 +- .../ObjectInDataStoreManagerImpl.java | 40 +- .../storage/db/ObjectInDataStoreDaoImpl.java | 22 +- .../storage/image/db/ImageDataDaoImpl.java | 74 +++- .../storage/image/db/ImageDataVO.java | 33 +- .../datastore/DefaultPrimaryDataStore.java | 22 +- .../DefaultPrimaryDataStoreDriverImpl.java | 36 +- .../volume/TemplateInstallStrategyImpl.java | 11 +- .../storage/volume/VolumeDataFactoryImpl.java | 3 +- .../storage/volume/VolumeObject.java | 22 +- .../storage/volume/VolumeServiceImpl.java | 70 +-- .../resource/XenServerStorageResource.java | 135 +++--- .../storage/dao/VMTemplateDetailsDaoImpl.java | 2 +- setup/db/4.1-new-db-schema.sql | 4 +- 25 files changed, 928 insertions(+), 233 deletions(-) create mode 100644 engine/storage/src/org/apache/cloudstack/storage/datastore/DataObjectManager.java create mode 100644 engine/storage/src/org/apache/cloudstack/storage/datastore/DataObjectManagerImpl.java diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/ImageServiceImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/ImageServiceImpl.java index b1e2a601d6b..82d0d71db9c 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/ImageServiceImpl.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/ImageServiceImpl.java @@ -77,7 +77,7 @@ public class ImageServiceImpl implements ImageService { ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject(template.getId(), template.getType(), store.getId(), store.getRole()); TemplateInfo templateOnStore = null; if (obj == null) { - templateOnStore = objectInDataStoreMgr.create(template, store); + templateOnStore = (TemplateInfo)objectInDataStoreMgr.create(template, store); obj = objectInDataStoreMgr.findObject(template.getId(), template.getType(), store.getId(), store.getRole()); } else { CommandResult result = new CommandResult(); @@ -87,7 +87,7 @@ public class ImageServiceImpl implements ImageService { } try { - objectInDataStoreMgr.update(templateOnStore, Event.CreateOnlyRequested); + objectInDataStoreMgr.update(obj, Event.CreateOnlyRequested); } catch (NoTransitionException e) { s_logger.debug("failed to transit", e); CommandResult result = new CommandResult(); @@ -125,8 +125,7 @@ public class ImageServiceImpl implements ImageService { future.complete(result); return null; } - - ObjectInDataStoreVO obj = context.obj; + ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject(templateOnStore.getId(), templateOnStore.getType(), templateOnStore.getDataStore().getId(), templateOnStore.getDataStore().getRole()); obj.setInstallPath(callbackResult.getPath()); if (callbackResult.getSize() != null) { @@ -134,15 +133,16 @@ public class ImageServiceImpl implements ImageService { } try { - objectInDataStoreMgr.update(templateOnStore, Event.OperationSuccessed); + objectInDataStoreMgr.update(obj, Event.OperationSuccessed); } catch (NoTransitionException e) { s_logger.debug("Failed to transit state", e); result.setResult(e.toString()); future.complete(result); return null; } - + template.setImageStoreId(templateOnStore.getDataStore().getId()); + template.setSize(callbackResult.getSize()); try { template.stateTransit(TemplateEvent.OperationSucceeded); } catch (NoTransitionException e) { diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/driver/DefaultImageDataStoreDriverImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/driver/DefaultImageDataStoreDriverImpl.java index b713fa66a53..dce5a939413 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/driver/DefaultImageDataStoreDriverImpl.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/driver/DefaultImageDataStoreDriverImpl.java @@ -34,11 +34,15 @@ import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.command.CreateObjectCommand; import org.apache.cloudstack.storage.endpoint.EndPointSelector; import org.apache.cloudstack.storage.image.ImageDataStoreDriver; +import org.apache.cloudstack.storage.image.db.ImageDataDao; +import org.apache.cloudstack.storage.image.db.ImageDataVO; //http-read-only based image store public class DefaultImageDataStoreDriverImpl implements ImageDataStoreDriver { @Inject EndPointSelector selector; + @Inject + ImageDataDao imageDataDao; public DefaultImageDataStoreDriverImpl() { } @@ -81,10 +85,13 @@ public class DefaultImageDataStoreDriverImpl implements ImageDataStoreDriver { CreateObjectCommand createCmd = new CreateObjectCommand(data.getUri()); CreateObjectAnswer answer = (CreateObjectAnswer)ep.sendMessage(createCmd); if (answer.getResult()) { + //update imagestorevo + result = new CreateCmdResult(answer.getPath(), answer.getSize()); } else { result.setResult(answer.getDetails()); } + } callback.complete(result); diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/HttpDataStoreImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/HttpDataStoreImpl.java index e1fdd3d5b63..34b4ff27f1a 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/HttpDataStoreImpl.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/HttpDataStoreImpl.java @@ -38,6 +38,7 @@ import org.apache.cloudstack.storage.image.db.ImageDataStoreVO; import org.apache.cloudstack.storage.snapshot.SnapshotInfo; import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.storage.encoding.EncodingType; public class HttpDataStoreImpl implements ImageDataStore { @@ -100,11 +101,9 @@ public class HttpDataStoreImpl implements ImageDataStore { @Override public String getUri() { - return this.imageDataStoreVO.getProtocol() + "://" + "?role=" + this.getRole(); + return this.imageDataStoreVO.getProtocol() + "://" + "?" + EncodingType.ROLE + "=" + this.getRole(); } - - @Override public Scope getScope() { // TODO Auto-generated method stub diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java index f926a66383a..1b0661c7691 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java @@ -18,8 +18,6 @@ */ package org.apache.cloudstack.storage.image.store; -import java.io.File; - import javax.inject.Inject; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectType; @@ -37,9 +35,11 @@ import org.apache.log4j.Logger; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.utils.storage.encoding.EncodingType; public class TemplateObject implements TemplateInfo { - private static final Logger s_logger = Logger.getLogger(TemplateObject.class); + private static final Logger s_logger = Logger + .getLogger(TemplateObject.class); private ImageDataVO imageVO; private DataStore dataStore; @Inject @@ -51,22 +51,26 @@ public class TemplateObject implements TemplateInfo { protected TemplateObject() { } - + protected void configure(ImageDataVO template, DataStore dataStore) { this.imageVO = template; this.dataStore = dataStore; } - + public static TemplateObject getTemplate(ImageDataVO vo, DataStore store) { TemplateObject to = ComponentContext.inject(TemplateObject.class); to.configure(vo, store); return to; } - + public void setImageStoreId(long id) { this.imageVO.setImageDataStoreId(id); } + public void setSize(Long size) { + this.imageVO.setSize(size); + } + public ImageDataVO getImage() { return this.imageVO; } @@ -89,14 +93,28 @@ public class TemplateObject implements TemplateInfo { @Override public String getUri() { + ImageDataVO image = imageDao.findById(this.imageVO.getId()); if (this.dataStore == null) { - return this.imageVO.getUrl(); + return image.getUrl(); } else { - ObjectInDataStoreVO obj = ojbectInStoreMgr.findObject(this.imageVO.getId(), DataObjectType.TEMPLATE, this.dataStore.getId(), this.dataStore.getRole()); - if (obj.getState() != ObjectInDataStoreStateMachine.State.Ready) { - return this.dataStore.getUri() + File.separator + "&objType=" + DataObjectType.TEMPLATE + "&size=" + this.imageVO.getSize() + "&path=" + this.imageVO.getUrl(); + ObjectInDataStoreVO obj = ojbectInStoreMgr.findObject( + this.imageVO.getId(), DataObjectType.TEMPLATE, + this.dataStore.getId(), this.dataStore.getRole()); + StringBuilder builder = new StringBuilder(); + if (obj.getState() == ObjectInDataStoreStateMachine.State.Ready + || obj.getState() == ObjectInDataStoreStateMachine.State.Copying) { + + builder.append(this.dataStore.getUri()); + builder.append("&" + EncodingType.OBJTYPE + "=" + DataObjectType.TEMPLATE); + builder.append("&" + EncodingType.PATH + "=" + obj.getInstallPath()); + builder.append("&" + EncodingType.SIZE + "=" + image.getSize()); + return builder.toString(); } else { - return this.dataStore.getUri() + File.separator + "&objType=" + DataObjectType.TEMPLATE + "&path=" + obj.getInstallPath(); + builder.append(this.dataStore.getUri()); + builder.append("&" + EncodingType.OBJTYPE + "=" + DataObjectType.TEMPLATE); + builder.append("&" + EncodingType.SIZE + "=" + image.getSize()); + builder.append("&" + EncodingType.PATH + "=" + image.getUrl()); + return builder.toString(); } } } @@ -104,9 +122,11 @@ public class TemplateObject implements TemplateInfo { @Override public Long getSize() { if (this.dataStore == null) { - return null; + return this.imageVO.getSize(); } - ObjectInDataStoreVO obj = ojbectInStoreMgr.findObject(this.imageVO.getId(), DataObjectType.TEMPLATE, this.dataStore.getId(), this.dataStore.getRole()); + ObjectInDataStoreVO obj = ojbectInStoreMgr.findObject( + this.imageVO.getId(), DataObjectType.TEMPLATE, + this.dataStore.getId(), this.dataStore.getRole()); return obj.getSize(); } @@ -119,8 +139,11 @@ public class TemplateObject implements TemplateInfo { public DiskFormat getFormat() { return DiskFormat.getFormat(this.imageVO.getFormat()); } - + public boolean stateTransit(TemplateEvent e) throws NoTransitionException { - return imageMgr.getStateMachine().transitTo(this.imageVO, e, null, imageDao); + boolean result= imageMgr.getStateMachine().transitTo(this.imageVO, e, null, + imageDao); + this.imageVO = imageDao.findById(this.imageVO.getId()); + return result; } } diff --git a/engine/storage/imagemotion/src/org/apache/cloudstack/storage/image/motion/DefaultImageMotionStrategy.java b/engine/storage/imagemotion/src/org/apache/cloudstack/storage/image/motion/DefaultImageMotionStrategy.java index 6572b85d9ab..390b0fd7e34 100644 --- a/engine/storage/imagemotion/src/org/apache/cloudstack/storage/image/motion/DefaultImageMotionStrategy.java +++ b/engine/storage/imagemotion/src/org/apache/cloudstack/storage/image/motion/DefaultImageMotionStrategy.java @@ -93,7 +93,7 @@ public class DefaultImageMotionStrategy implements ImageMotionStrategy { || srcStore.getRole() == DataStoreRole.ImageCache) { return true; } - return false; + return true; } @Override @@ -124,13 +124,14 @@ public class DefaultImageMotionStrategy implements ImageMotionStrategy { protected Void copyAsyncCallback(AsyncCallbackDispatcher callback, CreateTemplateContext context) { AsyncCompletionCallback parentCall = context.getParentCallback(); - CopyCmdAnswer answer = (CopyCmdAnswer)callback.getResult(); + Answer answer = (Answer)callback.getResult(); if (!answer.getResult()) { CopyCommandResult result = new CopyCommandResult(""); result.setResult(answer.getDetails()); parentCall.complete(result); } else { - CopyCommandResult result = new CopyCommandResult(answer.getPath()); + CopyCmdAnswer ans = (CopyCmdAnswer)answer; + CopyCommandResult result = new CopyCommandResult(ans.getPath()); parentCall.complete(result); } return null; diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java index c965322828e..9c30a2e8269 100644 --- a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java @@ -19,7 +19,6 @@ package org.apache.cloudstack.storage.test; import org.apache.cloudstack.acl.APIChecker; import org.apache.cloudstack.engine.service.api.OrchestrationService; import org.apache.cloudstack.storage.HostEndpointRpcServer; -import org.apache.cloudstack.storage.endpoint.DefaultEndPointSelector; import org.apache.cloudstack.storage.endpoint.EndPointSelector; import org.mockito.Mockito; import org.springframework.context.annotation.Bean; @@ -28,6 +27,8 @@ import com.cloud.agent.AgentManager; import com.cloud.cluster.ClusteredAgentRebalanceService; import com.cloud.cluster.agentlb.dao.HostTransferMapDao; import com.cloud.cluster.agentlb.dao.HostTransferMapDaoImpl; +import com.cloud.configuration.dao.ConfigurationDao; +import com.cloud.configuration.dao.ConfigurationDaoImpl; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.ClusterDaoImpl; import com.cloud.dc.dao.DataCenterDao; @@ -39,6 +40,8 @@ import com.cloud.dc.dao.DcDetailsDaoImpl; import com.cloud.dc.dao.HostPodDao; import com.cloud.dc.dao.HostPodDaoImpl; import com.cloud.dc.dao.PodVlanDaoImpl; +import com.cloud.domain.dao.DomainDao; +import com.cloud.domain.dao.DomainDaoImpl; import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostDetailsDao; import com.cloud.host.dao.HostDetailsDaoImpl; @@ -47,7 +50,10 @@ import com.cloud.host.dao.HostTagsDaoImpl; import com.cloud.server.auth.UserAuthenticator; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.StoragePoolHostDaoImpl; -import com.cloud.utils.component.ComponentContext; +import com.cloud.storage.dao.VMTemplateDetailsDao; +import com.cloud.storage.dao.VMTemplateDetailsDaoImpl; +import com.cloud.storage.dao.VMTemplateZoneDao; +import com.cloud.storage.dao.VMTemplateZoneDaoImpl; public class ChildTestConfiguration extends TestConfiguration { @@ -105,11 +111,22 @@ public class ChildTestConfiguration extends TestConfiguration { @Bean StoragePoolHostDao storagePoolHostDao() { return new StoragePoolHostDaoImpl(); } - + @Bean VMTemplateZoneDao templateZoneDao() { + return new VMTemplateZoneDaoImpl(); + } + @Bean VMTemplateDetailsDao templateDetailsDao() { + return new VMTemplateDetailsDaoImpl(); + } + @Bean ConfigurationDao configDao() { + return new ConfigurationDaoImpl(); + } @Bean public AgentManager agentMgr() { return new DirectAgentManagerSimpleImpl(); } + @Bean DomainDao domainDao() { + return new DomainDaoImpl(); + } @Bean public HostEndpointRpcServer rpcServer() { diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/MockRpcCallBack.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/MockRpcCallBack.java index 207cc52e989..e0ffb48281a 100644 --- a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/MockRpcCallBack.java +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/MockRpcCallBack.java @@ -19,14 +19,15 @@ package org.apache.cloudstack.storage.test; import javax.inject.Inject; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.utils.db.DB; -import com.cloud.utils.db.Transaction; public class MockRpcCallBack implements Runnable { + private static final Logger s_logger = Logger.getLogger(MockRpcCallBack.class); @Inject AgentManager agentMgr; private Command cmd; @@ -50,12 +51,9 @@ public class MockRpcCallBack implements Runnable { public void run() { try { Answer answer = agentMgr.send(hostId, cmd); - callback.complete(answer); } catch (Throwable e) { - //s_logger.debug("send command failed:" + e.toString()); - } finally { - int i =1; + s_logger.debug("send command failed:", e); } } diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/volumeServiceTest.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/volumeServiceTest.java index eeb7b54cc50..0e88f733e08 100644 --- a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/volumeServiceTest.java +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/volumeServiceTest.java @@ -25,12 +25,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ExecutionException; import javax.inject.Inject; import javax.naming.ConfigurationException; -import org.apache.cloudstack.engine.cloud.entity.api.TemplateEntity; -import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.CommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -38,11 +37,12 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreRole; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo; import org.apache.cloudstack.engine.subsystem.api.storage.ScopeType; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.type.RootDisk; import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.storage.HypervisorHostEndPoint; +import org.apache.cloudstack.storage.datastore.VolumeDataFactory; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreVO; import org.apache.cloudstack.storage.datastore.provider.DataStoreProvider; @@ -54,6 +54,7 @@ import org.apache.cloudstack.storage.image.TemplateInfo; import org.apache.cloudstack.storage.image.db.ImageDataDao; import org.apache.cloudstack.storage.image.db.ImageDataVO; import org.apache.cloudstack.storage.volume.VolumeService; +import org.apache.cloudstack.storage.volume.VolumeService.VolumeApiResult; import org.apache.cloudstack.storage.volume.db.VolumeDao2; import org.apache.cloudstack.storage.volume.db.VolumeVO; import org.mockito.Mockito; @@ -109,12 +110,14 @@ public class volumeServiceTest extends CloudStackTestNGBase { EndPointSelector selector; @Inject ImageDataFactory imageDataFactory; + @Inject + VolumeDataFactory volumeFactory; Long dcId; Long clusterId; Long podId; HostVO host; String primaryName = "my primary data store"; - PrimaryDataStoreInfo primaryStore; + DataStore primaryStore; @Test(priority = -1) public void setUp() { @@ -199,6 +202,7 @@ public class volumeServiceTest extends CloudStackTestNGBase { host.getPrivateIpAddress())); Mockito.when(selector.selectAll(Mockito.any(DataStore.class))).thenReturn(eps); Mockito.when(selector.select(Mockito.any(DataObject.class))).thenReturn(eps.get(0)); + Mockito.when(selector.select(Mockito.any(DataObject.class), Mockito.any(DataObject.class))).thenReturn(eps.get(0)); } private ImageDataVO createImageData() { @@ -227,13 +231,14 @@ public class volumeServiceTest extends CloudStackTestNGBase { return image; } - private TemplateEntity createTemplate() { + private TemplateInfo createTemplate() { try { DataStore store = createImageStore(); ImageDataVO image = createImageData(); TemplateInfo template = imageDataFactory.getTemplate(image.getId(), store); AsyncCallFuture future = imageService.createTemplateAsync(template, store); future.get(); + template = imageDataFactory.getTemplate(image.getId(), store); /*imageProviderMgr.configure("image Provider", new HashMap()); ImageDataVO image = createImageData(); ImageDataStoreProvider defaultProvider = imageProviderMgr.getProvider("DefaultProvider"); @@ -242,19 +247,19 @@ public class volumeServiceTest extends CloudStackTestNGBase { imageService.registerTemplate(image.getId(), store.getImageDataStoreId()); TemplateEntity te = imageService.getTemplateEntity(image.getId()); return te;*/ - return null; + return template; } catch (Exception e) { Assert.fail("failed", e); return null; } } - @Test + //@Test public void createTemplateTest() { createTemplate(); } - //@Test + @Test public void testCreatePrimaryStorage() { DataStoreProvider provider = dataStoreProviderMgr.getDataStoreProvider("default primary data store provider"); Map params = new HashMap(); @@ -278,9 +283,9 @@ public class volumeServiceTest extends CloudStackTestNGBase { params.put("providerId", String.valueOf(provider.getId())); DataStoreLifeCycle lifeCycle = provider.getLifeCycle(); - DataStore store = lifeCycle.initialize(params); + this.primaryStore = lifeCycle.initialize(params); ClusterScope scope = new ClusterScope(clusterId, podId, dcId); - lifeCycle.attachCluster(store, scope); + lifeCycle.attachCluster(this.primaryStore, scope); } private DataStore createImageStore() { @@ -296,15 +301,14 @@ public class volumeServiceTest extends CloudStackTestNGBase { DataStore store = lifeCycle.initialize(params); return store; } - @Test + //@Test public void testcreateImageStore() { createImageStore(); } - @Test - public PrimaryDataStoreInfo createPrimaryDataStore() { - try {/* + public DataStore createPrimaryDataStore() { + try { DataStoreProvider provider = dataStoreProviderMgr.getDataStoreProvider("default primary data store provider"); Map params = new HashMap(); URI uri = new URI(this.getPrimaryStorageUrl()); @@ -324,7 +328,7 @@ public class volumeServiceTest extends CloudStackTestNGBase { DataStore store = lifeCycle.initialize(params); ClusterScope scope = new ClusterScope(clusterId, podId, dcId); lifeCycle.attachCluster(store, scope); - */ + /* PrimaryDataStoreProvider provider = primaryDataStoreProviderMgr.getDataStoreProvider("default primary data store provider"); primaryDataStoreProviderMgr.configure("primary data store mgr", new HashMap()); @@ -349,7 +353,7 @@ public class volumeServiceTest extends CloudStackTestNGBase { lc.attachCluster(scope); return primaryDataStoreInfo; */ - return null; + return store; } catch (Exception e) { return null; } @@ -362,25 +366,72 @@ public class volumeServiceTest extends CloudStackTestNGBase { return volume; } - //@Test(priority=2) + @Test(priority=2) public void createVolumeFromTemplate() { - primaryStore = createPrimaryDataStore(); - TemplateEntity te = createTemplate(); + DataStore primaryStore = this.primaryStore; + TemplateInfo te = createTemplate(); VolumeVO volume = createVolume(te.getId(), primaryStore.getId()); - VolumeEntity ve = volumeService.getVolumeEntity(volume.getId()); + VolumeInfo vol = volumeFactory.getVolume(volume.getId(), primaryStore); //ve.createVolumeFromTemplate(primaryStore.getId(), new VHD(), te); - ve.destroy(); + AsyncCallFuture future = volumeService.createVolumeFromTemplateAsync(vol, primaryStore.getId(), te); + try { + future.get(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ExecutionException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } //@Test(priority=3) public void createDataDisk() { - primaryStore = createPrimaryDataStore(); + DataStore primaryStore = this.primaryStore; VolumeVO volume = createVolume(null, primaryStore.getId()); - VolumeEntity ve = volumeService.getVolumeEntity(volume.getId()); - //ve.createVolume(primaryStore.getId(), new VHD()); - ve.destroy(); + VolumeInfo vol = volumeFactory.getVolume(volume.getId(), primaryStore); + AsyncCallFuture future = volumeService.createVolumeAsync(vol, primaryStore.getId()); + try { + future.get(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ExecutionException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } + //@Test(priority=3) + public void createAndDeleteDataDisk() { + DataStore primaryStore = this.primaryStore; + VolumeVO volume = createVolume(null, primaryStore.getId()); + VolumeInfo vol = volumeFactory.getVolume(volume.getId(), primaryStore); + AsyncCallFuture future = volumeService.createVolumeAsync(vol, primaryStore.getId()); + try { + future.get(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ExecutionException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + //delete the volume + vol = volumeFactory.getVolume(volume.getId(), primaryStore); + future = volumeService.deleteVolumeAsync(vol); + try { + future.get(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ExecutionException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + //@Test(priority=3) public void tearDown() { List ds = primaryStoreDao.findPoolByName(this.primaryName); diff --git a/engine/storage/src/org/apache/cloudstack/storage/command/CopyCmd.java b/engine/storage/src/org/apache/cloudstack/storage/command/CopyCmd.java index dcb81ba486c..10478ef24c7 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/command/CopyCmd.java +++ b/engine/storage/src/org/apache/cloudstack/storage/command/CopyCmd.java @@ -16,25 +16,24 @@ // under the License. package org.apache.cloudstack.storage.command; -import org.apache.cloudstack.storage.to.ImageOnPrimayDataStoreTO; - import com.cloud.agent.api.Command; public class CopyCmd extends Command implements StorageSubSystemCommand { + private String srcUri; + private String destUri; - private ImageOnPrimayDataStoreTO imageTO; - - protected CopyCmd() { + public CopyCmd(String srcUri, String destUri) { super(); - } - - public CopyCmd(String destUri, String srcUri) { - super(); - // this.imageTO = image; + this.srcUri = srcUri; + this.destUri = destUri; } - public ImageOnPrimayDataStoreTO getImage() { - return this.imageTO; + public String getDestUri() { + return this.destUri; + } + + public String getSrcUri() { + return this.srcUri; } @Override diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/DataObjectManager.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/DataObjectManager.java new file mode 100644 index 00000000000..20bf0545dc9 --- /dev/null +++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/DataObjectManager.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore; + +import org.apache.cloudstack.engine.subsystem.api.storage.CommandResult; +import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.framework.async.AsyncCompletionCallback; + +public interface DataObjectManager { + public void createAsync(DataObject data, DataStore store, AsyncCompletionCallback callback, boolean noCopy); + /* + * Only create internal state, without actually send down create command. + * It's up to device driver decides whether to create object before copying + */ + public DataObject createInternalStateOnly(DataObject data, DataStore store); + public void update(DataObject data, String path, Long size); + public void copyAsync(DataObject srcData, DataObject destData, AsyncCompletionCallback callback); + public void deleteAsync(DataObject data, AsyncCompletionCallback callback); +} diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/DataObjectManagerImpl.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/DataObjectManagerImpl.java new file mode 100644 index 00000000000..657d32c7877 --- /dev/null +++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/DataObjectManagerImpl.java @@ -0,0 +1,402 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore; + +import javax.inject.Inject; + +import org.apache.cloudstack.engine.subsystem.api.storage.CommandResult; +import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; +import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; +import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.framework.async.AsyncRpcConext; +import org.apache.cloudstack.storage.db.ObjectInDataStoreVO; +import org.apache.cloudstack.storage.motion.DataMotionService; +import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine.Event; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.fsm.NoTransitionException; + +@Component +public class DataObjectManagerImpl implements DataObjectManager { + private static final Logger s_logger = Logger + .getLogger(DataObjectManagerImpl.class); + @Inject + ObjectInDataStoreManager objectInDataStoreMgr; + @Inject + DataMotionService motionSrv; + protected long waitingTime = 1800; // half an hour + protected long waitingRetries = 10; + + protected DataObject waitingForCreated(DataObject dataObj, + DataStore dataStore) { + long retries = this.waitingRetries; + ObjectInDataStoreVO obj = null; + do { + try { + Thread.sleep(waitingTime); + } catch (InterruptedException e) { + s_logger.debug("sleep interrupted", e); + throw new CloudRuntimeException("sleep interrupted", e); + } + + obj = objectInDataStoreMgr.findObject(dataObj.getId(), + dataObj.getType(), dataStore.getId(), dataStore.getRole()); + if (obj == null) { + s_logger.debug("can't find object in db, maybe it's cleaned up already, exit waiting"); + break; + } + if (obj.getState() == ObjectInDataStoreStateMachine.State.Ready) { + break; + } + retries--; + } while (retries > 0); + + if (obj == null || retries <= 0) { + s_logger.debug("waiting too long for template downloading, marked it as failed"); + throw new CloudRuntimeException( + "waiting too long for template downloading, marked it as failed"); + } + return objectInDataStoreMgr.get(dataObj, dataStore); + } + class CreateContext extends AsyncRpcConext { + final DataObject objInStrore; + + public CreateContext(AsyncCompletionCallback callback, + DataObject objInStore) { + super(callback); + this.objInStrore = objInStore; + } + + } + + @Override + public void createAsync(DataObject data, DataStore store, + AsyncCompletionCallback callback, boolean noCopy) { + ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject( + data.getId(), data.getType(), store.getId(), + store.getRole()); + DataObject objInStore = null; + boolean freshNewTemplate = false; + if (obj == null) { + try { + objInStore = objectInDataStoreMgr.create( + data, store); + freshNewTemplate = true; + } catch (Throwable e) { + obj = objectInDataStoreMgr.findObject(data.getId(), + data.getType(), store.getId(), store.getRole()); + if (obj == null) { + CreateCmdResult result = new CreateCmdResult( + null, null); + result.setSucess(false); + result.setResult(e.toString()); + callback.complete(result); + return; + } + } + } + + if (!freshNewTemplate + && obj.getState() != ObjectInDataStoreStateMachine.State.Ready) { + try { + objInStore = waitingForCreated( + data, store); + } catch (Exception e) { + CreateCmdResult result = new CreateCmdResult(null, null); + result.setSucess(false); + result.setResult(e.toString()); + callback.complete(result); + return; + } + + CreateCmdResult result = new CreateCmdResult( + null, null); + callback.complete(result); + return; + } + + try { + ObjectInDataStoreStateMachine.Event event = null; + if (noCopy) { + event = ObjectInDataStoreStateMachine.Event.CreateOnlyRequested; + } else { + event = ObjectInDataStoreStateMachine.Event.CreateRequested; + } + objectInDataStoreMgr.update(objInStore, + event); + } catch (NoTransitionException e) { + try { + objectInDataStoreMgr.update(objInStore, + ObjectInDataStoreStateMachine.Event.OperationFailed); + } catch (NoTransitionException e1) { + s_logger.debug("state transation failed", e1); + } + CreateCmdResult result = new CreateCmdResult(null, null); + result.setSucess(false); + result.setResult(e.toString()); + callback.complete(result); + return; + } + + CreateContext context = new CreateContext( + callback, objInStore); + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher + .create(this); + caller.setCallback( + caller.getTarget().createAsynCallback(null, null)) + .setContext(context); + + store.getDriver().createAsync(objInStore, caller); + return; + } + + protected Void createAsynCallback(AsyncCallbackDispatcher callback, + CreateContext context) { + CreateCmdResult result = callback.getResult(); + DataObject objInStrore = context.objInStrore; + CreateCmdResult upResult = new CreateCmdResult( + null, null); + if (result.isFailed()) { + upResult.setResult(result.getResult()); + context.getParentCallback().complete(upResult); + return null; + } + + ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject( + objInStrore.getId(), objInStrore + .getType(), objInStrore.getDataStore() + .getId(), objInStrore.getDataStore() + .getRole()); + + obj.setInstallPath(result.getPath()); + obj.setSize(result.getSize()); + try { + objectInDataStoreMgr.update(obj, + ObjectInDataStoreStateMachine.Event.OperationSuccessed); + } catch (NoTransitionException e) { + try { + objectInDataStoreMgr.update(obj, + ObjectInDataStoreStateMachine.Event.OperationFailed); + } catch (NoTransitionException e1) { + s_logger.debug("failed to change state", e1); + } + + upResult.setResult(e.toString()); + context.getParentCallback().complete(upResult); + return null; + } + + context.getParentCallback().complete(result); + return null; + } + + class CopyContext extends AsyncRpcConext { + DataObject destObj; + DataObject srcObj; + + public CopyContext(AsyncCompletionCallback callback, + DataObject srcObj, + DataObject destObj) { + super(callback); + this.srcObj = srcObj; + this.destObj = destObj; + } + } + + @Override + public void copyAsync(DataObject srcData, DataObject destData, + AsyncCompletionCallback callback) { + try { + objectInDataStoreMgr.update(destData, + ObjectInDataStoreStateMachine.Event.CopyingRequested); + } catch (NoTransitionException e) { + s_logger.debug("failed to change state", e); + try { + objectInDataStoreMgr.update(destData, + ObjectInDataStoreStateMachine.Event.OperationFailed); + } catch (NoTransitionException e1) { + + } + CreateCmdResult res = new CreateCmdResult(null, null); + res.setResult("Failed to change state: " + e.toString()); + callback.complete(res); + } + + CopyContext anotherCall = new CopyContext( + callback, srcData, destData); + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher + .create(this); + caller.setCallback(caller.getTarget().copyCallback(null, null)) + .setContext(anotherCall); + + motionSrv.copyAsync(srcData, destData, caller); + } + + protected Void copyCallback( + AsyncCallbackDispatcher callback, + CopyContext context) { + CopyCommandResult result = callback.getResult(); + DataObject destObj = context.destObj; + ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject( + destObj.getId(), destObj + .getType(), destObj.getDataStore() + .getId(), destObj.getDataStore() + .getRole()); + if (result.isFailed()) { + try { + objectInDataStoreMgr.update(obj, Event.OperationFailed); + } catch (NoTransitionException e) { + s_logger.debug("Failed to update copying state", e); + } + CreateCmdResult res = new CreateCmdResult( + null, null); + res.setResult(result.getResult()); + context.getParentCallback().complete(res); + } + + obj.setInstallPath(result.getPath()); + + try { + objectInDataStoreMgr.update(obj, + ObjectInDataStoreStateMachine.Event.OperationSuccessed); + } catch (NoTransitionException e) { + s_logger.debug("Failed to update copying state: ", e); + try { + objectInDataStoreMgr.update(destObj, + ObjectInDataStoreStateMachine.Event.OperationFailed); + } catch (NoTransitionException e1) { + } + CreateCmdResult res = new CreateCmdResult( + null, null); + res.setResult("Failed to update copying state: " + e.toString()); + context.getParentCallback().complete(res); + } + CreateCmdResult res = new CreateCmdResult( + result.getPath(), null); + context.getParentCallback().complete(res); + return null; + } + + + class DeleteContext extends AsyncRpcConext { + private final DataObject obj; + public DeleteContext(AsyncCompletionCallback callback, DataObject obj) { + super(callback); + this.obj = obj; + } + + } + @Override + public void deleteAsync(DataObject data, + AsyncCompletionCallback callback) { + ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject( + data.getId(), data.getType(), data.getDataStore().getId(), + data.getDataStore().getRole()); + try { + objectInDataStoreMgr.update(obj, Event.DestroyRequested); + } catch (NoTransitionException e) { + s_logger.debug("destroy failed", e); + CreateCmdResult res = new CreateCmdResult( + null, null); + callback.complete(res); + } + + DeleteContext context = new DeleteContext( + callback, data); + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher + .create(this); + caller.setCallback( + caller.getTarget().deleteAsynCallback(null, null)) + .setContext(context); + + data.getDataStore().getDriver().deleteAsync(data, caller); + return; + } + + protected Void deleteAsynCallback(AsyncCallbackDispatcher callback, + DeleteContext context) { + DataObject destObj = context.obj; + ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject( + destObj.getId(), destObj + .getType(), destObj.getDataStore() + .getId(), destObj.getDataStore() + .getRole()); + + CommandResult res = callback.getResult(); + if (res.isFailed()) { + try { + objectInDataStoreMgr.update(obj, Event.OperationFailed); + } catch (NoTransitionException e) { + s_logger.debug("delete failed", e); + } + + } else { + try { + objectInDataStoreMgr.update(obj, Event.OperationSuccessed); + } catch (NoTransitionException e) { + s_logger.debug("delete failed", e); + } + } + + context.getParentCallback().complete(res); + return null; + } + + @Override + public DataObject createInternalStateOnly(DataObject data, DataStore store) { + ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject( + data.getId(), data.getType(), store.getId(), + store.getRole()); + DataObject objInStore = null; + if (obj == null) { + objInStore = objectInDataStoreMgr.create( + data, store); + } + try { + ObjectInDataStoreStateMachine.Event event = null; + event = ObjectInDataStoreStateMachine.Event.CreateRequested; + objectInDataStoreMgr.update(objInStore, + event); + + objectInDataStoreMgr.update(objInStore, ObjectInDataStoreStateMachine.Event.OperationSuccessed); + } catch (NoTransitionException e) { + s_logger.debug("Failed to update state", e); + throw new CloudRuntimeException("Failed to update state", e); + } + + return objInStore; + } + + @Override + public void update(DataObject data, String path, Long size) { + ObjectInDataStoreVO obj = objectInDataStoreMgr.findObject( + data.getId(), data.getType(), data.getDataStore().getId(), + data.getDataStore().getRole()); + + obj.setInstallPath(path); + obj.setSize(size); + objectInDataStoreMgr.update(obj); + } +} diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java index 5f0004607f3..e707de6b8bd 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java +++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java @@ -22,18 +22,21 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreRole; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.storage.db.ObjectInDataStoreVO; -import org.apache.cloudstack.storage.image.TemplateInfo; import org.apache.cloudstack.storage.snapshot.SnapshotInfo; -import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine; import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine.Event; import com.cloud.utils.fsm.NoTransitionException; public interface ObjectInDataStoreManager { - public TemplateInfo create(TemplateInfo template, DataStore dataStore); + public DataObject create(DataObject template, DataStore dataStore); public VolumeInfo create(VolumeInfo volume, DataStore dataStore); public SnapshotInfo create(SnapshotInfo snapshot, DataStore dataStore); public ObjectInDataStoreVO findObject(long objectId, DataObjectType type, long dataStoreId, DataStoreRole role); + public DataObject get(DataObject dataObj, DataStore store); public boolean update(DataObject vo, Event event) throws NoTransitionException; + boolean update(ObjectInDataStoreVO obj, Event event) + throws NoTransitionException; + + boolean update(ObjectInDataStoreVO obj); } diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java index 23d69a4f9de..7eb4932348f 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java @@ -26,7 +26,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.storage.db.ObjectInDataStoreDao; import org.apache.cloudstack.storage.db.ObjectInDataStoreVO; import org.apache.cloudstack.storage.image.ImageDataFactory; -import org.apache.cloudstack.storage.image.TemplateInfo; import org.apache.cloudstack.storage.snapshot.SnapshotInfo; import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine; import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine.Event; @@ -83,17 +82,23 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { } @Override - public TemplateInfo create(TemplateInfo template, DataStore dataStore) { + public DataObject create(DataObject obj, DataStore dataStore) { + ObjectInDataStoreVO vo = new ObjectInDataStoreVO(); vo.setDataStoreId(dataStore.getId()); vo.setDataStoreRole(dataStore.getRole()); - vo.setObjectId(template.getId()); + vo.setObjectId(obj.getId()); + vo.setSize(obj.getSize()); - vo.setObjectType(template.getType()); + vo.setObjectType(obj.getType()); vo = objectDataStoreDao.persist(vo); - return imageFactory.getTemplate(template.getId(), dataStore); - + if (obj.getType() == DataObjectType.TEMPLATE) { + return imageFactory.getTemplate(obj.getId(), dataStore); + } else if (obj.getType() == DataObjectType.VOLUME) { + return volumeFactory.getVolume(obj.getId(), dataStore); + } + throw new CloudRuntimeException("unknown type"); } @Override @@ -145,4 +150,27 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { objectDataStoreDao); } + + @Override + public boolean update(ObjectInDataStoreVO obj, Event event) + throws NoTransitionException { + return this.stateMachines.transitTo(obj, event, null, + objectDataStoreDao); + + } + + @Override + public DataObject get(DataObject dataObj, DataStore store) { + if (dataObj.getType() == DataObjectType.TEMPLATE) { + return imageFactory.getTemplate(dataObj.getId(), store); + } else if (dataObj.getType() == DataObjectType.VOLUME) { + return volumeFactory.getVolume(dataObj.getId(), store); + } + throw new CloudRuntimeException("unknown type"); + } + + @Override + public boolean update(ObjectInDataStoreVO obj) { + return objectDataStoreDao.update(obj.getId(), obj); + } } diff --git a/engine/storage/src/org/apache/cloudstack/storage/db/ObjectInDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/db/ObjectInDataStoreDaoImpl.java index 7bd9f587404..4a5a913adca 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/db/ObjectInDataStoreDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/db/ObjectInDataStoreDaoImpl.java @@ -16,31 +16,43 @@ // under the License. package org.apache.cloudstack.storage.db; import java.util.Date; +import java.util.Map; -import org.springframework.stereotype.Component; +import javax.naming.ConfigurationException; import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine.Event; import org.apache.cloudstack.storage.volume.ObjectInDataStoreStateMachine.State; -import org.apache.cloudstack.storage.volume.db.VolumeVO; import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; +import com.cloud.storage.VolumeVO; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import com.cloud.utils.db.SearchCriteria2; -import com.cloud.utils.db.SearchCriteriaService; import com.cloud.utils.db.UpdateBuilder; +import com.cloud.utils.db.SearchCriteria.Op; @Component public class ObjectInDataStoreDaoImpl extends GenericDaoBase implements ObjectInDataStoreDao { private static final Logger s_logger = Logger.getLogger(ObjectInDataStoreDaoImpl.class); + private SearchBuilder updateStateSearch; + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + updateStateSearch = this.createSearchBuilder(); + updateStateSearch.and("id", updateStateSearch.entity().getId(), Op.EQ); + updateStateSearch.and("state", updateStateSearch.entity().getState(), Op.EQ); + updateStateSearch.and("updatedCount", updateStateSearch.entity().getUpdatedCount(), Op.EQ); + updateStateSearch.done(); + return true; + } @Override public boolean updateState(State currentState, Event event, State nextState, ObjectInDataStoreVO vo, Object data) { Long oldUpdated = vo.getUpdatedCount(); Date oldUpdatedTime = vo.getUpdated(); - SearchCriteria sc = this.createSearchCriteria(); + + SearchCriteria sc = updateStateSearch.create(); sc.setParameters("id", vo.getId()); sc.setParameters("state", currentState); sc.setParameters("updatedCount", vo.getUpdatedCount()); diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageDataDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageDataDaoImpl.java index 849f1973283..301b5861f8c 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageDataDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageDataDaoImpl.java @@ -68,7 +68,9 @@ import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Func; +import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.UpdateBuilder; import com.cloud.utils.exception.CloudRuntimeException; @Component @@ -76,18 +78,19 @@ public class ImageDataDaoImpl extends GenericDaoBase implemen private static final Logger s_logger = Logger.getLogger(VMTemplateDaoImpl.class); @Inject - VMTemplateZoneDao _templateZoneDao = null; + VMTemplateZoneDao templateZoneDao; @Inject - VMTemplateDetailsDao _templateDetailsDao = null; + VMTemplateDetailsDao templateDetailsDao; @Inject - ConfigurationDao _configDao = null; + ConfigurationDao configDao; @Inject - HostDao _hostDao = null; + HostDao hostDao; @Inject - DomainDao _domainDao = null; + DomainDao domainDao; @Inject - DataCenterDao _dcDao = null; + DataCenterDao dcDao; + private final String SELECT_TEMPLATE_HOST_REF = "SELECT t.id, h.data_center_id, t.unique_name, t.name, t.public, t.featured, t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, " + "t.checksum, t.display_text, t.enable_password, t.guest_os_id, t.bootable, t.prepopulate, t.cross_zones, t.hypervisor_type FROM vm_template t"; @@ -110,6 +113,7 @@ public class ImageDataDaoImpl extends GenericDaoBase implemen private SearchBuilder PublicIsoSearch; private SearchBuilder UserIsoSearch; private GenericSearchBuilder CountTemplatesByAccount; + private SearchBuilder updateStateSearch; //ResourceTagsDaoImpl _tagsDao = ComponentInject.inject(ResourceTagsDaoImpl.class); @Inject @@ -326,7 +330,7 @@ public class ImageDataDaoImpl extends GenericDaoBase implemen tmpltTypeHyperSearch = createSearchBuilder(); tmpltTypeHyperSearch.and("templateType", tmpltTypeHyperSearch.entity().getTemplateType(), SearchCriteria.Op.EQ); - SearchBuilder hostHyperSearch = _hostDao.createSearchBuilder(); + SearchBuilder hostHyperSearch = hostDao.createSearchBuilder(); hostHyperSearch.and("type", hostHyperSearch.entity().getType(), SearchCriteria.Op.EQ); hostHyperSearch.and("zoneId", hostHyperSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ); hostHyperSearch.groupBy(hostHyperSearch.entity().getHypervisorType()); @@ -348,7 +352,7 @@ public class ImageDataDaoImpl extends GenericDaoBase implemen AccountIdSearch.and("publicTemplate", AccountIdSearch.entity().isPublicTemplate(), SearchCriteria.Op.EQ); AccountIdSearch.done(); - SearchBuilder tmpltZoneSearch = _templateZoneDao.createSearchBuilder(); + SearchBuilder tmpltZoneSearch = templateZoneDao.createSearchBuilder(); tmpltZoneSearch.and("removed", tmpltZoneSearch.entity().getRemoved(), SearchCriteria.Op.NULL); tmpltZoneSearch.and("zoneId", tmpltZoneSearch.entity().getZoneId(), SearchCriteria.Op.EQ); @@ -367,6 +371,11 @@ public class ImageDataDaoImpl extends GenericDaoBase implemen CountTemplatesByAccount.and("removed", CountTemplatesByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); CountTemplatesByAccount.done(); + updateStateSearch = this.createSearchBuilder(); + updateStateSearch.and("id", updateStateSearch.entity().getId(), Op.EQ); + updateStateSearch.and("state", updateStateSearch.entity().getState(), Op.EQ); + updateStateSearch.and("updatedCount", updateStateSearch.entity().getUpdatedCount(), Op.EQ); + updateStateSearch.done(); return result; } @@ -572,7 +581,7 @@ public class ImageDataDaoImpl extends GenericDaoBase implemen for (Account account : permittedAccounts) { // accountType = account.getType(); // accountId = Long.toString(account.getId()); - DomainVO accountDomain = _domainDao.findById(account.getDomainId()); + DomainVO accountDomain = domainDao.findById(account.getDomainId()); // get all parent domain ID's all the way till root domain DomainVO domainTreeNode = accountDomain; @@ -580,7 +589,7 @@ public class ImageDataDaoImpl extends GenericDaoBase implemen relatedDomainIds.append(domainTreeNode.getId()); relatedDomainIds.append(","); if (domainTreeNode.getParent() != null) { - domainTreeNode = _domainDao.findById(domainTreeNode.getParent()); + domainTreeNode = domainDao.findById(domainTreeNode.getParent()); } else { break; } @@ -588,7 +597,7 @@ public class ImageDataDaoImpl extends GenericDaoBase implemen // get all child domain ID's if (isAdmin(account.getType())) { - List allChildDomains = _domainDao.findAllChildren(accountDomain.getPath(), accountDomain.getId()); + List allChildDomains = domainDao.findAllChildren(accountDomain.getPath(), accountDomain.getId()); for (DomainVO childDomain : allChildDomains) { relatedDomainIds.append(childDomain.getId()); relatedDomainIds.append(","); @@ -779,7 +788,7 @@ public class ImageDataDaoImpl extends GenericDaoBase implemen } private String getOrderByLimit(Long pageSize, Long startIndex) { - Boolean isAscending = Boolean.parseBoolean(_configDao.getValue("sortkey.algorithm")); + Boolean isAscending = Boolean.parseBoolean(configDao.getValue("sortkey.algorithm")); isAscending = (isAscending == null ? true : isAscending); String sql; @@ -806,17 +815,17 @@ public class ImageDataDaoImpl extends GenericDaoBase implemen throw new CloudRuntimeException("Failed to persist the template " + tmplt); } if (tmplt.getDetails() != null) { - _templateDetailsDao.persist(tmplt.getId(), tmplt.getDetails()); + templateDetailsDao.persist(tmplt.getId(), tmplt.getDetails()); } } - VMTemplateZoneVO tmpltZoneVO = _templateZoneDao.findByZoneTemplate(zoneId, tmplt.getId()); + VMTemplateZoneVO tmpltZoneVO = templateZoneDao.findByZoneTemplate(zoneId, tmplt.getId()); if (tmpltZoneVO == null) { tmpltZoneVO = new VMTemplateZoneVO(zoneId, tmplt.getId(), new Date()); - _templateZoneDao.persist(tmpltZoneVO); + templateZoneDao.persist(tmpltZoneVO); } else { tmpltZoneVO.setRemoved(null); tmpltZoneVO.setLastUpdated(new Date()); - _templateZoneDao.update(tmpltZoneVO.getId(), tmpltZoneVO); + templateZoneDao.update(tmpltZoneVO.getId(), tmpltZoneVO); } txn.commit(); @@ -931,7 +940,36 @@ public class ImageDataDaoImpl extends GenericDaoBase implemen @Override public boolean updateState(TemplateState currentState, TemplateEvent event, TemplateState nextState, ImageDataVO vo, Object data) { - // TODO Auto-generated method stub - return false; + Long oldUpdated = vo.getUpdatedCount(); + Date oldUpdatedTime = vo.getUpdated(); + + + SearchCriteria sc = updateStateSearch.create(); + sc.setParameters("id", vo.getId()); + sc.setParameters("state", currentState); + sc.setParameters("updatedCount", vo.getUpdatedCount()); + + vo.incrUpdatedCount(); + + UpdateBuilder builder = getUpdateBuilder(vo); + builder.set(vo, "state", nextState); + builder.set(vo, "updated", new Date()); + + int rows = update((ImageDataVO) vo, sc); + if (rows == 0 && s_logger.isDebugEnabled()) { + ImageDataVO dbVol = findByIdIncludingRemoved(vo.getId()); + if (dbVol != null) { + StringBuilder str = new StringBuilder("Unable to update ").append(vo.toString()); + str.append(": DB Data={id=").append(dbVol.getId()).append("; state=").append(dbVol.getState()).append("; updatecount=").append(dbVol.getUpdatedCount()).append(";updatedTime=") + .append(dbVol.getUpdated()); + str.append(": New Data={id=").append(vo.getId()).append("; state=").append(nextState).append("; event=").append(event).append("; updatecount=").append(vo.getUpdatedCount()) + .append("; updatedTime=").append(vo.getUpdated()); + str.append(": stale Data={id=").append(vo.getId()).append("; state=").append(currentState).append("; event=").append(event).append("; updatecount=").append(oldUpdated) + .append("; updatedTime=").append(oldUpdatedTime); + } else { + s_logger.debug("Unable to update objectIndatastore: id=" + vo.getId() + ", as there is no such object exists in the database anymore"); + } + } + return rows > 0; } } \ No newline at end of file diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageDataVO.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageDataVO.java index 38025969053..e3ddaed721a 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageDataVO.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageDataVO.java @@ -136,10 +136,17 @@ public class ImageDataVO implements Identity, StateObject { private long imageDataStoreId; @Column(name = "size") - private long size; + private Long size; @Column(name = "state") private TemplateState state; + + @Column(name="update_count", updatable = true) + protected long updatedCount; + + @Column(name = "updated") + @Temporal(value = TemporalType.TIMESTAMP) + Date updated; @Transient Map details; @@ -408,16 +415,36 @@ public class ImageDataVO implements Identity, StateObject { this.imageDataStoreId = dataStoreId; } - public void setSize(long size) { + public void setSize(Long size) { this.size = size; } - public long getSize() { + public Long getSize() { return this.size; } public TemplateState getState() { return this.state; } + + public long getUpdatedCount() { + return this.updatedCount; + } + + public void incrUpdatedCount() { + this.updatedCount++; + } + + public void decrUpdatedCount() { + this.updatedCount--; + } + + public Date getUpdated() { + return updated; + } + + public void setUpdated(Date updated) { + this.updated = updated; + } } diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/DefaultPrimaryDataStore.java b/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/DefaultPrimaryDataStore.java index 0f11c42fdc0..9c009c95623 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/DefaultPrimaryDataStore.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/DefaultPrimaryDataStore.java @@ -25,6 +25,7 @@ import javax.inject.Inject; import org.apache.cloudstack.engine.datacenter.entity.api.DataCenterResourceEntity.State; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectType; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreRole; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle; @@ -36,6 +37,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.disktype.DiskFormat; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreVO; import org.apache.cloudstack.storage.datastore.provider.DataStoreProvider; +import org.apache.cloudstack.storage.db.ObjectInDataStoreVO; import org.apache.cloudstack.storage.image.ImageDataFactory; import org.apache.cloudstack.storage.image.TemplateInfo; import org.apache.cloudstack.storage.snapshot.SnapshotDataFactory; @@ -48,6 +50,7 @@ import org.apache.log4j.Logger; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.storage.encoding.EncodingType; public class DefaultPrimaryDataStore implements PrimaryDataStore { private static final Logger s_logger = Logger @@ -126,11 +129,16 @@ public class DefaultPrimaryDataStore implements PrimaryDataStore { public String getUri() { String path = this.pdsv.getPath(); path.replaceFirst("/*", ""); - return this.pdsv.getPoolType() + ":" + File.separator + File.separator - + this.pdsv.getHostAddress() + File.separator - + this.pdsv.getPath() + File.separator - + "?role=" + this.getRole() - + "&storeUuid=" + this.pdsv.getUuid(); + StringBuilder builder = new StringBuilder(); + builder.append(this.pdsv.getPoolType()); + builder.append("://"); + builder.append(this.pdsv.getHostAddress()); + builder.append(File.separator); + builder.append(this.pdsv.getPath()); + builder.append(File.separator); + builder.append("?" + EncodingType.ROLE + "=" + this.getRole()); + builder.append("&" + EncodingType.STOREUUID + "=" + this.pdsv.getUuid()); + return builder.toString(); } @Override @@ -212,6 +220,10 @@ public class DefaultPrimaryDataStore implements PrimaryDataStore { @Override public TemplateInfo getTemplate(long templateId) { + ObjectInDataStoreVO obj = objectInStoreMgr.findObject(templateId, DataObjectType.TEMPLATE, this.getId(), this.getRole()); + if (obj == null) { + return null; + } return imageDataFactory.getTemplate(templateId, this); } diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/driver/DefaultPrimaryDataStoreDriverImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/driver/DefaultPrimaryDataStoreDriverImpl.java index 39b2aef86b2..dfe4518edab 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/driver/DefaultPrimaryDataStoreDriverImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/driver/DefaultPrimaryDataStoreDriverImpl.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.storage.datastore.driver; +import java.net.URISyntaxException; import java.util.Set; import javax.inject.Inject; @@ -29,17 +30,20 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcConext; -import org.apache.cloudstack.storage.command.CreateObjectCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; +import org.apache.cloudstack.storage.command.CreateObjectCommand; import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.datastore.DataObjectManager; import org.apache.cloudstack.storage.endpoint.EndPointSelector; import org.apache.cloudstack.storage.snapshot.SnapshotInfo; import org.apache.cloudstack.storage.volume.PrimaryDataStoreDriver; import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; -import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.storage.encoding.DecodedDataObject; +import com.cloud.utils.storage.encoding.Decoder; public class DefaultPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver { @@ -48,6 +52,8 @@ public class DefaultPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver EndPointSelector selector; @Inject StoragePoolHostDao storeHostDao; + @Inject + DataObjectManager dataObjMgr; public DefaultPrimaryDataStoreDriverImpl() { } @@ -158,7 +164,7 @@ public class DefaultPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver EndPoint ep = selector.select(vol); CreateObjectCommand createCmd = new CreateObjectCommand(vol.getUri()); - CreateVolumeContext context = null; + CreateVolumeContext context = new CreateVolumeContext(callback, vol); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setContext(context) .setCallback(caller.getTarget().createAsyncCallback(null, null)); @@ -168,8 +174,28 @@ public class DefaultPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver @Override public String grantAccess(DataObject object, EndPoint ep) { - StoragePoolHostVO poolHost = storeHostDao.findByPoolHost(object.getDataStore().getId(), ep.getId()); - return object.getUri() + "&storagePath=" + poolHost.getLocalPath(); + //StoragePoolHostVO poolHost = storeHostDao.findByPoolHost(object.getDataStore().getId(), ep.getId()); + + String uri = object.getUri(); + try { + DecodedDataObject obj = Decoder.decode(uri); + if (obj.getPath() == null) { + //create an obj + EndPoint newEp = selector.select(object); + CreateObjectCommand createCmd = new CreateObjectCommand(uri); + CreateObjectAnswer answer = (CreateObjectAnswer)ep.sendMessage(createCmd); + if (answer.getResult()) { + dataObjMgr.update(object, answer.getPath(), answer.getSize()); + } else { + s_logger.debug("failed to create object" + answer.getDetails()); + throw new CloudRuntimeException("failed to create object" + answer.getDetails()); + } + } + + return object.getUri(); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("uri parsed error", e); + } } @Override diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/TemplateInstallStrategyImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/TemplateInstallStrategyImpl.java index ec6b0326aeb..80e098d769a 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/TemplateInstallStrategyImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/TemplateInstallStrategyImpl.java @@ -106,8 +106,8 @@ public class TemplateInstallStrategyImpl implements TemplateInstallStrategy { boolean freshNewTemplate = false; if (obj == null) { try { - templateOnPrimaryStoreObj = objectInDataStoreMgr.create( - template, store); + /*templateOnPrimaryStoreObj = objectInDataStoreMgr.create( + template, store);*/ freshNewTemplate = true; } catch (Throwable e) { obj = objectInDataStoreMgr.findObject(template.getId(), @@ -201,12 +201,13 @@ public class TemplateInstallStrategyImpl implements TemplateInstallStrategy { .getRole()); obj.setInstallPath(result.getPath()); + obj.setSize(result.getSize()); try { - objectInDataStoreMgr.update(templateOnPrimaryStoreObj, + objectInDataStoreMgr.update(obj, ObjectInDataStoreStateMachine.Event.OperationSuccessed); } catch (NoTransitionException e) { try { - objectInDataStoreMgr.update(templateOnPrimaryStoreObj, + objectInDataStoreMgr.update(obj, ObjectInDataStoreStateMachine.Event.OperationFailed); } catch (NoTransitionException e1) { s_logger.debug("failed to change state", e1); @@ -273,7 +274,7 @@ public class TemplateInstallStrategyImpl implements TemplateInstallStrategy { CreateBaseImageResult res = new CreateBaseImageResult( templateOnPrimaryStoreObj); try { - objectInDataStoreMgr.update(templateOnPrimaryStoreObj, + objectInDataStoreMgr.update(obj, ObjectInDataStoreStateMachine.Event.OperationSuccessed); } catch (NoTransitionException e) { s_logger.debug("Failed to update copying state: ", e); diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java index 38c35f811cd..64af097bb32 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java @@ -44,7 +44,8 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory { VolumeVO volumeVO = volumeDao.findById(volumeId); ObjectInDataStoreVO obj = objMap.findObject(volumeId, DataObjectType.VOLUME, store.getId(), store.getRole()); if (obj == null) { - return null; + VolumeObject vol = VolumeObject.getVolumeObject(null, volumeVO); + return vol; } VolumeObject vol = VolumeObject.getVolumeObject(store, volumeVO); return vol; diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeObject.java index 5ae587c323c..9e04909135e 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -16,8 +16,6 @@ // under the License. package org.apache.cloudstack.storage.volume; -import java.io.File; - import javax.inject.Inject; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectType; @@ -35,6 +33,7 @@ import com.cloud.utils.component.ComponentContext; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.utils.fsm.StateMachine2; +import com.cloud.utils.storage.encoding.EncodingType; public class VolumeObject implements VolumeInfo { private static final Logger s_logger = Logger.getLogger(VolumeObject.class); @@ -48,14 +47,18 @@ public class VolumeObject implements VolumeInfo { @Inject ObjectInDataStoreManager ojbectInStoreMgr; - private VolumeObject(DataStore dataStore, VolumeVO volumeVO) { + protected VolumeObject() { + + } + + protected void configure(DataStore dataStore, VolumeVO volumeVO) { this.volumeVO = volumeVO; this.dataStore = dataStore; } public static VolumeObject getVolumeObject(DataStore dataStore, VolumeVO volumeVO) { - VolumeObject vo = new VolumeObject(dataStore, volumeVO); - vo = ComponentContext.inject(vo); + VolumeObject vo = ComponentContext.inject(VolumeObject.class); + vo.configure(dataStore, volumeVO); return vo; } @@ -121,9 +124,14 @@ public class VolumeObject implements VolumeInfo { } ObjectInDataStoreVO obj = ojbectInStoreMgr.findObject(this.volumeVO.getId(), DataObjectType.VOLUME, this.dataStore.getId(), this.dataStore.getRole()); if (obj.getState() != ObjectInDataStoreStateMachine.State.Ready) { - return this.dataStore.getUri() + File.separator + "&objType=" + DataObjectType.VOLUME + "&size=" + this.volumeVO.getSize() + "&name=" + this.volumeVO.getName(); + return this.dataStore.getUri() + + "&" + EncodingType.OBJTYPE + "=" + DataObjectType.VOLUME + + "&" + EncodingType.SIZE + "=" + this.volumeVO.getSize() + + "&" + EncodingType.NAME + "=" + this.volumeVO.getName(); } else { - return this.dataStore.getUri() + File.separator + "&objType=" + DataObjectType.VOLUME + "&path=" + obj.getInstallPath(); + return this.dataStore.getUri() + + "&" + EncodingType.OBJTYPE + "=" + DataObjectType.VOLUME + + "&" + EncodingType.PATH + "=" + obj.getInstallPath(); } } diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index f3da3680a7e..8cfbae455e7 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -22,7 +22,8 @@ import javax.inject.Inject; import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity; import org.apache.cloudstack.engine.subsystem.api.storage.CommandResult; -import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; +import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; @@ -31,6 +32,7 @@ import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcConext; +import org.apache.cloudstack.storage.datastore.DataObjectManager; import org.apache.cloudstack.storage.datastore.ObjectInDataStoreManager; import org.apache.cloudstack.storage.datastore.PrimaryDataStore; import org.apache.cloudstack.storage.datastore.PrimaryDataStoreProviderManager; @@ -55,6 +57,8 @@ public class VolumeServiceImpl implements VolumeService { @Inject ObjectInDataStoreManager objectInDataStoreMgr; @Inject + DataObjectManager dataObjectMgr; + @Inject ImageMotionService imageMotion; @Inject TemplateInstallStrategy templateInstallStrategy; @@ -85,6 +89,8 @@ public class VolumeServiceImpl implements VolumeService { } + + @Override public AsyncCallFuture createVolumeAsync(VolumeInfo volume, long dataStoreId) { PrimaryDataStore dataStore = dataStoreMgr.getPrimaryDataStore(dataStoreId); @@ -107,16 +113,16 @@ public class VolumeServiceImpl implements VolumeService { vo.stateTransit(Volume.Event.CreateRequested); CreateVolumeContext context = new CreateVolumeContext(null, vo, future); - AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createVolumeCallback(null, null)) .setContext(context); - //dataStore.createVolumeAsync(vo, null, caller); + dataObjectMgr.createAsync(volume, dataStore, caller, true); return future; } - protected Void createVolumeCallback(AsyncCallbackDispatcher callback, CreateVolumeContext context) { - CommandResult result = callback.getResult(); + protected Void createVolumeCallback(AsyncCallbackDispatcher callback, CreateVolumeContext context) { + CreateCmdResult result = callback.getResult(); VolumeObject vo = context.getVolume(); VolumeApiResult volResult = new VolumeApiResult(vo); if (result.isSuccess()) { @@ -172,7 +178,7 @@ public class VolumeServiceImpl implements VolumeService { caller.setCallback(caller.getTarget().deleteVolumeCallback(null, null)) .setContext(context); - dataStore.getDriver().deleteAsync(volume, caller); + dataObjectMgr.deleteAsync(volume, caller); return future; } @@ -279,21 +285,23 @@ public class VolumeServiceImpl implements VolumeService { @DB protected void createBaseImageAsync(VolumeInfo volume, PrimaryDataStore dataStore, TemplateInfo template, AsyncCallFuture future) { - CreateBaseImageContext context = new CreateBaseImageContext(null, volume, + CreateBaseImageContext context = new CreateBaseImageContext(null, volume, dataStore, template, future); - AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); - caller.setCallback(caller.getTarget().createBaseImageCallback(null, null)) + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); + caller.setCallback(caller.getTarget().copyBaseImageCallback(null, null)) .setContext(context); - - templateInstallStrategy.installAsync(template, dataStore, caller); + DataObject templateOnPrimaryStoreObj = dataObjectMgr.createInternalStateOnly(template, dataStore); + + dataObjectMgr.copyAsync(context.srcTemplate, templateOnPrimaryStoreObj, caller); + return; } - + @DB - protected Void createBaseImageCallback(AsyncCallbackDispatcher callback, CreateBaseImageContext context) { - CreateBaseImageResult result = callback.getResult(); + protected Void copyBaseImageCallback(AsyncCallbackDispatcher callback, CreateBaseImageContext context) { + CreateCmdResult result = callback.getResult(); VolumeApiResult res = new VolumeApiResult(context.getVolume()); AsyncCallFuture future = context.getFuture(); @@ -302,23 +310,26 @@ public class VolumeServiceImpl implements VolumeService { future.complete(res); return null; } - - //now create volume on base image - TemplateInfo templateOnPrimaryStoreObj = result.template; - VolumeInfo volume = context.getVolume(); - PrimaryDataStore pd = context.getDataStore(); + DataObject templateOnPrimaryStoreObj = objectInDataStoreMgr.get(context.srcTemplate, context.dataStore); - createVolumeFromBaseImageAsync(volume, templateOnPrimaryStoreObj, pd, future); + createVolumeFromBaseImageAsync(context.volume, templateOnPrimaryStoreObj, context.dataStore, future); return null; } private class CreateVolumeFromBaseImageContext extends AsyncRpcConext { private final VolumeObject vo; private final AsyncCallFuture future; - public CreateVolumeFromBaseImageContext(AsyncCompletionCallback callback, VolumeObject vo, AsyncCallFuture future) { + private final DataStore primaryStore; + private final DataObject templateOnStore; + public CreateVolumeFromBaseImageContext(AsyncCompletionCallback callback, VolumeObject vo, + DataStore primaryStore, + DataObject templateOnStore, + AsyncCallFuture future) { super(callback); this.vo = vo; this.future = future; + this.primaryStore = primaryStore; + this.templateOnStore = templateOnStore; } public VolumeObject getVolumeObject() { @@ -331,7 +342,7 @@ public class VolumeServiceImpl implements VolumeService { } @DB - protected void createVolumeFromBaseImageAsync(VolumeInfo volume, TemplateInfo templateOnPrimaryStore, PrimaryDataStore pd, AsyncCallFuture future) { + protected void createVolumeFromBaseImageAsync(VolumeInfo volume, DataObject templateOnPrimaryStore, PrimaryDataStore pd, AsyncCallFuture future) { VolumeObject vo = (VolumeObject) volume; try { vo.stateTransit(Volume.Event.CreateRequested); @@ -342,19 +353,22 @@ public class VolumeServiceImpl implements VolumeService { return; } - CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext(null, vo, future); - AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); - caller.setCallback(caller.getTarget().createVolumeFromBaseImageCallback(null, null)) + CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext(null, vo, pd, templateOnPrimaryStore, future); + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); + caller.setCallback(caller.getTarget().copyBaseImageCallBack(null, null)) .setContext(context); - pd.getDriver().copyAsync(volume, templateOnPrimaryStore, caller); + DataObject volumeOnPrimaryStorage = dataObjectMgr.createInternalStateOnly(volume, pd); + dataObjectMgr.copyAsync(context.templateOnStore, volumeOnPrimaryStorage, caller); + return; } @DB - public Object createVolumeFromBaseImageCallback(AsyncCallbackDispatcher callback, CreateVolumeFromBaseImageContext context) { + public Void copyBaseImageCallBack(AsyncCallbackDispatcher callback, CreateVolumeFromBaseImageContext context) { VolumeObject vo = context.getVolumeObject(); - CopyCommandResult result = callback.getResult(); + CreateCmdResult result = callback.getResult(); VolumeApiResult volResult = new VolumeApiResult(vo); + if (result.isSuccess()) { if (result.getPath() != null) { vo.setPath(result.getPath()); diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java index 1aa96c39980..06367afe2c8 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java @@ -28,22 +28,18 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; import org.apache.cloudstack.storage.command.AttachPrimaryDataStoreAnswer; import org.apache.cloudstack.storage.command.AttachPrimaryDataStoreCmd; import org.apache.cloudstack.storage.command.CopyCmd; import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.command.CreateObjectCommand; import org.apache.cloudstack.storage.command.CreatePrimaryDataStoreCmd; -import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.command.CreateVolumeFromBaseImageCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.datastore.protocol.DataStoreProtocol; import org.apache.cloudstack.storage.to.ImageOnPrimayDataStoreTO; -import org.apache.cloudstack.storage.to.NfsPrimaryDataStoreTO; -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.storage.to.TemplateTO; import org.apache.cloudstack.storage.to.VolumeTO; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -57,8 +53,10 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.storage.DeleteVolumeCommand; import com.cloud.hypervisor.xen.resource.CitrixResourceBase.SRType; -import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.storage.encoding.DataStore; +import com.cloud.utils.storage.encoding.DecodedDataObject; +import com.cloud.utils.storage.encoding.Decoder; import com.xensource.xenapi.Connection; import com.xensource.xenapi.Host; import com.xensource.xenapi.PBD; @@ -137,28 +135,23 @@ public class XenServerStorageResource { } protected CreateObjectAnswer execute(CreateObjectCommand cmd) { String uriString = cmd.getObjectUri(); - Map params = null; - - try { - URI uri = new URI(uriString); - params = getParameters(uri); - } catch (URISyntaxException e1) { - s_logger.debug("uri exception", e1); - return new CreateObjectAnswer(cmd, false, e1.toString()); - } - - if (params.get("objType").equalsIgnoreCase("template")) { - return getTemplateSize(cmd, params.get("path")); - } - - long size = Long.parseLong(params.get("size")); - String name = params.get("name"); - String storeUuid = params.get("storagePath"); + DecodedDataObject obj = null; + Connection conn = hypervisorResource.getConnection(); VDI vdi = null; boolean result = false; String errorMsg = null; + try { + obj = Decoder.decode(uriString); + DataStore store = obj.getStore(); + if (obj.getObjType().equalsIgnoreCase("template") && store.getRole().equalsIgnoreCase("image")) { + return getTemplateSize(cmd, obj.getPath()); + } + + long size = obj.getSize(); + String name = obj.getName(); + String storeUuid = store.getUuid(); SR primaryDataStoreSR = getSRByNameLabel(conn, storeUuid); vdi = createVdi(conn, name, primaryDataStoreSR, size); VDI.Record record = vdi.getRecord(conn); @@ -173,6 +166,9 @@ public class XenServerStorageResource { } catch (XmlRpcException e) { s_logger.debug("Failed to create volume", e); errorMsg = e.toString(); + } catch (URISyntaxException e) { + s_logger.debug("Failed to create volume", e); + errorMsg = e.toString(); } finally { if (!result && vdi != null) { try { @@ -227,13 +223,13 @@ public class XenServerStorageResource { } } - protected SR getNfsSR(Connection conn, URI uri) { + protected SR getNfsSR(Connection conn, DataStore store) { Map deviceConfig = new HashMap(); - Map params = getParameters(uri); - String uuid = params.get("storeUuid"); + + String uuid = store.getUuid(); try { - String server = uri.getHost(); - String serverpath = uri.getPath(); + String server = store.getServer(); + String serverpath = store.getPath(); serverpath = serverpath.replace("//", "/"); Set srs = SR.getAll(conn); @@ -410,25 +406,17 @@ public class XenServerStorageResource { protected Answer execute(CreatePrimaryDataStoreCmd cmd) { Connection conn = hypervisorResource.getConnection(); String storeUrl = cmd.getDataStore(); - String scheme = null; - String type = null; - URI storeUri = null; + try { - storeUri = new URI(storeUrl); - } catch(URISyntaxException e) { - return new Answer(cmd, false, e.toString()); - } - - scheme = storeUri.getScheme(); - - try { - if (scheme.equalsIgnoreCase("nfs")) { - SR sr = getNfsSR(conn, storeUri); - } else if (scheme.equalsIgnoreCase("iscsi")) { + DecodedDataObject obj = Decoder.decode(storeUrl); + DataStore store = obj.getStore(); + if (store.getScheme().equalsIgnoreCase("nfs")) { + SR sr = getNfsSR(conn, store); + } else if (store.getScheme().equalsIgnoreCase("iscsi")) { //getIscsiSR(conn, dataStore); - } else if (scheme.equalsIgnoreCase("presetup")) { + } else if (store.getScheme().equalsIgnoreCase("presetup")) { } else { - return new Answer(cmd, false, "The pool type: " + scheme + " is not supported."); + return new Answer(cmd, false, "The pool type: " + store.getScheme() + " is not supported."); } return new Answer(cmd, true, "success"); } catch (Exception e) { @@ -509,31 +497,30 @@ public class XenServerStorageResource { } - protected Answer directDownloadHttpTemplate(CopyCmd cmd, TemplateTO template, PrimaryDataStoreTO primarDataStore) { - String primaryStoreUuid = primarDataStore.getUuid(); + protected Answer directDownloadHttpTemplate(CopyCmd cmd, DecodedDataObject srcObj, DecodedDataObject destObj) { Connection conn = hypervisorResource.getConnection(); SR poolsr = null; VDI vdi = null; boolean result = false; try { - - Set srs = SR.getByNameLabel(conn, primaryStoreUuid); + if (destObj.getPath() == null) { + //need to create volume at first + + } + vdi = VDI.getByUuid(conn, destObj.getPath()); + if (vdi == null) { + throw new CloudRuntimeException("can't find volume: " + destObj.getPath()); + } + String destStoreUuid = destObj.getStore().getUuid(); + Set srs = SR.getByNameLabel(conn, destStoreUuid); if (srs.size() != 1) { - throw new CloudRuntimeException("storage uuid: " + primaryStoreUuid + " is not unique"); + throw new CloudRuntimeException("storage uuid: " + destStoreUuid + " is not unique"); } poolsr = srs.iterator().next(); - VDI.Record vdir = new VDI.Record(); - vdir.nameLabel = "Base-Image-" + UUID.randomUUID().toString(); - vdir.SR = poolsr; - vdir.type = Types.VdiType.USER; - - vdir.virtualSize = getTemplateSize(conn, template.getPath()); - vdi = VDI.create(conn, vdir); - - vdir = vdi.getRecord(conn); + VDI.Record vdir = vdi.getRecord(conn); String vdiLocation = vdir.location; String pbdLocation = null; - if (primarDataStore.getType().equalsIgnoreCase(DataStoreProtocol.NFS.toString())) { + if (destObj.getStore().getScheme().equalsIgnoreCase(DataStoreProtocol.NFS.toString())) { pbdLocation = "/run/sr-mount/" + poolsr.getUuid(conn); } else { Set pbds = poolsr.getPBDs(conn); @@ -551,7 +538,7 @@ public class XenServerStorageResource { String vdiPath = pbdLocation + "/" + vdiLocation + ".vhd"; //download a url into vdipath //downloadHttpToLocalFile(vdiPath, template.getPath()); - hypervisorResource.callHostPlugin(conn, "storagePlugin", "downloadTemplateFromUrl", "destPath", vdiPath, "srcUrl", template.getPath()); + hypervisorResource.callHostPlugin(conn, "storagePlugin", "downloadTemplateFromUrl", "destPath", vdiPath, "srcUrl", srcObj.getPath()); result = true; return new CopyCmdAnswer(cmd, vdi.getUuid(conn)); } catch (BadServerResponse e) { @@ -581,16 +568,10 @@ public class XenServerStorageResource { protected Answer execute(AttachPrimaryDataStoreCmd cmd) { String dataStoreUri = cmd.getDataStore(); Connection conn = hypervisorResource.getConnection(); - Map params = null; try { - try { - URI uri = new URI(dataStoreUri); - params = getParameters(uri); - } catch (URISyntaxException e1) { - s_logger.debug("uri exception", e1); - return new CreateObjectAnswer(cmd, false, e1.toString()); - } - SR sr = hypervisorResource.getStorageRepository(conn, params.get("storeUuid")); + DecodedDataObject obj = Decoder.decode(dataStoreUri); + DataStore store = obj.getStore(); + SR sr = hypervisorResource.getStorageRepository(conn, store.getUuid()); hypervisorResource.setupHeartbeatSr(conn, sr, false); long capacity = sr.getPhysicalSize(conn); long available = capacity - sr.getPhysicalUtilisation(conn); @@ -616,10 +597,18 @@ public class XenServerStorageResource { } protected Answer execute(CopyCmd cmd) { - ImageOnPrimayDataStoreTO imageTO = cmd.getImage(); - TemplateTO template = imageTO.getTemplate(); - if (template.getPath().startsWith("http")) { - return directDownloadHttpTemplate(cmd, template, imageTO.getPrimaryDataStore()); + DecodedDataObject srcObj = null; + DecodedDataObject destObj = null; + try { + srcObj = Decoder.decode(cmd.getSrcUri()); + destObj = Decoder.decode(cmd.getDestUri()); + } catch (URISyntaxException e) { + return new Answer(cmd, false, e.toString()); + } + + + if (srcObj.getPath().startsWith("http")) { + return directDownloadHttpTemplate(cmd, srcObj, destObj); } else { return new Answer(cmd, false, "not implemented yet"); /* diff --git a/server/src/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java b/server/src/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java index e066dff3840..04b553c2f8b 100644 --- a/server/src/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java +++ b/server/src/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java @@ -37,7 +37,7 @@ public class VMTemplateDetailsDaoImpl extends GenericDaoBase TemplateSearch; protected final SearchBuilder DetailSearch; - protected VMTemplateDetailsDaoImpl() { + public VMTemplateDetailsDaoImpl() { TemplateSearch = createSearchBuilder(); TemplateSearch.and("templateId", TemplateSearch.entity().getTemplateId(), SearchCriteria.Op.EQ); TemplateSearch.done(); diff --git a/setup/db/4.1-new-db-schema.sql b/setup/db/4.1-new-db-schema.sql index d5415373eec..e8bafa22bef 100644 --- a/setup/db/4.1-new-db-schema.sql +++ b/setup/db/4.1-new-db-schema.sql @@ -18,6 +18,8 @@ alter table vm_template add image_data_store_id bigint unsigned; alter table vm_template add size bigint unsigned; alter table vm_template add state varchar(255); +alter table vm_template add update_count bigint unsigned; +alter table vm_template add updated datetime; alter table storage_pool add storage_provider_id bigint unsigned; alter table storage_pool add scope varchar(255); alter table storage_pool modify id bigint unsigned AUTO_INCREMENT UNIQUE NOT NULL; @@ -67,7 +69,7 @@ CREATE TABLE `cloud`.`object_datastore_ref` ( `size` bigint unsigned COMMENT 'the size of the template on the pool', `state` varchar(255) NOT NULL, `update_count` bigint unsigned NOT NULL, - `updated` DATETIME NOT NULL, + `updated` DATETIME, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;