apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix a bug in create cache object for S3.

Browse Source
This commit is contained in:
Min Chen 2013-05-10 17:53:32 -07:00
parent a83b87b7e5
commit 8a9a7a4adc
8 changed files with 138 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
core/src/com/cloud/storage/resource/LocalNfsSecondaryStorageResource.java
View File

@ -4,15 +4,18 @@ import static com.cloud.utils.StringUtils.join;
import static java.lang.String.format;
import static java.util.Arrays.asList;
import java.io.File;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import org.apache.cloudstack.storage.command.DownloadSystemTemplateCommand;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import com.amazonaws.services.s3.model.S3ObjectSummary;
@ -23,19 +26,26 @@ import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;
import com.cloud.agent.api.to.SwiftTO;
import com.cloud.storage.JavaStorageLayer;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.template.DownloadManagerImpl;
import com.cloud.storage.template.DownloadManagerImpl.ZfsPathParser;
import com.cloud.utils.S3Utils;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
@Component
public class LocalNfsSecondaryStorageResource extends
NfsSecondaryStorageResource {
private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
public LocalNfsSecondaryStorageResource(){
this._dlMgr = new DownloadManagerImpl();
((DownloadManagerImpl)_dlMgr).setThreadPool(Executors.newFixedThreadPool(10));
_storage = new JavaStorageLayer();
this._inSystemVM = false;
}
@Override
@ -100,4 +110,73 @@ public class LocalNfsSecondaryStorageResource extends
return new Answer(cmd, false, "Unsupported image data store: " + dstore);
}
}
@Override
protected String mount(String root, String nfsPath) {
File file = new File(root);
if (!file.exists()) {
if (_storage.mkdir(root)) {
s_logger.debug("create mount point: " + root);
} else {
s_logger.debug("Unable to create mount point: " + root);
return null;
}
}
Script script = null;
String result = null;
script = new Script(!_inSystemVM, "mount", _timeout, s_logger);
List<String> res = new ArrayList<String>();
ZfsPathParser parser = new ZfsPathParser(root);
script.execute(parser);
res.addAll(parser.getPaths());
for (String s : res) {
if (s.contains(root)) {
return root;
}
}
Script command = new Script(!_inSystemVM, "mount", _timeout, s_logger);
command.add("-t", "nfs");
if ("Mac OS X".equalsIgnoreCase(System.getProperty("os.name"))) {
command.add("-o", "resvport");
}
if (_inSystemVM) {
// Fedora Core 12 errors out with any -o option executed from java
command.add("-o", "soft,timeo=133,retrans=2147483647,tcp,acdirmax=0,acdirmin=0");
}
command.add(nfsPath);
command.add(root);
result = command.execute();
if (result != null) {
s_logger.warn("Unable to mount " + nfsPath + " due to " + result);
file = new File(root);
if (file.exists())
file.delete();
return null;
}
// Change permissions for the mountpoint
script = new Script(true, "chmod", _timeout, s_logger);
script.add("777", root);
result = script.execute();
if (result != null) {
s_logger.warn("Unable to set permissions for " + root + " due to " + result);
return null;
}
// XXX: Adding the check for creation of snapshots dir here. Might have
// to move it somewhere more logical later.
if (!checkForSnapshotsDir(root)) {
return null;
}
// Create the volumes dir
if (!checkForVolumesDir(root)) {
return null;
}
return root;
}
}

2
core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
View File

@ -148,7 +148,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
String _role;
Map<String, Object> _params;
protected StorageLayer _storage;
boolean _inSystemVM = false;
protected boolean _inSystemVM = false;
boolean _sslCopy = false;
protected DownloadManager _dlMgr;

10
engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/HostScope.java
View File

@ -24,8 +24,11 @@ import com.cloud.storage.ScopeType;
public class HostScope extends AbstractScope {
private ScopeType type = ScopeType.HOST;
private Long hostId;
public HostScope(Long hostId) {
private Long zoneId;
public HostScope(Long hostId, Long zoneId) {
this.hostId = hostId;
this.zoneId = zoneId;
}
@Override
public ScopeType getScopeType() {
@ -36,4 +39,9 @@ public class HostScope extends AbstractScope {
public Long getScopeId() {
return this.hostId;
}
public Long getZoneId() {
return zoneId;
}
}

18
engine/api/test/org/apache/cloudstack/engine/subsystem/api/storage/ScopeTest.java
View File

@ -30,28 +30,28 @@ public class ScopeTest {
ZoneScope zoneScope = new ZoneScope(1L);
ZoneScope zoneScope2 = new ZoneScope(1L);
Assert.assertTrue(zoneScope.isSameScope(zoneScope2));
ZoneScope zoneScope3 = new ZoneScope(2L);
Assert.assertFalse(zoneScope.isSameScope(zoneScope3));
}
@Test
public void testClusterScope() {
ClusterScope clusterScope = new ClusterScope(1L, 1L, 1L);
ClusterScope clusterScope2 = new ClusterScope(1L, 1L, 1L);
Assert.assertTrue(clusterScope.isSameScope(clusterScope2));
ClusterScope clusterScope3 = new ClusterScope(2L, 2L, 1L);
Assert.assertFalse(clusterScope.isSameScope(clusterScope3));
}
@Test
public void testHostScope() {
HostScope hostScope = new HostScope(1L);
HostScope hostScope2 = new HostScope(1L);
HostScope hostScope3 = new HostScope(2L);
HostScope hostScope = new HostScope(1L, 1L);
HostScope hostScope2 = new HostScope(1L, 1L);
HostScope hostScope3 = new HostScope(2L, 1L);
Assert.assertTrue(hostScope.isSameScope(hostScope2));
Assert.assertFalse(hostScope.isSameScope(hostScope3));
}

52
engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
View File

@ -23,6 +23,7 @@ import java.util.List;
import javax.inject.Inject;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
@ -30,6 +31,9 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectType;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.DataTO;
import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
@ -137,7 +141,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
// TODO Auto-generated method stub
return true;
}
protected boolean needCacheStorage(DataObject srcData, DataObject destData) {
DataTO srcTO = srcData.getTO();
DataTO destTO = destData.getTO();
@ -146,7 +150,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
if (srcStoreTO instanceof NfsTO || srcStoreTO.getRole() == DataStoreRole.ImageCache) {
return false;
}
if (destStoreTO instanceof NfsTO || destStoreTO.getRole() == DataStoreRole.ImageCache) {
return false;
}
@ -161,7 +165,15 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
if (needCacheStorage(srcData, destData)) {
//need to copy it to image cache store
DataObject cacheData = cacheMgr.createCacheObject(srcData, destData.getDataStore().getScope());
Scope destScope = destData.getDataStore().getScope();
if (destScope instanceof ClusterScope){
ClusterScope clusterScope = (ClusterScope)destScope;
destScope = new ZoneScope(clusterScope.getZoneId());
} else if (destScope instanceof HostScope){
HostScope hostScope = (HostScope)destScope;
destScope = new ZoneScope(hostScope.getZoneId());
}
DataObject cacheData = cacheMgr.createCacheObject(srcData, destScope);
CopyCommand cmd = new CopyCommand(cacheData.getTO(), destData.getTO(), _copyvolumewait);
EndPoint ep = selector.select(cacheData, destData);
Answer answer = ep.sendMessage(cmd);
@ -180,7 +192,15 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
int _primaryStorageDownloadWait = NumbersUtil.parseInt(value, Integer.parseInt(Config.PrimaryStorageDownloadWait.getDefaultValue()));
if (needCacheStorage(srcData, destData)) {
//need to copy it to image cache store
DataObject cacheData = cacheMgr.createCacheObject(srcData, destData.getDataStore().getScope());
Scope destScope = destData.getDataStore().getScope();
if (destScope instanceof ClusterScope){
ClusterScope clusterScope = (ClusterScope)destScope;
destScope = new ZoneScope(clusterScope.getZoneId());
} else if (destScope instanceof HostScope){
HostScope hostScope = (HostScope)destScope;
destScope = new ZoneScope(hostScope.getZoneId());
}
DataObject cacheData = cacheMgr.createCacheObject(srcData, destScope);
CopyCommand cmd = new CopyCommand(cacheData.getTO(), destData.getTO(), _primaryStorageDownloadWait);
EndPoint ep = selector.select(cacheData, destData);
Answer answer = ep.sendMessage(cmd);
@ -193,7 +213,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
return answer;
}
}
protected DataObject cacheSnapshotChain(SnapshotInfo snapshot) {
DataObject leafData = null;
while(snapshot != null) {
@ -205,9 +225,9 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
}
return leafData;
}
protected void deleteSnapshotCacheChain(SnapshotInfo snapshot) {
}
protected Answer copyVolumeFromSnapshot(DataObject snapObj, DataObject volObj) {
@ -223,15 +243,15 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
if (!(storTO instanceof NfsTO)) {
srcData = cacheSnapshotChain(snapshot);
}
String value = configDao
.getValue(Config.CreateVolumeFromSnapshotWait.toString());
int _createVolumeFromSnapshotWait = NumbersUtil.parseInt(value,
Integer.parseInt(Config.CreateVolumeFromSnapshotWait
.getDefaultValue()));
CopyCommand cmd = new CopyCommand(srcData.getTO(), volObj.getTO(), _createVolumeFromSnapshotWait);
Answer answer = this.storageMgr
.sendToPool(pool, cmd);
@ -301,7 +321,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
destData.getType() == DataObjectType.SNAPSHOT) {
answer = copySnapshot(srcData, destData);
}
if (answer != null && !answer.getResult()) {
errMsg = answer.getDetails();
}
@ -318,7 +338,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
@DB
protected Answer createTemplateFromSnapshot(DataObject srcData,
DataObject destData) {
String value = configDao
.getValue(Config.CreatePrivateTemplateFromSnapshotWait
.toString());
@ -341,13 +361,13 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
private Answer createTemplateFromVolume(DataObject srcData,
DataObject destData) {
String value = configDao
.getValue(Config.CreatePrivateTemplateFromVolumeWait.toString());
int _createprivatetemplatefromvolumewait = NumbersUtil.parseInt(value,
Integer.parseInt(Config.CreatePrivateTemplateFromVolumeWait
.getDefaultValue()));
if (needCacheStorage(srcData, destData)) {
//need to copy it to image cache store
DataObject cacheData = cacheMgr.createCacheObject(srcData, destData.getDataStore().getScope());
@ -372,7 +392,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
try {
if (needCacheStorage(srcData, destData)) {
cacheData = cacheMgr.getCacheObject(srcData, destData.getDataStore().getScope());
CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait);
cmd.setCacheTO(cacheData.getTO());
EndPoint ep = selector.select(srcData, destData);
@ -391,7 +411,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
}
throw new CloudRuntimeException(e.toString());
}
}
}

2
engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java
View File

@ -134,6 +134,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager {
TemplateDataStoreVO ts = new TemplateDataStoreVO();
ts.setTemplateId(obj.getId());
ts.setDataStoreId(dataStore.getId());
ts.setDataStoreRole(dataStore.getRole());
ts.setInstallPath(TemplateConstants.DEFAULT_TMPLT_ROOT_DIR + "/" + TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR + templateDao.findById(obj.getId()).getAccountId() + "/" + obj.getId());
ts.setState(ObjectInDataStoreStateMachine.State.Allocated);
ts = templateDataStoreDao.persist(ts);
@ -143,6 +144,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager {
ss.setSnapshotId(obj.getId());
ss.setDataStoreId(dataStore.getId());
ss.setRole(dataStore.getRole());
ss.setRole(dataStore.getRole());
ss.setInstallPath(TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR + "/" + snapshotDao.findById(obj.getId()).getAccountId() + "/" + obj.getId());
ss.setState(ObjectInDataStoreStateMachine.State.Allocated);
ss = snapshotDataStoreDao.persist(ss);

2
engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java
View File

@ -162,7 +162,7 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore {
} else if (vo.getScope() == ScopeType.HOST) {
List<StoragePoolHostVO> poolHosts = poolHostDao.listByPoolId(vo.getId());
if (poolHosts.size() > 0) {
return new HostScope(poolHosts.get(0).getHostId());
return new HostScope(poolHosts.get(0).getHostId(), vo.getDataCenterId());
}
s_logger.debug("can't find a local storage in pool host table: " + vo.getId());
}

2
server/src/com/cloud/storage/StorageManagerImpl.java
View File

@ -760,7 +760,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
DataStoreRole.Primary);
}
HostScope scope = new HostScope(host.getId());
HostScope scope = new HostScope(host.getId(), pool.getDataCenterId());
lifeCycle.attachHost(store, scope, pInfo);
} catch (Exception e) {
s_logger.warn("Unable to setup the local storage pool for " + host, e);