apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

add copy stuff from s3 into nfs cache storage

Browse Source
This commit is contained in:
Edison Su 2013-04-18 13:44:49 -07:00
parent 1fe5d7c886
commit ef30ee52ee
8 changed files with 97 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
View File

@ -18,7 +18,6 @@ package com.cloud.storage.resource;
import static com.cloud.utils.S3Utils.deleteDirectory;
import static com.cloud.utils.S3Utils.getDirectory;
import static com.cloud.utils.S3Utils.putDirectory;
import static com.cloud.utils.StringUtils.join;
import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock;
import static java.lang.String.format;
@ -29,7 +28,6 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
@ -49,6 +47,7 @@ import java.util.concurrent.Callable;
import javax.naming.ConfigurationException;
import org.apache.cloudstack.engine.subsystem.api.storage.DataTO;
import org.apache.cloudstack.storage.command.CopyCmd;
import org.apache.log4j.Logger;
@ -63,7 +62,6 @@ import com.cloud.agent.api.DeleteSnapshotBackupCommand;
import com.cloud.agent.api.DeleteSnapshotsDirCommand;
import com.cloud.agent.api.DeleteTemplateFromS3Command;
import com.cloud.agent.api.DownloadSnapshotFromS3Command;
import com.cloud.agent.api.DownloadTemplateFromS3ToSecondaryStorageCommand;
import com.cloud.agent.api.GetStorageStatsAnswer;
import com.cloud.agent.api.GetStorageStatsCommand;
import com.cloud.agent.api.PingCommand;
@ -102,20 +100,19 @@ import com.cloud.exception.InternalErrorException;
import com.cloud.host.Host;
import com.cloud.host.Host.Type;
import com.cloud.resource.ServerResourceBase;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.StorageLayer;
import com.cloud.storage.template.DownloadManager;
import com.cloud.storage.template.DownloadManagerImpl;
import com.cloud.storage.template.DownloadManagerImpl.ZfsPathParser;
import com.cloud.storage.template.TemplateProp;
import com.cloud.storage.template.TemplateLocation;
import com.cloud.storage.template.TemplateProp;
import com.cloud.storage.template.UploadManager;
import com.cloud.storage.template.UploadManagerImpl;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.S3Utils;
import com.cloud.utils.UriUtils;
import com.cloud.utils.S3Utils.FileNamingStrategy;
import com.cloud.utils.S3Utils.ObjectNamingStrategy;
import com.cloud.utils.component.ComponentContext;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.net.NetUtils;
import com.cloud.utils.script.OutputInterpreter;
@ -208,8 +205,6 @@ SecondaryStorageResource {
return execute((DeleteSnapshotsDirCommand)cmd);
} else if (cmd instanceof downloadTemplateFromSwiftToSecondaryStorageCommand) {
return execute((downloadTemplateFromSwiftToSecondaryStorageCommand) cmd);
} else if (cmd instanceof DownloadTemplateFromS3ToSecondaryStorageCommand) {
return execute((DownloadTemplateFromS3ToSecondaryStorageCommand) cmd);
} else if (cmd instanceof uploadTemplateToSwiftFromSecondaryStorageCommand) {
return execute((uploadTemplateToSwiftFromSecondaryStorageCommand) cmd);
} else if (cmd instanceof UploadTemplateToS3FromSecondaryStorageCommand) {
@ -227,7 +222,80 @@ SecondaryStorageResource {
}
}
protected Answer downloadFromS3ToNfs(CopyCmd cmd, DataTO srcData, S3TO s3,
DataTO destData, NfsTO destImageStore) {
final String storagePath = destImageStore.getUrl();
final String destPath = destData.getPath();
try {
final File downloadDirectory = _storage
.getFile(determineStorageTemplatePath(storagePath,
destPath));
downloadDirectory.mkdirs();
if (!downloadDirectory.exists()) {
final String errMsg = format(
"Unable to create directory "
+ "download directory %1$s for download from S3.", downloadDirectory.getName()
);
s_logger.error(errMsg);
return new Answer(cmd, false, errMsg);
}
getDirectory(s3, s3.getBucketName(),
destPath,
downloadDirectory, new FileNamingStrategy() {
@Override
public String determineFileName(final String key) {
return substringAfterLast(key, S3Utils.SEPARATOR);
}
});
return new Answer(cmd, true, format("Successfully downloaded "
+ "from S3 to directory %2$s",
downloadDirectory.getName()));
} catch (Exception e) {
final String errMsg = format("Failed to download"
+ "due to $2%s", e.getMessage());
s_logger.error(errMsg, e);
return new Answer(cmd, false, errMsg);
}
}
protected Answer downloadFromSwiftToNfs(CopyCmd cmd, DataTO srcData, SwiftTO srcImageStore,
DataTO destData, NfsTO destImageStore) {
return Answer.createUnsupportedCommandAnswer(cmd);
}
protected Answer execute(CopyCmd cmd) {
DataTO srcData = cmd.getSrcTO();
DataTO destData = cmd.getDestTO();
DataStoreTO srcDataStore = srcData.getDataStore();
DataStoreTO destDataStore = destData.getDataStore();
if (srcDataStore.getRole() == DataStoreRole.Image
&& destDataStore.getRole() == DataStoreRole.ImageCache
) {
if (!(destDataStore instanceof NfsTO)) {
s_logger.debug("only support nfs as cache storage");
return Answer.createUnsupportedCommandAnswer(cmd);
}
if (srcDataStore instanceof S3TO) {
return downloadFromS3ToNfs(cmd, srcData, (S3TO)srcDataStore,
destData, (NfsTO)destDataStore);
} else if (srcDataStore instanceof SwiftTO) {
return downloadFromSwiftToNfs(cmd, srcData, (SwiftTO)srcDataStore,
destData, (NfsTO)destDataStore);
} else {
return Answer.createUnsupportedCommandAnswer(cmd);
}
}
return Answer.createUnsupportedCommandAnswer(cmd);
}
@ -240,59 +308,9 @@ SecondaryStorageResource {
@SuppressWarnings("unchecked")
private String determineStorageTemplatePath(final String storagePath,
final Long accountId, final Long templateId) {
String dataPath) {
return join(
asList(getRootDir(storagePath), TEMPLATE_ROOT_DIR, accountId,
templateId), File.separator);
}
private Answer execute(
final DownloadTemplateFromS3ToSecondaryStorageCommand cmd) {
final S3TO s3 = cmd.getS3();
final String storagePath = cmd.getStoragePath();
final Long accountId = cmd.getAccountId();
final Long templateId = cmd.getTemplateId();
try {
final File downloadDirectory = _storage
.getFile(determineStorageTemplatePath(storagePath,
accountId, templateId));
downloadDirectory.mkdirs();
if (!downloadDirectory.exists()) {
final String errMsg = format(
"Unable to create directory "
+ "download directory %1$s for download of template id "
+ "%2$s from S3.", downloadDirectory.getName(),
templateId);
s_logger.error(errMsg);
return new Answer(cmd, false, errMsg);
}
getDirectory(s3, s3.getBucketName(),
determineS3TemplateDirectory(accountId, templateId),
downloadDirectory, new FileNamingStrategy() {
@Override
public String determineFileName(final String key) {
return substringAfterLast(key, S3Utils.SEPARATOR);
}
});
return new Answer(cmd, true, format("Successfully downloaded "
+ "template id %1$s from S3 to directory %2$s", templateId,
downloadDirectory.getName()));
} catch (Exception e) {
final String errMsg = format("Failed to upload template id %1$s "
+ "due to $2%s", templateId, e.getMessage());
s_logger.error(errMsg, e);
return new Answer(cmd, false, errMsg);
}
asList(getRootDir(storagePath), dataPath), File.separator);
}
private Answer execute(downloadTemplateFromSwiftToSecondaryStorageCommand cmd) {
@ -411,7 +429,7 @@ SecondaryStorageResource {
}
private Answer execute(UploadTemplateToS3FromSecondaryStorageCommand cmd) {
/*
final S3TO s3 = cmd.getS3();
final Long accountId = cmd.getAccountId();
final Long templateId = cmd.getTemplateId();
@ -485,7 +503,8 @@ SecondaryStorageResource {
return new Answer(cmd, false, errMsg);
}
*/
return new Answer(cmd, false, "not supported ");
}
private Answer execute(DeleteObjectFromSwiftCommand cmd) {

2
engine/api/src/org/apache/cloudstack/storage/to/ImageStoreTO.java
View File

@ -38,7 +38,7 @@ public class ImageStoreTO implements DataStoreTO {
this.role = dataStore.getRole();
}
public String getType() {
public String getProtocol() {
return this.type;
}

2
engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentTest.java
View File

@ -131,7 +131,7 @@ public class DirectAgentTest extends CloudStackTestNGBase {
//Mockito.when(image.get).thenReturn(primaryStore);
ImageStoreTO imageStore = Mockito.mock(ImageStoreTO.class);
Mockito.when(imageStore.getType()).thenReturn("http");
Mockito.when(imageStore.getProtocol()).thenReturn("http");
TemplateObjectTO template = Mockito.mock(TemplateObjectTO.class);
Mockito.when(template.getPath()).thenReturn(this.getTemplateUrl());

2
engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java
View File

@ -133,7 +133,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager {
ts.setTemplateId(obj.getId());
ts.setDataStoreId(dataStore.getId());
if (dataStore.getRole() == DataStoreRole.ImageCache) {
ts.setInstallPath("/templates/" + templateDao.findById(obj.getId()).getAccountId() + "/" + obj.getId());
ts.setInstallPath("template/tmpl/" + templateDao.findById(obj.getId()).getAccountId() + "/" + obj.getId());
}
ts = templateDataStoreDao.persist(ts);
break;

8
plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java
View File

@ -312,7 +312,7 @@ public class VmwareServerDiscoverer extends DiscovererBase implements
.decode(uriFromCluster.getPath()));
if (morCluster == null
|| !morCluster.getType().equalsIgnoreCase(
|| !morCluster.getProtocol().equalsIgnoreCase(
"ClusterComputeResource")) {
s_logger.warn("Cluster url does not point to a valid vSphere cluster, url: "
+ clusterDetails.get("url"));
@ -345,7 +345,7 @@ public class VmwareServerDiscoverer extends DiscovererBase implements
details.put("url", hostMo.getHostName());
details.put("username", username);
details.put("password", password);
String guid = morHost.getType() + ":" + morHost.getValue()
String guid = morHost.getProtocol() + ":" + morHost.getValue()
+ "@" + url.getHost();
details.put("guid", guid);
@ -402,7 +402,7 @@ public class VmwareServerDiscoverer extends DiscovererBase implements
for (ManagedObjectReference morHost : morHosts) {
ManagedObjectReference morParent = (ManagedObjectReference) context
.getVimClient().getDynamicProperty(morHost, "parent");
if (morParent.getType().equalsIgnoreCase(
if (morParent.getProtocol().equalsIgnoreCase(
"ClusterComputeResource"))
return false;
}
@ -410,7 +410,7 @@ public class VmwareServerDiscoverer extends DiscovererBase implements
for (ManagedObjectReference morHost : morHosts) {
ManagedObjectReference morParent = (ManagedObjectReference) context
.getVimClient().getDynamicProperty(morHost, "parent");
if (!morParent.getType().equalsIgnoreCase(
if (!morParent.getProtocol().equalsIgnoreCase(
"ClusterComputeResource"))
return false;

8
plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java
View File

@ -336,7 +336,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
if(mor != null) {
List<ManagedObjectReference> returnedHostList = new ArrayList<ManagedObjectReference>();
if(mor.getType().equals("ComputeResource")) {
if(mor.getProtocol().equals("ComputeResource")) {
List<ManagedObjectReference> hosts = (List<ManagedObjectReference>)serviceContext.getVimClient().getDynamicProperty(mor, "host");
assert(hosts != null && hosts.size() > 0);
@ -346,7 +346,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
prepareHost(hostMo, privateTrafficLabel);
returnedHostList.add(hosts.get(0));
return returnedHostList;
} else if(mor.getType().equals("ClusterComputeResource")) {
} else if(mor.getProtocol().equals("ClusterComputeResource")) {
List<ManagedObjectReference> hosts = (List<ManagedObjectReference>)serviceContext.getVimClient().getDynamicProperty(mor, "host");
assert(hosts != null);
@ -368,14 +368,14 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
returnedHostList.add(morHost);
}
return returnedHostList;
} else if(mor.getType().equals("HostSystem")) {
} else if(mor.getProtocol().equals("HostSystem")) {
// For ESX host, we need to enable host firewall to allow VNC access
HostMO hostMo = new HostMO(serviceContext, mor);
prepareHost(hostMo, privateTrafficLabel);
returnedHostList.add(mor);
return returnedHostList;
} else {
s_logger.error("Unsupport host type " + mor.getType() + ":" + mor.getValue() + " from inventory path: " + hostInventoryPath);
s_logger.error("Unsupport host type " + mor.getProtocol() + ":" + mor.getValue() + " from inventory path: " + hostInventoryPath);
return null;
}
}

4
plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java
View File

@ -4470,7 +4470,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
ManagedObjectReference morParent = vmOwnerHost.getParentMor();
HashMap<String, Integer> portInfo;
if(morParent.getType().equalsIgnoreCase("ClusterComputeResource")) {
if(morParent.getProtocol().equalsIgnoreCase("ClusterComputeResource")) {
ClusterMO clusterMo = new ClusterMO(vmOwnerHost.getContext(), morParent);
portInfo = clusterMo.getVmVncPortsOnCluster();
} else {
@ -5067,7 +5067,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
@Override
public VmwareHypervisorHost getHyperHost(VmwareContext context, Command cmd) {
if (_morHyperHost.getType().equalsIgnoreCase("HostSystem")) {
if (_morHyperHost.getProtocol().equalsIgnoreCase("HostSystem")) {
return new HostMO(context, _morHyperHost);
}
return new ClusterMO(context, _morHyperHost);

4
plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java
View File

@ -221,7 +221,7 @@ public class VmwareSecondaryStorageResourceHandler implements SecondaryStorageRe
morHyperHost.setType(hostTokens[0]);
morHyperHost.setValue(hostTokens[1]);
if(morHyperHost.getType().equalsIgnoreCase("HostSystem")) {
if(morHyperHost.getProtocol().equalsIgnoreCase("HostSystem")) {
HostMO hostMo = new HostMO(context, morHyperHost);
try {
@ -278,7 +278,7 @@ public class VmwareSecondaryStorageResourceHandler implements SecondaryStorageRe
morHyperHost.setType(hostTokens[0]);
morHyperHost.setValue(hostTokens[1]);
if(morHyperHost.getType().equalsIgnoreCase("HostSystem")) {
if(morHyperHost.getProtocol().equalsIgnoreCase("HostSystem")) {
HostMO hostMo = new HostMO(context, morHyperHost);
try {
VmwareHypervisorHostNetworkSummary netSummary = hostMo.getHyperHostNetworkSummary(