apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-11-02 20:02:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix create template from snapshot if it's swift

Browse Source
This commit is contained in:
Edison Su 2013-07-19 18:17:37 -07:00 committed by Min Chen
parent b7a483608f
commit be3883b678
14 changed files with 319 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
core/src/com/cloud/agent/api/GetStorageStatsCommand.java
View File

@ -17,6 +17,7 @@
package com.cloud.agent.api;
import com.cloud.agent.api.LogLevel.Log4jLevel;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.storage.Storage.StoragePoolType;
@LogLevel(Log4jLevel.Trace)
@ -25,6 +26,7 @@ public class GetStorageStatsCommand extends Command {
private String localPath;
private StoragePoolType pooltype;
private String secUrl;
private DataStoreTO store;
public String getSecUrl() {
@ -46,6 +48,10 @@ public class GetStorageStatsCommand extends Command {
this.pooltype = pooltype;
}
public GetStorageStatsCommand(DataStoreTO store) {
this.store = store;
}
public GetStorageStatsCommand(String secUrl) {
this.secUrl = secUrl;
}
@ -69,6 +75,10 @@ public class GetStorageStatsCommand extends Command {
return this.localPath;
}
public DataStoreTO getStore() {
return this.store;
}
@Override
public boolean executeInSequence() {
return false;

16
core/src/com/cloud/storage/JavaStorageLayer.java
View File

@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.ejb.Local;
import javax.naming.ConfigurationException;
@ -163,6 +164,21 @@ public class JavaStorageLayer implements StorageLayer {
return file.length();
}
@Override
public File createUniqDir() {
String dirName = System.getProperty("java.io.tmpdir");
if (dirName != null) {
File dir = new File(dirName);
if (dir.exists()) {
String uniqDirName = dir.getAbsolutePath() + File.separator + UUID.randomUUID().toString();
if (this.mkdir(uniqDirName)) {
return new File(uniqDirName);
}
}
}
return null;
}
@Override
public boolean mkdirs(String path) {
synchronized(path.intern()) {

2
core/src/com/cloud/storage/StorageLayer.java
View File

@ -39,6 +39,8 @@ public interface StorageLayer extends Manager {
*/
long getSize(String path);
File createUniqDir();
/**
* Is this path a directory?
* @param path path to check.

2
engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java
View File

@ -93,7 +93,7 @@ public class SnapshotDataStoreVO implements StateObject<ObjectInDataStoreStateMa
ObjectInDataStoreStateMachine.State state;
@Column(name = "ref_cnt")
Long refCnt;
Long refCnt = 0L;
public String getInstallPath() {
return installPath;

2
engine/api/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java
View File

@ -113,7 +113,7 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
ObjectInDataStoreStateMachine.State state;
@Column(name = "ref_cnt")
Long refCnt;
Long refCnt = 0L;
public TemplateDataStoreVO(Long hostId, long templateId) {
super();

2
engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java
View File

@ -112,7 +112,7 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
ObjectInDataStoreStateMachine.State state;
@Column(name = "ref_cnt")
Long refCnt;
Long refCnt = 0L;
public String getInstallPath() {
return installPath;

1
engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
View File

@ -395,6 +395,7 @@ public class VolumeServiceImpl implements VolumeService {
int storagePoolMaxWaitSeconds = NumbersUtil.parseInt(
configDao.getValue(Config.StoragePoolMaxWaitSeconds.key()), 3600);
templatePoolRef = _tmpltPoolDao.acquireInLockTable(templatePoolRefId, storagePoolMaxWaitSeconds);
if (templatePoolRef == null) {
templatePoolRef = _tmpltPoolDao.findByPoolTemplate(dataStore.getId(), template.getId());
if (templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready ) {

25
plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageProcessor.java
View File

@ -90,6 +90,7 @@ import java.util.UUID;
public class XenServerStorageProcessor implements StorageProcessor {
private static final Logger s_logger = Logger.getLogger(XenServerStorageProcessor.class);
protected CitrixResourceBase hypervisorResource;
private String BaseMountPointOnHost = "/var/run/cloud_mount";
public XenServerStorageProcessor(CitrixResourceBase resource) {
this.hypervisorResource = resource;
@ -1043,10 +1044,10 @@ public class XenServerStorageProcessor implements StorageProcessor {
return false;
}
protected String deleteSnapshotBackup(Connection conn, String path, String secondaryStorageMountPath, String backupUUID) {
protected String deleteSnapshotBackup(Connection conn, String localMountPoint, String path, String secondaryStorageMountPath, String backupUUID) {
// If anybody modifies the formatting below again, I'll skin them
String result = hypervisorResource.callHostPlugin(conn, "vmopsSnapshot", "deleteSnapshotBackup", "backupUUID", backupUUID, "path", path, "secondaryStorageMountPath", secondaryStorageMountPath);
String result = hypervisorResource.callHostPlugin(conn, "vmopsSnapshot", "deleteSnapshotBackup", "backupUUID", backupUUID, "path", path, "secondaryStorageMountPath", secondaryStorageMountPath, "localMountPoint", localMountPoint);
return result;
}
@ -1147,7 +1148,7 @@ public class XenServerStorageProcessor implements StorageProcessor {
}
protected String backupSnapshot(Connection conn, String primaryStorageSRUuid, String path, String secondaryStorageMountPath, String snapshotUuid, String prevBackupUuid, Boolean isISCSI, int wait) {
protected String backupSnapshot(Connection conn, String primaryStorageSRUuid, String localMountPoint, String path, String secondaryStorageMountPath, String snapshotUuid, String prevBackupUuid, Boolean isISCSI, int wait) {
String backupSnapshotUuid = null;
if (prevBackupUuid == null) {
@ -1159,7 +1160,7 @@ public class XenServerStorageProcessor implements StorageProcessor {
String backupUuid = UUID.randomUUID().toString();
String results = hypervisorResource.callHostPluginAsync(conn, "vmopsSnapshot", "backupSnapshot", wait,
"primaryStorageSRUuid", primaryStorageSRUuid, "path", path, "secondaryStorageMountPath", secondaryStorageMountPath,
"snapshotUuid", snapshotUuid, "prevBackupUuid", prevBackupUuid, "backupUuid", backupUuid, "isISCSI", isISCSI.toString());
"snapshotUuid", snapshotUuid, "prevBackupUuid", prevBackupUuid, "backupUuid", backupUuid, "isISCSI", isISCSI.toString(), "localMountPath", localMountPoint);
String errMsg = null;
if (results == null || results.isEmpty()) {
errMsg = "Could not copy backupUuid: " + backupSnapshotUuid
@ -1282,6 +1283,8 @@ public class XenServerStorageProcessor implements StorageProcessor {
DataStoreTO destStore = destData.getDataStore();
String folder = destPath;
String finalPath = null;
String localMountPoint = BaseMountPointOnHost + File.separator + UUID.nameUUIDFromBytes(secondaryStorageUrl.getBytes()).toString();
if (fullbackup) {
// the first snapshot is always a full snapshot
@ -1300,11 +1303,11 @@ public class XenServerStorageProcessor implements StorageProcessor {
if( destStore instanceof SwiftTO) {
try {
String container = "S-" + snapshotTO.getVolume().getVolumeId().toString();
snapshotBackupUuid = swiftBackupSnapshot(conn, (SwiftTO)destStore, snapshotSr.getUuid(conn), snapshotBackupUuid, container, false, wait);
String swiftPath = container + File.separator + snapshotBackupUuid;
finalPath = container + File.separator + swiftPath;
String destSnapshotName = swiftBackupSnapshot(conn, (SwiftTO)destStore, snapshotSr.getUuid(conn), snapshotBackupUuid, container, false, wait);
String swiftPath = container + File.separator + destSnapshotName;
finalPath = swiftPath;
} finally {
deleteSnapshotBackup(conn, folder, secondaryStorageMountPath, snapshotBackupUuid);
deleteSnapshotBackup(conn, localMountPoint, folder, secondaryStorageMountPath, snapshotBackupUuid);
}
} else if (destStore instanceof S3TO) {
@ -1312,7 +1315,7 @@ public class XenServerStorageProcessor implements StorageProcessor {
backupSnapshotToS3(conn, (S3TO)destStore, snapshotSr.getUuid(conn), snapshotBackupUuid, isISCSI, wait);
snapshotBackupUuid = snapshotBackupUuid + ".vhd";
} finally {
deleteSnapshotBackup(conn, folder, secondaryStorageMountPath, snapshotBackupUuid);
deleteSnapshotBackup(conn, localMountPoint, folder, secondaryStorageMountPath, snapshotBackupUuid);
}
finalPath = folder + File.separator + snapshotBackupUuid;
} else {
@ -1334,8 +1337,8 @@ public class XenServerStorageProcessor implements StorageProcessor {
backupSnapshotToS3(conn, (S3TO)destStore, primaryStorageSRUuid, snapshotPaUuid, isISCSI, wait);
finalPath = folder + File.separator + snapshotPaUuid;
} else {
snapshotBackupUuid = backupSnapshot(conn, primaryStorageSRUuid, folder + File.separator + UUID.nameUUIDFromBytes(secondaryStorageMountPath.getBytes())
, secondaryStorageMountPath, snapshotUuid, prevBackupUuid, isISCSI, wait);
snapshotBackupUuid = backupSnapshot(conn, primaryStorageSRUuid, localMountPoint, folder,
secondaryStorageMountPath, snapshotUuid, prevBackupUuid, isISCSI, wait);
finalPath = folder + File.separator + snapshotBackupUuid;
}
}

28
scripts/vm/hypervisor/xenserver/vmopsSnapshot
View File

@ -321,19 +321,18 @@ def umount(localDir):
util.SMlog("Successfully unmounted " + localDir)
return
def mountSnapshotsDir(secondaryStorageMountPath, relativeDir, dcId, accountId, instanceId, secHostId):
def mountSnapshotsDir(secondaryStorageMountPath, localMountPoint, path):
# The aim is to mount secondaryStorageMountPath on
# And create <accountId>/<instanceId> dir on it, if it doesn't exist already.
# Assuming that secondaryStorageMountPath exists remotely
# Just mount secondaryStorageMountPath/<relativeDir>/SecondaryStorageHost/ everytime
# Never unmount.
# path is like "snapshots/account/volumeId", we mount secondary_storage:/snapshots
relativeDir = path.split("/")[0]
restDir = "/".join(path.split("/")[1:])
snapshotsDir = os.path.join(secondaryStorageMountPath, relativeDir)
# Mkdir local mount point dir, if it doesn't exist.
localMountPointPath = os.path.join(CLOUD_DIR, dcId)
localMountPointPath = os.path.join(localMountPointPath, relativeDir, secHostId)
makedirs(localMountPointPath)
# if something is not mounted already on localMountPointPath,
# mount secondaryStorageMountPath on localMountPath
@ -346,8 +345,7 @@ def mountSnapshotsDir(secondaryStorageMountPath, relativeDir, dcId, accountId, i
mount(snapshotsDir, localMountPointPath)
# Create accountId/instanceId dir on localMountPointPath, if it doesn't exist
backupsDir = os.path.join(localMountPointPath, accountId)
backupsDir = os.path.join(backupsDir, instanceId)
backupsDir = os.path.join(localMountPointPath, restDir)
makedirs(backupsDir)
return backupsDir
@ -485,16 +483,13 @@ def getVhdParent(session, args):
def backupSnapshot(session, args):
util.SMlog("Called backupSnapshot with " + str(args))
primaryStorageSRUuid = args['primaryStorageSRUuid']
dcId = args['dcId']
accountId = args['accountId']
volumeId = args['volumeId']
secondaryStorageMountPath = args['secondaryStorageMountPath']
snapshotUuid = args['snapshotUuid']
prevBackupUuid = args['prevBackupUuid']
backupUuid = args['backupUuid']
isISCSI = getIsTrueString(args['isISCSI'])
secHostId = args['secHostId']
path = args['path']
localMountPoint = args['localMountPoint']
primarySRPath = getPrimarySRPath(primaryStorageSRUuid, isISCSI)
util.SMlog("primarySRPath: " + primarySRPath)
@ -507,7 +502,7 @@ def backupSnapshot(session, args):
# Mount secondary storage mount path on XenServer along the path
# /var/run/sr-mount/<dcId>/snapshots/ and create <accountId>/<volumeId> dir
# on it.
backupsDir = mountSnapshotsDir(secondaryStorageMountPath, "snapshots", dcId, accountId, volumeId, secHostId)
backupsDir = mountSnapshotsDir(secondaryStorageMountPath, localMountPoint, path)
util.SMlog("Backups dir " + backupsDir)
# Check existence of snapshot on primary storage
@ -538,13 +533,12 @@ def backupSnapshot(session, args):
@echo
def deleteSnapshotBackup(session, args):
util.SMlog("Calling deleteSnapshotBackup with " + str(args))
dcId = args['dcId']
accountId = args['accountId']
volumeId = args['volumeId']
secondaryStorageMountPath = args['secondaryStorageMountPath']
backupUUID = args['backupUUID']
path = args['path']
localMountPoint = args['localMountPoint']
backupsDir = mountSnapshotsDir(secondaryStorageMountPath, "snapshots", dcId, accountId, volumeId)
backupsDir = mountSnapshotsDir(secondaryStorageMountPath, localMountPoint, path)
# chdir to the backupsDir for convenience
chdir(backupsDir)

3
server/src/com/cloud/server/StatsCollector.java
View File

@ -534,7 +534,8 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
if ( store.getUri() == null ) {
continue;
}
GetStorageStatsCommand command = new GetStorageStatsCommand(store.getUri());
GetStorageStatsCommand command = new GetStorageStatsCommand(store.getTO());
EndPoint ssAhost = _epSelector.select(store);
if (ssAhost == null) {
s_logger.debug("There is no secondary storage VM for secondary storage host " + store.getName());

167
services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
View File

@ -164,6 +164,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
public void disconnected() {
}
public void setInSystemVM(boolean inSystemVM) {
this._inSystemVM = inSystemVM;
}
@Override
public Answer executeRequest(Command cmd) {
if (cmd instanceof DownloadProgressCommand) {
@ -215,6 +219,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
protected CopyCmdAnswer postProcessing(File destFile, String downloadPath, String destPath, DataTO srcData, DataTO destData) throws ConfigurationException {
if (destData.getObjectType() == DataObjectType.SNAPSHOT) {
SnapshotObjectTO snapshot = new SnapshotObjectTO();
snapshot.setPath(destPath + File.separator + destFile.getName());
CopyCmdAnswer answer = new CopyCmdAnswer(snapshot);
return answer;
}
// do post processing to unzip the file if it is compressed
String scriptsDir = "scripts/storage/secondary";
String createTmpltScr = Script.findScript(scriptsDir, "createtmplt.sh");
@ -238,7 +249,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
String extension = null;
if (srcData.getObjectType() == DataObjectType.TEMPLATE) {
extension = ((TemplateObjectTO) srcData).getFormat().getFileExtension();
} else {
} else if (srcData.getObjectType() == DataObjectType.VOLUME) {
extension = ((VolumeObjectTO) srcData).getFormat().getFileExtension();
}
@ -403,11 +414,51 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
if (srcData.getHypervisorType() == HypervisorType.XenServer) {
return copySnapshotToTemplateFromNfsToNfsXenserver(cmd, srcData, srcDataStore, destData, destDataStore);
} else if (srcData.getHypervisorType() == HypervisorType.KVM) {
File srcFile = getFile(srcData.getPath(), srcDataStore.getUrl());
File destFile = getFile(destData.getPath(), destDataStore.getUrl());
s_logger.debug("copy snapshot to template");
Script.runSimpleBashScript("cp " + srcFile.getAbsolutePath() + " " + destFile.getAbsolutePath());
QCOW2Processor processor = new QCOW2Processor();
Map<String, Object> params = new HashMap<String, Object>();
params.put(StorageLayer.InstanceConfigKey, _storage);
try {
processor.configure("qcow2 processor", params);
String destPath = destFile.getAbsolutePath();
String templateName = srcFile.getName();
FormatInfo info = processor.process(destPath, null, templateName);
TemplateLocation loc = new TemplateLocation(_storage, destPath);
loc.create(1, true, srcFile.getName());
loc.addFormat(info);
loc.save();
TemplateProp prop = loc.getTemplateInfo();
TemplateObjectTO newTemplate = new TemplateObjectTO();
newTemplate.setPath(destData.getPath() + File.separator + templateName);
newTemplate.setFormat(ImageFormat.VHD);
newTemplate.setSize(prop.getSize());
return new CopyCmdAnswer(newTemplate);
} catch (ConfigurationException e) {
s_logger.debug("Failed to create template:" + e.toString());
return new CopyCmdAnswer(e.toString());
} catch (IOException e) {
s_logger.debug("Failed to create template:" + e.toString());
return new CopyCmdAnswer(e.toString());
}
}
return new CopyCmdAnswer("");
}
protected File getFile(String path, String nfsPath) {
String filePath = getRootDir(nfsPath) + File.separator + path;
File f = new File(filePath);
if (!f.exists()) {
_storage.mkdirs(filePath);
f = new File(filePath);
}
return f;
}
protected Answer createTemplateFromSnapshot(CopyCommand cmd) {
DataTO srcData = cmd.getSrcTO();
DataTO destData = cmd.getDestTO();
@ -422,8 +473,32 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
if (destDataStore instanceof NfsTO) {
return copySnapshotToTemplateFromNfsToNfs(cmd, (SnapshotObjectTO) srcData, (NfsTO) srcDataStore,
(TemplateObjectTO) destData, (NfsTO) destDataStore);
}
} else if (destDataStore instanceof SwiftTO) {
//create template on the same data store
CopyCmdAnswer answer = (CopyCmdAnswer)copySnapshotToTemplateFromNfsToNfs(cmd, (SnapshotObjectTO) srcData, (NfsTO) srcDataStore,
(TemplateObjectTO) destData, (NfsTO) srcDataStore);
if (!answer.getResult()) {
return answer;
}
s_logger.debug("starting copy template to swift");
DataTO newTemplate = (DataTO)answer.getNewData();
File templateFile = getFile(newTemplate.getPath(), ((NfsTO) srcDataStore).getUrl());
SwiftTO swift = (SwiftTO)destDataStore;
String containterName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId());
String swiftPath = SwiftUtil.putObject(swift, templateFile, containterName, templateFile.getName());
//upload template.properties
File properties = new File(templateFile.getParent() + File.separator + _tmpltpp);
if (properties.exists()) {
SwiftUtil.putObject(swift, properties, containterName, _tmpltpp);
}
TemplateObjectTO template = new TemplateObjectTO();
template.setPath(swiftPath);
template.setSize(templateFile.length());
SnapshotObjectTO snapshot = (SnapshotObjectTO)srcData;
template.setFormat(snapshot.getVolume().getFormat());
return new CopyCmdAnswer(template);
}
}
return new CopyCmdAnswer("");
}
@ -581,9 +656,29 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
File file = null;
try {
NfsTO nfsCacheStore = (NfsTO)cacheStore;
String fileName = UUID.randomUUID().toString() + "." + cmd.getFormat().getFileExtension();
String fileName = cmd.getName() + "." + cmd.getFormat().getFileExtension();
file = downloadFromUrlToNfs(cmd.getUrl(), nfsCacheStore, path, fileName);
String swiftPath = SwiftUtil.putObject(swiftTO, file, "T-" + cmd.getId());
String container = "T-" + cmd.getId();
String swiftPath = SwiftUtil.putObject(swiftTO, file, container, null);
//put metda file
File uniqDir = _storage.createUniqDir();
String metaFileName = uniqDir.getAbsolutePath() + File.separator + "template.properties";
_storage.create(uniqDir.getAbsolutePath(), "template.properties");
File metaFile = new File(metaFileName);
FileWriter writer = new FileWriter(metaFile);
BufferedWriter bufferWriter = new BufferedWriter(writer);
bufferWriter.write("uniquename=" + cmd.getName());
bufferWriter.write("\n");
bufferWriter.write("filename=" + fileName);
bufferWriter.write("\n");
bufferWriter.write("size=" + file.length());
bufferWriter.close();
writer.close();
SwiftUtil.putObject(swiftTO, metaFile, container, "template.properties");
metaFile.delete();
uniqDir.delete();
String md5sum = null;
try {
md5sum = DigestUtils.md5Hex(new FileInputStream(file));
@ -1361,32 +1456,52 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
Map<String, TemplateProp> swiftListTemplate(SwiftTO swift) {
String[] containers = swiftList(swift, "", "");
String[] containers = SwiftUtil.list(swift, "", null);
if (containers == null) {
return null;
}
Map<String, TemplateProp> tmpltInfos = new HashMap<String, TemplateProp>();
for (String container : containers) {
if (container.startsWith("T-")) {
String ldir = _tmpltDir + "/" + UUID.randomUUID().toString();
createLocalDir(ldir);
String lFullPath = ldir + "/" + _tmpltpp;
swiftDownload(swift, container, _tmpltpp, lFullPath);
TemplateLocation loc = new TemplateLocation(_storage, ldir);
try {
if (!loc.load()) {
s_logger.warn("Can not parse template.properties file for template " + container);
continue;
}
} catch (IOException e) {
s_logger.warn("Unable to load template location " + ldir + " due to " + e.toString(), e);
String[] files = SwiftUtil.list(swift, container, "template.properties");
if (files.length != 1) {
continue;
}
try {
File tempFile = File.createTempFile("template", ".tmp");
File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + "template.properties");
if (tmpFile == null) {
continue;
}
FileReader fr = new FileReader(tmpFile);
BufferedReader brf = new BufferedReader(fr);
String line = null;
String uniqName = null;
Long size = null;
String name = null;
while ((line = brf.readLine()) != null) {
if (line.startsWith("uniquename=")) {
uniqName = line.split("=")[1];
} else if (line.startsWith("size=")) {
size = Long.parseLong(line.split("=")[1]);
} else if (line.startsWith("filename=")) {
name = line.split("=")[1];
}
}
brf.close();
tempFile.delete();
if (uniqName != null) {
TemplateProp prop = new TemplateProp(uniqName, container + File.separator + name, size, size, true, false);
tmpltInfos.put(uniqName, prop);
}
} catch (IOException e) {
s_logger.debug("Failed to create templ file:" + e.toString());
continue;
} catch (Exception e) {
s_logger.debug("Failed to get properties: " + e.toString());
continue;
}
TemplateProp tInfo = loc.getTemplateInfo();
tInfo.setInstallPath(container);
tmpltInfos.put(tInfo.getTemplateName(), tInfo);
loc.purge();
deleteLocalDir(ldir);
}
}
return tmpltInfos;
@ -1612,7 +1727,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
protected GetStorageStatsAnswer execute(final GetStorageStatsCommand cmd) {
String rootDir = getRootDir(cmd.getSecUrl());
DataStoreTO store = cmd.getStore();
if (store instanceof S3TO || store instanceof SwiftTO) {
long infinity = Integer.MAX_VALUE;
return new GetStorageStatsAnswer(cmd, infinity, 0L);
}
String rootDir = getRootDir(((NfsTO) store).getUrl());
final long usedSize = getUsedSize(rootDir);
final long totalSize = getTotalSize(rootDir);
if (usedSize == -1 || totalSize == -1) {

18
services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
View File

@ -20,6 +20,8 @@ package org.apache.cloudstack.storage.resource;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.storage.DownloadAnswer;
import com.cloud.agent.api.storage.ListTemplateAnswer;
import com.cloud.agent.api.storage.ListTemplateCommand;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.SwiftTO;
@ -28,9 +30,11 @@ import com.cloud.storage.Storage;
import com.cloud.utils.SwiftUtil;
import junit.framework.Assert;
import junit.framework.TestCase;
import org.apache.cloudstack.api.command.user.tag.ListTagsCmd;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.DownloadCommand;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.junit.Before;
import org.junit.Test;
@ -39,6 +43,7 @@ import org.mockito.Mockito;
import javax.naming.ConfigurationException;
import java.util.HashMap;
import java.util.UUID;
public class LocalNfsSecondaryStorageResourceTest extends TestCase {
LocalNfsSecondaryStorageResource resource;
@ -46,6 +51,8 @@ public class LocalNfsSecondaryStorageResourceTest extends TestCase {
@Override
public void setUp() throws ConfigurationException {
resource = new LocalNfsSecondaryStorageResource();
resource.setInSystemVM(true);
resource.setParentPath("/mnt");
System.setProperty("paths.script", "/Users/edison/develop/asf-master/script");
//resource.configure("test", new HashMap<String, Object>());
@ -59,14 +66,15 @@ public class LocalNfsSecondaryStorageResourceTest extends TestCase {
Mockito.when(swift.getEndPoint()).thenReturn("https://objects.dreamhost.com/auth");
Mockito.when(swift.getAccount()).thenReturn("cloudstack");
Mockito.when(swift.getUserName()).thenReturn("images");
//Mockito.when(swift.getKey()).thenReturn("something");
Mockito.when(swift.getKey()).thenReturn("oxvELQaOD1U5_VyosGfA-wpZ7uBWEff-CUBGCM0u");
Mockito.when(template.getDataStore()).thenReturn(swift);
Mockito.when(template.getPath()).thenReturn("template/1/1/");
Mockito.when(template.isRequiresHvm()).thenReturn(true);
Mockito.when(template.getId()).thenReturn(1L);
Mockito.when(template.getFormat()).thenReturn(Storage.ImageFormat.VHD);
Mockito.when(template.getOrigUrl()).thenReturn("http://nfs1.lab.vmops.com/templates/ttylinux_pv.vhd");
Mockito.when(template.getOrigUrl()).thenReturn("http://nfs1.lab.vmops.com/templates/test.bz2");
Mockito.when(template.getName()).thenReturn(UUID.randomUUID().toString());
Mockito.when(template.getObjectType()).thenReturn(DataObjectType.TEMPLATE);
DownloadCommand cmd = new DownloadCommand(template, 100000L);
@ -86,5 +94,11 @@ public class LocalNfsSecondaryStorageResourceTest extends TestCase {
CopyCmdAnswer copyCmdAnswer = (CopyCmdAnswer)resource.executeRequest(cpyCmd);
Assert.assertTrue(copyCmdAnswer.getResult());
//list template
ListTemplateCommand listCmd = new ListTemplateCommand(swift);
ListTemplateAnswer listAnswer = (ListTemplateAnswer)resource.executeRequest(listCmd);
Assert.assertTrue(listAnswer.getTemplateInfo().size() > 0);
}
}

2
setup/db/db/schema-410to420.sql
View File

@ -161,7 +161,7 @@ CREATE TABLE `cloud`.`template_store_ref` (
`destroyed` tinyint(1) COMMENT 'indicates whether the template_store entry was destroyed by the user or not',
`is_copy` tinyint(1) NOT NULL DEFAULT 0 COMMENT 'indicates whether this was copied ',
`update_count` bigint unsigned,
`ref_cnt` bigint unsigned,
`ref_cnt` bigint unsigned DEFAULT 0,
`updated` datetime,
PRIMARY KEY (`id`),
-- CONSTRAINT `fk_template_store_ref__store_id` FOREIGN KEY `fk_template_store_ref__store_id` (`store_id`) REFERENCES `image_store` (`id`) ON DELETE CASCADE,

104
utils/src/com/cloud/utils/SwiftUtil.java
View File

@ -24,6 +24,8 @@ import com.cloud.utils.script.Script;
import org.apache.log4j.Logger;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
public class SwiftUtil {
@ -45,8 +47,34 @@ public class SwiftUtil {
return swiftCLI;
}
public static String putObject(SwiftClientCfg cfg, File srcFile, String container) {
public static boolean postMeta(SwiftClientCfg cfg, String container, String object, Map<String, String> metas) {
String swiftCli = getSwiftCLIPath();
StringBuilder cms = new StringBuilder();
for(Map.Entry<String, String> entry : metas.entrySet()) {
cms.append(" -m ");
cms.append(entry.getKey());
cms.append(":");
cms.append(entry.getValue());
cms.append(" ");
}
Script command = new Script("/bin/bash", logger);
command.add("-c");
command.add("/usr/bin/python " + swiftCli + " -A "
+ cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K "
+ cfg.getKey() + " post " + container + " " + object + " " + cms.toString());
OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
String result = command.execute(parser);
if (result != null) {
throw new CloudRuntimeException("Failed to post meta" + result);
}
return true;
}
public static String putObject(SwiftClientCfg cfg, File srcFile, String container, String fileName) {
String swiftCli = getSwiftCLIPath();
if (fileName == null) {
fileName = srcFile.getName();
}
String srcDirectory = srcFile.getParent();
Script command = new Script("/bin/bash", logger);
long size = srcFile.length();
@ -55,12 +83,12 @@ public class SwiftUtil {
command.add("cd " + srcDirectory
+ ";/usr/bin/python " + swiftCli + " -A "
+ cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K "
+ cfg.getKey() + " upload " + container + " " + srcFile.getName());
+ cfg.getKey() + " upload " + container + " " + fileName);
} else {
command.add("cd " + srcDirectory
+ ";/usr/bin/python " + swiftCli + " -A "
+ cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K "
+ cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + srcFile.getName());
+ cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + fileName);
}
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
@ -71,19 +99,74 @@ public class SwiftUtil {
if (parser.getLines() != null) {
String[] lines = parser.getLines().split("\\n");
for (String line : lines) {
if (line.contains("Errno") || line.contains("failed")) {
if (line.contains("Errno") || line.contains("failed") || line.contains("not found")) {
throw new CloudRuntimeException("Failed to upload file: " + lines.toString());
}
}
}
return container + File.separator + srcFile.getName();
}
private static StringBuilder buildSwiftCmd(SwiftClientCfg swift) {
String swiftCli = getSwiftCLIPath();
StringBuilder sb = new StringBuilder();
sb.append(" /usr/bin/python ");
sb.append(swiftCli);
sb.append(" -A ");
sb.append(swift.getEndPoint());
sb.append(" -U ");
sb.append(swift.getAccount());
sb.append(":");
sb.append(swift.getUserName());
sb.append(" -K ");
sb.append(swift.getKey());
sb.append(" ");
return sb;
}
public static String[] list(SwiftClientCfg swift, String container, String rFilename) {
String swiftCli = getSwiftCLIPath();
Script command = new Script("/bin/bash", logger);
command.add("-c");
StringBuilder swiftCmdBuilder = buildSwiftCmd(swift);
swiftCmdBuilder.append(" list ");
swiftCmdBuilder.append(container);
if (rFilename != null) {
swiftCmdBuilder.append(" -p ");
swiftCmdBuilder.append(rFilename);
}
command.add(swiftCmdBuilder.toString());
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result == null && parser.getLines() != null && !parser.getLines().equalsIgnoreCase("")) {
String[] lines = parser.getLines().split("\\n");
return lines;
} else {
if (result != null) {
String errMsg = "swiftList failed , err=" + result;
logger.debug("Failed to list " + errMsg);
} else {
String errMsg = "swiftList failed, no lines returns";
logger.debug("Failed to list " + errMsg);
}
}
return new String[0];
}
public static File getObject(SwiftClientCfg cfg, File destDirectory, String swiftPath) {
int firstIndexOfSeparator = swiftPath.indexOf(File.separator);
String container = swiftPath.substring(0, firstIndexOfSeparator);
String srcPath = swiftPath.substring(firstIndexOfSeparator + 1);
String destFilePath = destDirectory.getAbsolutePath() + File.separator + srcPath;
String destFilePath = null;
if (destDirectory.isDirectory()) {
destFilePath = destDirectory.getAbsolutePath() + File.separator + srcPath;
} else {
destFilePath = destDirectory.getAbsolutePath();
}
String swiftCli = getSwiftCLIPath();
Script command = new Script("/bin/bash", logger);
command.add("-c");
@ -109,4 +192,15 @@ public class SwiftUtil {
}
return new File(destFilePath);
}
public static String getContainerName(String type, Long id) {
if (type.startsWith("T")) {
return "T-" + id;
} else if (type.startsWith("S")) {
return "S-" + id;
} else if (type.startsWith("V")) {
return "V-" + id;
}
return null;
}
}