mirror of
https://github.com/apache/cloudstack.git
synced 2025-11-04 00:02:37 +01:00
fix create template from snapshot if it's swift
This commit is contained in:
Edison Su
Min Chen
parent
b7a483608f
commit
be3883b678
@ -17,6 +17,7 @@
|
||||
package com.cloud.agent.api;
|
||||
|
||||
import com.cloud.agent.api.LogLevel.Log4jLevel;
|
||||
import com.cloud.agent.api.to.DataStoreTO;
|
||||
import com.cloud.storage.Storage.StoragePoolType;
|
||||
|
||||
@LogLevel(Log4jLevel.Trace)
|
||||
@ -25,6 +26,7 @@ public class GetStorageStatsCommand extends Command {
|
||||
private String localPath;
|
||||
private StoragePoolType pooltype;
|
||||
private String secUrl;
|
||||
private DataStoreTO store;
|
||||
|
||||
|
||||
public String getSecUrl() {
|
||||
@ -46,6 +48,10 @@ public class GetStorageStatsCommand extends Command {
|
||||
this.pooltype = pooltype;
|
||||
}
|
||||
|
||||
public GetStorageStatsCommand(DataStoreTO store) {
|
||||
this.store = store;
|
||||
}
|
||||
|
||||
public GetStorageStatsCommand(String secUrl) {
|
||||
this.secUrl = secUrl;
|
||||
}
|
||||
@ -69,6 +75,10 @@ public class GetStorageStatsCommand extends Command {
|
||||
return this.localPath;
|
||||
}
|
||||
|
||||
public DataStoreTO getStore() {
|
||||
return this.store;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean executeInSequence() {
|
||||
return false;
|
||||
|
||||
@ -22,6 +22,7 @@ import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
import javax.ejb.Local;
|
||||
import javax.naming.ConfigurationException;
|
||||
@ -163,6 +164,21 @@ public class JavaStorageLayer implements StorageLayer {
|
||||
return file.length();
|
||||
}
|
||||
|
||||
@Override
|
||||
public File createUniqDir() {
|
||||
String dirName = System.getProperty("java.io.tmpdir");
|
||||
if (dirName != null) {
|
||||
File dir = new File(dirName);
|
||||
if (dir.exists()) {
|
||||
String uniqDirName = dir.getAbsolutePath() + File.separator + UUID.randomUUID().toString();
|
||||
if (this.mkdir(uniqDirName)) {
|
||||
return new File(uniqDirName);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean mkdirs(String path) {
|
||||
synchronized(path.intern()) {
|
||||
|
||||
@ -39,6 +39,8 @@ public interface StorageLayer extends Manager {
|
||||
*/
|
||||
long getSize(String path);
|
||||
|
||||
File createUniqDir();
|
||||
|
||||
/**
|
||||
* Is this path a directory?
|
||||
* @param path path to check.
|
||||
|
||||
@ -93,7 +93,7 @@ public class SnapshotDataStoreVO implements StateObject<ObjectInDataStoreStateMa
|
||||
ObjectInDataStoreStateMachine.State state;
|
||||
|
||||
@Column(name = "ref_cnt")
|
||||
Long refCnt;
|
||||
Long refCnt = 0L;
|
||||
|
||||
public String getInstallPath() {
|
||||
return installPath;
|
||||
|
||||
@ -113,7 +113,7 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
|
||||
ObjectInDataStoreStateMachine.State state;
|
||||
|
||||
@Column(name = "ref_cnt")
|
||||
Long refCnt;
|
||||
Long refCnt = 0L;
|
||||
|
||||
public TemplateDataStoreVO(Long hostId, long templateId) {
|
||||
super();
|
||||
|
||||
@ -112,7 +112,7 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
|
||||
ObjectInDataStoreStateMachine.State state;
|
||||
|
||||
@Column(name = "ref_cnt")
|
||||
Long refCnt;
|
||||
Long refCnt = 0L;
|
||||
|
||||
public String getInstallPath() {
|
||||
return installPath;
|
||||
|
||||
@ -395,6 +395,7 @@ public class VolumeServiceImpl implements VolumeService {
|
||||
int storagePoolMaxWaitSeconds = NumbersUtil.parseInt(
|
||||
configDao.getValue(Config.StoragePoolMaxWaitSeconds.key()), 3600);
|
||||
templatePoolRef = _tmpltPoolDao.acquireInLockTable(templatePoolRefId, storagePoolMaxWaitSeconds);
|
||||
|
||||
if (templatePoolRef == null) {
|
||||
templatePoolRef = _tmpltPoolDao.findByPoolTemplate(dataStore.getId(), template.getId());
|
||||
if (templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready ) {
|
||||
|
||||
@ -90,6 +90,7 @@ import java.util.UUID;
|
||||
public class XenServerStorageProcessor implements StorageProcessor {
|
||||
private static final Logger s_logger = Logger.getLogger(XenServerStorageProcessor.class);
|
||||
protected CitrixResourceBase hypervisorResource;
|
||||
private String BaseMountPointOnHost = "/var/run/cloud_mount";
|
||||
|
||||
public XenServerStorageProcessor(CitrixResourceBase resource) {
|
||||
this.hypervisorResource = resource;
|
||||
@ -1043,10 +1044,10 @@ public class XenServerStorageProcessor implements StorageProcessor {
|
||||
return false;
|
||||
}
|
||||
|
||||
protected String deleteSnapshotBackup(Connection conn, String path, String secondaryStorageMountPath, String backupUUID) {
|
||||
protected String deleteSnapshotBackup(Connection conn, String localMountPoint, String path, String secondaryStorageMountPath, String backupUUID) {
|
||||
|
||||
// If anybody modifies the formatting below again, I'll skin them
|
||||
String result = hypervisorResource.callHostPlugin(conn, "vmopsSnapshot", "deleteSnapshotBackup", "backupUUID", backupUUID, "path", path, "secondaryStorageMountPath", secondaryStorageMountPath);
|
||||
String result = hypervisorResource.callHostPlugin(conn, "vmopsSnapshot", "deleteSnapshotBackup", "backupUUID", backupUUID, "path", path, "secondaryStorageMountPath", secondaryStorageMountPath, "localMountPoint", localMountPoint);
|
||||
|
||||
return result;
|
||||
}
|
||||
@ -1147,7 +1148,7 @@ public class XenServerStorageProcessor implements StorageProcessor {
|
||||
|
||||
}
|
||||
|
||||
protected String backupSnapshot(Connection conn, String primaryStorageSRUuid, String path, String secondaryStorageMountPath, String snapshotUuid, String prevBackupUuid, Boolean isISCSI, int wait) {
|
||||
protected String backupSnapshot(Connection conn, String primaryStorageSRUuid, String localMountPoint, String path, String secondaryStorageMountPath, String snapshotUuid, String prevBackupUuid, Boolean isISCSI, int wait) {
|
||||
String backupSnapshotUuid = null;
|
||||
|
||||
if (prevBackupUuid == null) {
|
||||
@ -1159,7 +1160,7 @@ public class XenServerStorageProcessor implements StorageProcessor {
|
||||
String backupUuid = UUID.randomUUID().toString();
|
||||
String results = hypervisorResource.callHostPluginAsync(conn, "vmopsSnapshot", "backupSnapshot", wait,
|
||||
"primaryStorageSRUuid", primaryStorageSRUuid, "path", path, "secondaryStorageMountPath", secondaryStorageMountPath,
|
||||
"snapshotUuid", snapshotUuid, "prevBackupUuid", prevBackupUuid, "backupUuid", backupUuid, "isISCSI", isISCSI.toString());
|
||||
"snapshotUuid", snapshotUuid, "prevBackupUuid", prevBackupUuid, "backupUuid", backupUuid, "isISCSI", isISCSI.toString(), "localMountPath", localMountPoint);
|
||||
String errMsg = null;
|
||||
if (results == null || results.isEmpty()) {
|
||||
errMsg = "Could not copy backupUuid: " + backupSnapshotUuid
|
||||
@ -1282,6 +1283,8 @@ public class XenServerStorageProcessor implements StorageProcessor {
|
||||
DataStoreTO destStore = destData.getDataStore();
|
||||
String folder = destPath;
|
||||
String finalPath = null;
|
||||
|
||||
String localMountPoint = BaseMountPointOnHost + File.separator + UUID.nameUUIDFromBytes(secondaryStorageUrl.getBytes()).toString();
|
||||
if (fullbackup) {
|
||||
// the first snapshot is always a full snapshot
|
||||
|
||||
@ -1300,11 +1303,11 @@ public class XenServerStorageProcessor implements StorageProcessor {
|
||||
if( destStore instanceof SwiftTO) {
|
||||
try {
|
||||
String container = "S-" + snapshotTO.getVolume().getVolumeId().toString();
|
||||
snapshotBackupUuid = swiftBackupSnapshot(conn, (SwiftTO)destStore, snapshotSr.getUuid(conn), snapshotBackupUuid, container, false, wait);
|
||||
String swiftPath = container + File.separator + snapshotBackupUuid;
|
||||
finalPath = container + File.separator + swiftPath;
|
||||
String destSnapshotName = swiftBackupSnapshot(conn, (SwiftTO)destStore, snapshotSr.getUuid(conn), snapshotBackupUuid, container, false, wait);
|
||||
String swiftPath = container + File.separator + destSnapshotName;
|
||||
finalPath = swiftPath;
|
||||
} finally {
|
||||
deleteSnapshotBackup(conn, folder, secondaryStorageMountPath, snapshotBackupUuid);
|
||||
deleteSnapshotBackup(conn, localMountPoint, folder, secondaryStorageMountPath, snapshotBackupUuid);
|
||||
}
|
||||
|
||||
} else if (destStore instanceof S3TO) {
|
||||
@ -1312,7 +1315,7 @@ public class XenServerStorageProcessor implements StorageProcessor {
|
||||
backupSnapshotToS3(conn, (S3TO)destStore, snapshotSr.getUuid(conn), snapshotBackupUuid, isISCSI, wait);
|
||||
snapshotBackupUuid = snapshotBackupUuid + ".vhd";
|
||||
} finally {
|
||||
deleteSnapshotBackup(conn, folder, secondaryStorageMountPath, snapshotBackupUuid);
|
||||
deleteSnapshotBackup(conn, localMountPoint, folder, secondaryStorageMountPath, snapshotBackupUuid);
|
||||
}
|
||||
finalPath = folder + File.separator + snapshotBackupUuid;
|
||||
} else {
|
||||
@ -1334,8 +1337,8 @@ public class XenServerStorageProcessor implements StorageProcessor {
|
||||
backupSnapshotToS3(conn, (S3TO)destStore, primaryStorageSRUuid, snapshotPaUuid, isISCSI, wait);
|
||||
finalPath = folder + File.separator + snapshotPaUuid;
|
||||
} else {
|
||||
snapshotBackupUuid = backupSnapshot(conn, primaryStorageSRUuid, folder + File.separator + UUID.nameUUIDFromBytes(secondaryStorageMountPath.getBytes())
|
||||
, secondaryStorageMountPath, snapshotUuid, prevBackupUuid, isISCSI, wait);
|
||||
snapshotBackupUuid = backupSnapshot(conn, primaryStorageSRUuid, localMountPoint, folder,
|
||||
secondaryStorageMountPath, snapshotUuid, prevBackupUuid, isISCSI, wait);
|
||||
finalPath = folder + File.separator + snapshotBackupUuid;
|
||||
}
|
||||
}
|
||||
|
||||
@ -321,19 +321,18 @@ def umount(localDir):
|
||||
util.SMlog("Successfully unmounted " + localDir)
|
||||
return
|
||||
|
||||
def mountSnapshotsDir(secondaryStorageMountPath, relativeDir, dcId, accountId, instanceId, secHostId):
|
||||
def mountSnapshotsDir(secondaryStorageMountPath, localMountPoint, path):
|
||||
# The aim is to mount secondaryStorageMountPath on
|
||||
# And create <accountId>/<instanceId> dir on it, if it doesn't exist already.
|
||||
# Assuming that secondaryStorageMountPath exists remotely
|
||||
|
||||
# Just mount secondaryStorageMountPath/<relativeDir>/SecondaryStorageHost/ everytime
|
||||
# Never unmount.
|
||||
# path is like "snapshots/account/volumeId", we mount secondary_storage:/snapshots
|
||||
relativeDir = path.split("/")[0]
|
||||
restDir = "/".join(path.split("/")[1:])
|
||||
snapshotsDir = os.path.join(secondaryStorageMountPath, relativeDir)
|
||||
|
||||
# Mkdir local mount point dir, if it doesn't exist.
|
||||
localMountPointPath = os.path.join(CLOUD_DIR, dcId)
|
||||
localMountPointPath = os.path.join(localMountPointPath, relativeDir, secHostId)
|
||||
|
||||
makedirs(localMountPointPath)
|
||||
# if something is not mounted already on localMountPointPath,
|
||||
# mount secondaryStorageMountPath on localMountPath
|
||||
@ -346,8 +345,7 @@ def mountSnapshotsDir(secondaryStorageMountPath, relativeDir, dcId, accountId, i
|
||||
mount(snapshotsDir, localMountPointPath)
|
||||
|
||||
# Create accountId/instanceId dir on localMountPointPath, if it doesn't exist
|
||||
backupsDir = os.path.join(localMountPointPath, accountId)
|
||||
backupsDir = os.path.join(backupsDir, instanceId)
|
||||
backupsDir = os.path.join(localMountPointPath, restDir)
|
||||
makedirs(backupsDir)
|
||||
return backupsDir
|
||||
|
||||
@ -485,16 +483,13 @@ def getVhdParent(session, args):
|
||||
def backupSnapshot(session, args):
|
||||
util.SMlog("Called backupSnapshot with " + str(args))
|
||||
primaryStorageSRUuid = args['primaryStorageSRUuid']
|
||||
dcId = args['dcId']
|
||||
accountId = args['accountId']
|
||||
volumeId = args['volumeId']
|
||||
secondaryStorageMountPath = args['secondaryStorageMountPath']
|
||||
snapshotUuid = args['snapshotUuid']
|
||||
prevBackupUuid = args['prevBackupUuid']
|
||||
backupUuid = args['backupUuid']
|
||||
isISCSI = getIsTrueString(args['isISCSI'])
|
||||
secHostId = args['secHostId']
|
||||
|
||||
path = args['path']
|
||||
localMountPoint = args['localMountPoint']
|
||||
primarySRPath = getPrimarySRPath(primaryStorageSRUuid, isISCSI)
|
||||
util.SMlog("primarySRPath: " + primarySRPath)
|
||||
|
||||
@ -507,7 +502,7 @@ def backupSnapshot(session, args):
|
||||
# Mount secondary storage mount path on XenServer along the path
|
||||
# /var/run/sr-mount/<dcId>/snapshots/ and create <accountId>/<volumeId> dir
|
||||
# on it.
|
||||
backupsDir = mountSnapshotsDir(secondaryStorageMountPath, "snapshots", dcId, accountId, volumeId, secHostId)
|
||||
backupsDir = mountSnapshotsDir(secondaryStorageMountPath, localMountPoint, path)
|
||||
util.SMlog("Backups dir " + backupsDir)
|
||||
|
||||
# Check existence of snapshot on primary storage
|
||||
@ -538,13 +533,12 @@ def backupSnapshot(session, args):
|
||||
@echo
|
||||
def deleteSnapshotBackup(session, args):
|
||||
util.SMlog("Calling deleteSnapshotBackup with " + str(args))
|
||||
dcId = args['dcId']
|
||||
accountId = args['accountId']
|
||||
volumeId = args['volumeId']
|
||||
secondaryStorageMountPath = args['secondaryStorageMountPath']
|
||||
backupUUID = args['backupUUID']
|
||||
path = args['path']
|
||||
localMountPoint = args['localMountPoint']
|
||||
|
||||
backupsDir = mountSnapshotsDir(secondaryStorageMountPath, "snapshots", dcId, accountId, volumeId)
|
||||
backupsDir = mountSnapshotsDir(secondaryStorageMountPath, localMountPoint, path)
|
||||
# chdir to the backupsDir for convenience
|
||||
chdir(backupsDir)
|
||||
|
||||
|
||||
@ -534,7 +534,8 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
|
||||
if ( store.getUri() == null ) {
|
||||
continue;
|
||||
}
|
||||
GetStorageStatsCommand command = new GetStorageStatsCommand(store.getUri());
|
||||
|
||||
GetStorageStatsCommand command = new GetStorageStatsCommand(store.getTO());
|
||||
EndPoint ssAhost = _epSelector.select(store);
|
||||
if (ssAhost == null) {
|
||||
s_logger.debug("There is no secondary storage VM for secondary storage host " + store.getName());
|
||||
|
||||
@ -164,6 +164,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
||||
public void disconnected() {
|
||||
}
|
||||
|
||||
public void setInSystemVM(boolean inSystemVM) {
|
||||
this._inSystemVM = inSystemVM;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Answer executeRequest(Command cmd) {
|
||||
if (cmd instanceof DownloadProgressCommand) {
|
||||
@ -215,6 +219,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
||||
|
||||
|
||||
protected CopyCmdAnswer postProcessing(File destFile, String downloadPath, String destPath, DataTO srcData, DataTO destData) throws ConfigurationException {
|
||||
if (destData.getObjectType() == DataObjectType.SNAPSHOT) {
|
||||
SnapshotObjectTO snapshot = new SnapshotObjectTO();
|
||||
snapshot.setPath(destPath + File.separator + destFile.getName());
|
||||
|
||||
CopyCmdAnswer answer = new CopyCmdAnswer(snapshot);
|
||||
return answer;
|
||||
}
|
||||
// do post processing to unzip the file if it is compressed
|
||||
String scriptsDir = "scripts/storage/secondary";
|
||||
String createTmpltScr = Script.findScript(scriptsDir, "createtmplt.sh");
|
||||
@ -238,7 +249,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
||||
String extension = null;
|
||||
if (srcData.getObjectType() == DataObjectType.TEMPLATE) {
|
||||
extension = ((TemplateObjectTO) srcData).getFormat().getFileExtension();
|
||||
} else {
|
||||
} else if (srcData.getObjectType() == DataObjectType.VOLUME) {
|
||||
extension = ((VolumeObjectTO) srcData).getFormat().getFileExtension();
|
||||
}
|
||||
|
||||
@ -403,11 +414,51 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
||||
|
||||
if (srcData.getHypervisorType() == HypervisorType.XenServer) {
|
||||
return copySnapshotToTemplateFromNfsToNfsXenserver(cmd, srcData, srcDataStore, destData, destDataStore);
|
||||
} else if (srcData.getHypervisorType() == HypervisorType.KVM) {
|
||||
File srcFile = getFile(srcData.getPath(), srcDataStore.getUrl());
|
||||
File destFile = getFile(destData.getPath(), destDataStore.getUrl());
|
||||
s_logger.debug("copy snapshot to template");
|
||||
Script.runSimpleBashScript("cp " + srcFile.getAbsolutePath() + " " + destFile.getAbsolutePath());
|
||||
QCOW2Processor processor = new QCOW2Processor();
|
||||
Map<String, Object> params = new HashMap<String, Object>();
|
||||
params.put(StorageLayer.InstanceConfigKey, _storage);
|
||||
try {
|
||||
processor.configure("qcow2 processor", params);
|
||||
String destPath = destFile.getAbsolutePath();
|
||||
String templateName = srcFile.getName();
|
||||
FormatInfo info = processor.process(destPath, null, templateName);
|
||||
TemplateLocation loc = new TemplateLocation(_storage, destPath);
|
||||
loc.create(1, true, srcFile.getName());
|
||||
loc.addFormat(info);
|
||||
loc.save();
|
||||
TemplateProp prop = loc.getTemplateInfo();
|
||||
TemplateObjectTO newTemplate = new TemplateObjectTO();
|
||||
newTemplate.setPath(destData.getPath() + File.separator + templateName);
|
||||
newTemplate.setFormat(ImageFormat.VHD);
|
||||
newTemplate.setSize(prop.getSize());
|
||||
return new CopyCmdAnswer(newTemplate);
|
||||
} catch (ConfigurationException e) {
|
||||
s_logger.debug("Failed to create template:" + e.toString());
|
||||
return new CopyCmdAnswer(e.toString());
|
||||
} catch (IOException e) {
|
||||
s_logger.debug("Failed to create template:" + e.toString());
|
||||
return new CopyCmdAnswer(e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
return new CopyCmdAnswer("");
|
||||
}
|
||||
|
||||
protected File getFile(String path, String nfsPath) {
|
||||
String filePath = getRootDir(nfsPath) + File.separator + path;
|
||||
File f = new File(filePath);
|
||||
if (!f.exists()) {
|
||||
_storage.mkdirs(filePath);
|
||||
f = new File(filePath);
|
||||
}
|
||||
return f;
|
||||
}
|
||||
|
||||
protected Answer createTemplateFromSnapshot(CopyCommand cmd) {
|
||||
DataTO srcData = cmd.getSrcTO();
|
||||
DataTO destData = cmd.getDestTO();
|
||||
@ -422,8 +473,32 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
||||
if (destDataStore instanceof NfsTO) {
|
||||
return copySnapshotToTemplateFromNfsToNfs(cmd, (SnapshotObjectTO) srcData, (NfsTO) srcDataStore,
|
||||
(TemplateObjectTO) destData, (NfsTO) destDataStore);
|
||||
}
|
||||
} else if (destDataStore instanceof SwiftTO) {
|
||||
//create template on the same data store
|
||||
CopyCmdAnswer answer = (CopyCmdAnswer)copySnapshotToTemplateFromNfsToNfs(cmd, (SnapshotObjectTO) srcData, (NfsTO) srcDataStore,
|
||||
(TemplateObjectTO) destData, (NfsTO) srcDataStore);
|
||||
if (!answer.getResult()) {
|
||||
return answer;
|
||||
}
|
||||
s_logger.debug("starting copy template to swift");
|
||||
DataTO newTemplate = (DataTO)answer.getNewData();
|
||||
File templateFile = getFile(newTemplate.getPath(), ((NfsTO) srcDataStore).getUrl());
|
||||
SwiftTO swift = (SwiftTO)destDataStore;
|
||||
String containterName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId());
|
||||
String swiftPath = SwiftUtil.putObject(swift, templateFile, containterName, templateFile.getName());
|
||||
//upload template.properties
|
||||
File properties = new File(templateFile.getParent() + File.separator + _tmpltpp);
|
||||
if (properties.exists()) {
|
||||
SwiftUtil.putObject(swift, properties, containterName, _tmpltpp);
|
||||
}
|
||||
|
||||
TemplateObjectTO template = new TemplateObjectTO();
|
||||
template.setPath(swiftPath);
|
||||
template.setSize(templateFile.length());
|
||||
SnapshotObjectTO snapshot = (SnapshotObjectTO)srcData;
|
||||
template.setFormat(snapshot.getVolume().getFormat());
|
||||
return new CopyCmdAnswer(template);
|
||||
}
|
||||
}
|
||||
return new CopyCmdAnswer("");
|
||||
}
|
||||
@ -581,9 +656,29 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
||||
File file = null;
|
||||
try {
|
||||
NfsTO nfsCacheStore = (NfsTO)cacheStore;
|
||||
String fileName = UUID.randomUUID().toString() + "." + cmd.getFormat().getFileExtension();
|
||||
String fileName = cmd.getName() + "." + cmd.getFormat().getFileExtension();
|
||||
file = downloadFromUrlToNfs(cmd.getUrl(), nfsCacheStore, path, fileName);
|
||||
String swiftPath = SwiftUtil.putObject(swiftTO, file, "T-" + cmd.getId());
|
||||
String container = "T-" + cmd.getId();
|
||||
String swiftPath = SwiftUtil.putObject(swiftTO, file, container, null);
|
||||
|
||||
//put metda file
|
||||
File uniqDir = _storage.createUniqDir();
|
||||
String metaFileName = uniqDir.getAbsolutePath() + File.separator + "template.properties";
|
||||
_storage.create(uniqDir.getAbsolutePath(), "template.properties");
|
||||
File metaFile = new File(metaFileName);
|
||||
FileWriter writer = new FileWriter(metaFile);
|
||||
BufferedWriter bufferWriter = new BufferedWriter(writer);
|
||||
bufferWriter.write("uniquename=" + cmd.getName());
|
||||
bufferWriter.write("\n");
|
||||
bufferWriter.write("filename=" + fileName);
|
||||
bufferWriter.write("\n");
|
||||
bufferWriter.write("size=" + file.length());
|
||||
bufferWriter.close();
|
||||
writer.close();
|
||||
|
||||
SwiftUtil.putObject(swiftTO, metaFile, container, "template.properties");
|
||||
metaFile.delete();
|
||||
uniqDir.delete();
|
||||
String md5sum = null;
|
||||
try {
|
||||
md5sum = DigestUtils.md5Hex(new FileInputStream(file));
|
||||
@ -1361,32 +1456,52 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
||||
}
|
||||
|
||||
Map<String, TemplateProp> swiftListTemplate(SwiftTO swift) {
|
||||
String[] containers = swiftList(swift, "", "");
|
||||
String[] containers = SwiftUtil.list(swift, "", null);
|
||||
if (containers == null) {
|
||||
return null;
|
||||
}
|
||||
Map<String, TemplateProp> tmpltInfos = new HashMap<String, TemplateProp>();
|
||||
for (String container : containers) {
|
||||
if (container.startsWith("T-")) {
|
||||
String ldir = _tmpltDir + "/" + UUID.randomUUID().toString();
|
||||
createLocalDir(ldir);
|
||||
String lFullPath = ldir + "/" + _tmpltpp;
|
||||
swiftDownload(swift, container, _tmpltpp, lFullPath);
|
||||
TemplateLocation loc = new TemplateLocation(_storage, ldir);
|
||||
try {
|
||||
if (!loc.load()) {
|
||||
s_logger.warn("Can not parse template.properties file for template " + container);
|
||||
continue;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
s_logger.warn("Unable to load template location " + ldir + " due to " + e.toString(), e);
|
||||
String[] files = SwiftUtil.list(swift, container, "template.properties");
|
||||
if (files.length != 1) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
File tempFile = File.createTempFile("template", ".tmp");
|
||||
File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + "template.properties");
|
||||
if (tmpFile == null) {
|
||||
continue;
|
||||
}
|
||||
FileReader fr = new FileReader(tmpFile);
|
||||
BufferedReader brf = new BufferedReader(fr);
|
||||
String line = null;
|
||||
String uniqName = null;
|
||||
Long size = null;
|
||||
String name = null;
|
||||
while ((line = brf.readLine()) != null) {
|
||||
if (line.startsWith("uniquename=")) {
|
||||
uniqName = line.split("=")[1];
|
||||
} else if (line.startsWith("size=")) {
|
||||
size = Long.parseLong(line.split("=")[1]);
|
||||
} else if (line.startsWith("filename=")) {
|
||||
name = line.split("=")[1];
|
||||
}
|
||||
}
|
||||
brf.close();
|
||||
tempFile.delete();
|
||||
if (uniqName != null) {
|
||||
TemplateProp prop = new TemplateProp(uniqName, container + File.separator + name, size, size, true, false);
|
||||
tmpltInfos.put(uniqName, prop);
|
||||
}
|
||||
|
||||
} catch (IOException e) {
|
||||
s_logger.debug("Failed to create templ file:" + e.toString());
|
||||
continue;
|
||||
} catch (Exception e) {
|
||||
s_logger.debug("Failed to get properties: " + e.toString());
|
||||
continue;
|
||||
}
|
||||
TemplateProp tInfo = loc.getTemplateInfo();
|
||||
tInfo.setInstallPath(container);
|
||||
tmpltInfos.put(tInfo.getTemplateName(), tInfo);
|
||||
loc.purge();
|
||||
deleteLocalDir(ldir);
|
||||
}
|
||||
}
|
||||
return tmpltInfos;
|
||||
@ -1612,7 +1727,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
||||
}
|
||||
|
||||
protected GetStorageStatsAnswer execute(final GetStorageStatsCommand cmd) {
|
||||
String rootDir = getRootDir(cmd.getSecUrl());
|
||||
DataStoreTO store = cmd.getStore();
|
||||
if (store instanceof S3TO || store instanceof SwiftTO) {
|
||||
long infinity = Integer.MAX_VALUE;
|
||||
return new GetStorageStatsAnswer(cmd, infinity, 0L);
|
||||
}
|
||||
|
||||
String rootDir = getRootDir(((NfsTO) store).getUrl());
|
||||
final long usedSize = getUsedSize(rootDir);
|
||||
final long totalSize = getTotalSize(rootDir);
|
||||
if (usedSize == -1 || totalSize == -1) {
|
||||
|
||||
@ -20,6 +20,8 @@ package org.apache.cloudstack.storage.resource;
|
||||
|
||||
import com.cloud.agent.api.Answer;
|
||||
import com.cloud.agent.api.storage.DownloadAnswer;
|
||||
import com.cloud.agent.api.storage.ListTemplateAnswer;
|
||||
import com.cloud.agent.api.storage.ListTemplateCommand;
|
||||
import com.cloud.agent.api.to.DataObjectType;
|
||||
import com.cloud.agent.api.to.NfsTO;
|
||||
import com.cloud.agent.api.to.SwiftTO;
|
||||
@ -28,9 +30,11 @@ import com.cloud.storage.Storage;
|
||||
import com.cloud.utils.SwiftUtil;
|
||||
import junit.framework.Assert;
|
||||
import junit.framework.TestCase;
|
||||
import org.apache.cloudstack.api.command.user.tag.ListTagsCmd;
|
||||
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
|
||||
import org.apache.cloudstack.storage.command.CopyCommand;
|
||||
import org.apache.cloudstack.storage.command.DownloadCommand;
|
||||
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
|
||||
import org.apache.cloudstack.storage.to.TemplateObjectTO;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
@ -39,6 +43,7 @@ import org.mockito.Mockito;
|
||||
|
||||
import javax.naming.ConfigurationException;
|
||||
import java.util.HashMap;
|
||||
import java.util.UUID;
|
||||
|
||||
public class LocalNfsSecondaryStorageResourceTest extends TestCase {
|
||||
LocalNfsSecondaryStorageResource resource;
|
||||
@ -46,6 +51,8 @@ public class LocalNfsSecondaryStorageResourceTest extends TestCase {
|
||||
@Override
|
||||
public void setUp() throws ConfigurationException {
|
||||
resource = new LocalNfsSecondaryStorageResource();
|
||||
resource.setInSystemVM(true);
|
||||
|
||||
resource.setParentPath("/mnt");
|
||||
System.setProperty("paths.script", "/Users/edison/develop/asf-master/script");
|
||||
//resource.configure("test", new HashMap<String, Object>());
|
||||
@ -59,14 +66,15 @@ public class LocalNfsSecondaryStorageResourceTest extends TestCase {
|
||||
Mockito.when(swift.getEndPoint()).thenReturn("https://objects.dreamhost.com/auth");
|
||||
Mockito.when(swift.getAccount()).thenReturn("cloudstack");
|
||||
Mockito.when(swift.getUserName()).thenReturn("images");
|
||||
//Mockito.when(swift.getKey()).thenReturn("something");
|
||||
Mockito.when(swift.getKey()).thenReturn("oxvELQaOD1U5_VyosGfA-wpZ7uBWEff-CUBGCM0u");
|
||||
|
||||
Mockito.when(template.getDataStore()).thenReturn(swift);
|
||||
Mockito.when(template.getPath()).thenReturn("template/1/1/");
|
||||
Mockito.when(template.isRequiresHvm()).thenReturn(true);
|
||||
Mockito.when(template.getId()).thenReturn(1L);
|
||||
Mockito.when(template.getFormat()).thenReturn(Storage.ImageFormat.VHD);
|
||||
Mockito.when(template.getOrigUrl()).thenReturn("http://nfs1.lab.vmops.com/templates/ttylinux_pv.vhd");
|
||||
Mockito.when(template.getOrigUrl()).thenReturn("http://nfs1.lab.vmops.com/templates/test.bz2");
|
||||
Mockito.when(template.getName()).thenReturn(UUID.randomUUID().toString());
|
||||
Mockito.when(template.getObjectType()).thenReturn(DataObjectType.TEMPLATE);
|
||||
|
||||
DownloadCommand cmd = new DownloadCommand(template, 100000L);
|
||||
@ -86,5 +94,11 @@ public class LocalNfsSecondaryStorageResourceTest extends TestCase {
|
||||
CopyCmdAnswer copyCmdAnswer = (CopyCmdAnswer)resource.executeRequest(cpyCmd);
|
||||
Assert.assertTrue(copyCmdAnswer.getResult());
|
||||
|
||||
//list template
|
||||
ListTemplateCommand listCmd = new ListTemplateCommand(swift);
|
||||
ListTemplateAnswer listAnswer = (ListTemplateAnswer)resource.executeRequest(listCmd);
|
||||
|
||||
Assert.assertTrue(listAnswer.getTemplateInfo().size() > 0);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -161,7 +161,7 @@ CREATE TABLE `cloud`.`template_store_ref` (
|
||||
`destroyed` tinyint(1) COMMENT 'indicates whether the template_store entry was destroyed by the user or not',
|
||||
`is_copy` tinyint(1) NOT NULL DEFAULT 0 COMMENT 'indicates whether this was copied ',
|
||||
`update_count` bigint unsigned,
|
||||
`ref_cnt` bigint unsigned,
|
||||
`ref_cnt` bigint unsigned DEFAULT 0,
|
||||
`updated` datetime,
|
||||
PRIMARY KEY (`id`),
|
||||
-- CONSTRAINT `fk_template_store_ref__store_id` FOREIGN KEY `fk_template_store_ref__store_id` (`store_id`) REFERENCES `image_store` (`id`) ON DELETE CASCADE,
|
||||
|
||||
@ -24,6 +24,8 @@ import com.cloud.utils.script.Script;
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
public class SwiftUtil {
|
||||
@ -45,8 +47,34 @@ public class SwiftUtil {
|
||||
return swiftCLI;
|
||||
}
|
||||
|
||||
public static String putObject(SwiftClientCfg cfg, File srcFile, String container) {
|
||||
public static boolean postMeta(SwiftClientCfg cfg, String container, String object, Map<String, String> metas) {
|
||||
String swiftCli = getSwiftCLIPath();
|
||||
StringBuilder cms = new StringBuilder();
|
||||
for(Map.Entry<String, String> entry : metas.entrySet()) {
|
||||
cms.append(" -m ");
|
||||
cms.append(entry.getKey());
|
||||
cms.append(":");
|
||||
cms.append(entry.getValue());
|
||||
cms.append(" ");
|
||||
}
|
||||
Script command = new Script("/bin/bash", logger);
|
||||
command.add("-c");
|
||||
command.add("/usr/bin/python " + swiftCli + " -A "
|
||||
+ cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K "
|
||||
+ cfg.getKey() + " post " + container + " " + object + " " + cms.toString());
|
||||
OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
|
||||
String result = command.execute(parser);
|
||||
if (result != null) {
|
||||
throw new CloudRuntimeException("Failed to post meta" + result);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public static String putObject(SwiftClientCfg cfg, File srcFile, String container, String fileName) {
|
||||
String swiftCli = getSwiftCLIPath();
|
||||
if (fileName == null) {
|
||||
fileName = srcFile.getName();
|
||||
}
|
||||
String srcDirectory = srcFile.getParent();
|
||||
Script command = new Script("/bin/bash", logger);
|
||||
long size = srcFile.length();
|
||||
@ -55,12 +83,12 @@ public class SwiftUtil {
|
||||
command.add("cd " + srcDirectory
|
||||
+ ";/usr/bin/python " + swiftCli + " -A "
|
||||
+ cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K "
|
||||
+ cfg.getKey() + " upload " + container + " " + srcFile.getName());
|
||||
+ cfg.getKey() + " upload " + container + " " + fileName);
|
||||
} else {
|
||||
command.add("cd " + srcDirectory
|
||||
+ ";/usr/bin/python " + swiftCli + " -A "
|
||||
+ cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K "
|
||||
+ cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + srcFile.getName());
|
||||
+ cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + fileName);
|
||||
}
|
||||
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
|
||||
String result = command.execute(parser);
|
||||
@ -71,19 +99,74 @@ public class SwiftUtil {
|
||||
if (parser.getLines() != null) {
|
||||
String[] lines = parser.getLines().split("\\n");
|
||||
for (String line : lines) {
|
||||
if (line.contains("Errno") || line.contains("failed")) {
|
||||
if (line.contains("Errno") || line.contains("failed") || line.contains("not found")) {
|
||||
throw new CloudRuntimeException("Failed to upload file: " + lines.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return container + File.separator + srcFile.getName();
|
||||
}
|
||||
|
||||
private static StringBuilder buildSwiftCmd(SwiftClientCfg swift) {
|
||||
String swiftCli = getSwiftCLIPath();
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(" /usr/bin/python ");
|
||||
sb.append(swiftCli);
|
||||
sb.append(" -A ");
|
||||
sb.append(swift.getEndPoint());
|
||||
sb.append(" -U ");
|
||||
sb.append(swift.getAccount());
|
||||
sb.append(":");
|
||||
sb.append(swift.getUserName());
|
||||
sb.append(" -K ");
|
||||
sb.append(swift.getKey());
|
||||
sb.append(" ");
|
||||
return sb;
|
||||
}
|
||||
|
||||
public static String[] list(SwiftClientCfg swift, String container, String rFilename) {
|
||||
String swiftCli = getSwiftCLIPath();
|
||||
Script command = new Script("/bin/bash", logger);
|
||||
command.add("-c");
|
||||
|
||||
StringBuilder swiftCmdBuilder = buildSwiftCmd(swift);
|
||||
swiftCmdBuilder.append(" list ");
|
||||
swiftCmdBuilder.append(container);
|
||||
|
||||
if (rFilename != null) {
|
||||
swiftCmdBuilder.append(" -p ");
|
||||
swiftCmdBuilder.append(rFilename);
|
||||
}
|
||||
|
||||
command.add(swiftCmdBuilder.toString());
|
||||
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
|
||||
String result = command.execute(parser);
|
||||
if (result == null && parser.getLines() != null && !parser.getLines().equalsIgnoreCase("")) {
|
||||
String[] lines = parser.getLines().split("\\n");
|
||||
return lines;
|
||||
} else {
|
||||
if (result != null) {
|
||||
String errMsg = "swiftList failed , err=" + result;
|
||||
logger.debug("Failed to list " + errMsg);
|
||||
} else {
|
||||
String errMsg = "swiftList failed, no lines returns";
|
||||
logger.debug("Failed to list " + errMsg);
|
||||
}
|
||||
}
|
||||
return new String[0];
|
||||
}
|
||||
|
||||
public static File getObject(SwiftClientCfg cfg, File destDirectory, String swiftPath) {
|
||||
int firstIndexOfSeparator = swiftPath.indexOf(File.separator);
|
||||
String container = swiftPath.substring(0, firstIndexOfSeparator);
|
||||
String srcPath = swiftPath.substring(firstIndexOfSeparator + 1);
|
||||
String destFilePath = destDirectory.getAbsolutePath() + File.separator + srcPath;
|
||||
String destFilePath = null;
|
||||
if (destDirectory.isDirectory()) {
|
||||
destFilePath = destDirectory.getAbsolutePath() + File.separator + srcPath;
|
||||
} else {
|
||||
destFilePath = destDirectory.getAbsolutePath();
|
||||
}
|
||||
String swiftCli = getSwiftCLIPath();
|
||||
Script command = new Script("/bin/bash", logger);
|
||||
command.add("-c");
|
||||
@ -109,4 +192,15 @@ public class SwiftUtil {
|
||||
}
|
||||
return new File(destFilePath);
|
||||
}
|
||||
|
||||
public static String getContainerName(String type, Long id) {
|
||||
if (type.startsWith("T")) {
|
||||
return "T-" + id;
|
||||
} else if (type.startsWith("S")) {
|
||||
return "S-" + id;
|
||||
} else if (type.startsWith("V")) {
|
||||
return "V-" + id;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user