From be3883b67820126328ff5dd9098b267c2d656354 Mon Sep 17 00:00:00 2001 From: Edison Su Date: Fri, 19 Jul 2013 18:17:37 -0700 Subject: [PATCH] fix create template from snapshot if it's swift --- .../agent/api/GetStorageStatsCommand.java | 10 ++ .../com/cloud/storage/JavaStorageLayer.java | 16 ++ core/src/com/cloud/storage/StorageLayer.java | 2 + .../datastore/db/SnapshotDataStoreVO.java | 2 +- .../datastore/db/TemplateDataStoreVO.java | 2 +- .../datastore/db/VolumeDataStoreVO.java | 2 +- .../storage/volume/VolumeServiceImpl.java | 1 + .../resource/XenServerStorageProcessor.java | 25 +-- scripts/vm/hypervisor/xenserver/vmopsSnapshot | 28 ++- .../src/com/cloud/server/StatsCollector.java | 3 +- .../resource/NfsSecondaryStorageResource.java | 167 +++++++++++++++--- .../LocalNfsSecondaryStorageResourceTest.java | 18 +- setup/db/db/schema-410to420.sql | 2 +- utils/src/com/cloud/utils/SwiftUtil.java | 104 ++++++++++- 14 files changed, 319 insertions(+), 63 deletions(-) diff --git a/core/src/com/cloud/agent/api/GetStorageStatsCommand.java b/core/src/com/cloud/agent/api/GetStorageStatsCommand.java index f7ebd51fb9f..8f51e3562d5 100755 --- a/core/src/com/cloud/agent/api/GetStorageStatsCommand.java +++ b/core/src/com/cloud/agent/api/GetStorageStatsCommand.java @@ -17,6 +17,7 @@ package com.cloud.agent.api; import com.cloud.agent.api.LogLevel.Log4jLevel; +import com.cloud.agent.api.to.DataStoreTO; import com.cloud.storage.Storage.StoragePoolType; @LogLevel(Log4jLevel.Trace) @@ -25,6 +26,7 @@ public class GetStorageStatsCommand extends Command { private String localPath; private StoragePoolType pooltype; private String secUrl; + private DataStoreTO store; public String getSecUrl() { @@ -46,6 +48,10 @@ public class GetStorageStatsCommand extends Command { this.pooltype = pooltype; } + public GetStorageStatsCommand(DataStoreTO store) { + this.store = store; + } + public GetStorageStatsCommand(String secUrl) { this.secUrl = secUrl; } @@ -69,6 +75,10 @@ public class GetStorageStatsCommand extends Command { return this.localPath; } + public DataStoreTO getStore() { + return this.store; + } + @Override public boolean executeInSequence() { return false; diff --git a/core/src/com/cloud/storage/JavaStorageLayer.java b/core/src/com/cloud/storage/JavaStorageLayer.java index 525d42997e1..bfaa767eaed 100644 --- a/core/src/com/cloud/storage/JavaStorageLayer.java +++ b/core/src/com/cloud/storage/JavaStorageLayer.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.UUID; import javax.ejb.Local; import javax.naming.ConfigurationException; @@ -163,6 +164,21 @@ public class JavaStorageLayer implements StorageLayer { return file.length(); } + @Override + public File createUniqDir() { + String dirName = System.getProperty("java.io.tmpdir"); + if (dirName != null) { + File dir = new File(dirName); + if (dir.exists()) { + String uniqDirName = dir.getAbsolutePath() + File.separator + UUID.randomUUID().toString(); + if (this.mkdir(uniqDirName)) { + return new File(uniqDirName); + } + } + } + return null; + } + @Override public boolean mkdirs(String path) { synchronized(path.intern()) { diff --git a/core/src/com/cloud/storage/StorageLayer.java b/core/src/com/cloud/storage/StorageLayer.java index b640191f940..7d8583de1ce 100644 --- a/core/src/com/cloud/storage/StorageLayer.java +++ b/core/src/com/cloud/storage/StorageLayer.java @@ -39,6 +39,8 @@ public interface StorageLayer extends Manager { */ long getSize(String path); + File createUniqDir(); + /** * Is this path a directory? * @param path path to check. diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java b/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java index 2ae3e8cdbda..929b2c8254d 100644 --- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java +++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java @@ -93,7 +93,7 @@ public class SnapshotDataStoreVO implements StateObject/ dir on it, if it doesn't exist already. # Assuming that secondaryStorageMountPath exists remotely # Just mount secondaryStorageMountPath//SecondaryStorageHost/ everytime # Never unmount. + # path is like "snapshots/account/volumeId", we mount secondary_storage:/snapshots + relativeDir = path.split("/")[0] + restDir = "/".join(path.split("/")[1:]) snapshotsDir = os.path.join(secondaryStorageMountPath, relativeDir) - # Mkdir local mount point dir, if it doesn't exist. - localMountPointPath = os.path.join(CLOUD_DIR, dcId) - localMountPointPath = os.path.join(localMountPointPath, relativeDir, secHostId) - makedirs(localMountPointPath) # if something is not mounted already on localMountPointPath, # mount secondaryStorageMountPath on localMountPath @@ -346,8 +345,7 @@ def mountSnapshotsDir(secondaryStorageMountPath, relativeDir, dcId, accountId, i mount(snapshotsDir, localMountPointPath) # Create accountId/instanceId dir on localMountPointPath, if it doesn't exist - backupsDir = os.path.join(localMountPointPath, accountId) - backupsDir = os.path.join(backupsDir, instanceId) + backupsDir = os.path.join(localMountPointPath, restDir) makedirs(backupsDir) return backupsDir @@ -485,16 +483,13 @@ def getVhdParent(session, args): def backupSnapshot(session, args): util.SMlog("Called backupSnapshot with " + str(args)) primaryStorageSRUuid = args['primaryStorageSRUuid'] - dcId = args['dcId'] - accountId = args['accountId'] - volumeId = args['volumeId'] secondaryStorageMountPath = args['secondaryStorageMountPath'] snapshotUuid = args['snapshotUuid'] prevBackupUuid = args['prevBackupUuid'] backupUuid = args['backupUuid'] isISCSI = getIsTrueString(args['isISCSI']) - secHostId = args['secHostId'] - + path = args['path'] + localMountPoint = args['localMountPoint'] primarySRPath = getPrimarySRPath(primaryStorageSRUuid, isISCSI) util.SMlog("primarySRPath: " + primarySRPath) @@ -507,7 +502,7 @@ def backupSnapshot(session, args): # Mount secondary storage mount path on XenServer along the path # /var/run/sr-mount//snapshots/ and create / dir # on it. - backupsDir = mountSnapshotsDir(secondaryStorageMountPath, "snapshots", dcId, accountId, volumeId, secHostId) + backupsDir = mountSnapshotsDir(secondaryStorageMountPath, localMountPoint, path) util.SMlog("Backups dir " + backupsDir) # Check existence of snapshot on primary storage @@ -538,13 +533,12 @@ def backupSnapshot(session, args): @echo def deleteSnapshotBackup(session, args): util.SMlog("Calling deleteSnapshotBackup with " + str(args)) - dcId = args['dcId'] - accountId = args['accountId'] - volumeId = args['volumeId'] secondaryStorageMountPath = args['secondaryStorageMountPath'] backupUUID = args['backupUUID'] + path = args['path'] + localMountPoint = args['localMountPoint'] - backupsDir = mountSnapshotsDir(secondaryStorageMountPath, "snapshots", dcId, accountId, volumeId) + backupsDir = mountSnapshotsDir(secondaryStorageMountPath, localMountPoint, path) # chdir to the backupsDir for convenience chdir(backupsDir) diff --git a/server/src/com/cloud/server/StatsCollector.java b/server/src/com/cloud/server/StatsCollector.java index 1a6e42436b0..3be74611406 100755 --- a/server/src/com/cloud/server/StatsCollector.java +++ b/server/src/com/cloud/server/StatsCollector.java @@ -534,7 +534,8 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc if ( store.getUri() == null ) { continue; } - GetStorageStatsCommand command = new GetStorageStatsCommand(store.getUri()); + + GetStorageStatsCommand command = new GetStorageStatsCommand(store.getTO()); EndPoint ssAhost = _epSelector.select(store); if (ssAhost == null) { s_logger.debug("There is no secondary storage VM for secondary storage host " + store.getName()); diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 003c87acd5b..a82714b7fca 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -164,6 +164,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S public void disconnected() { } + public void setInSystemVM(boolean inSystemVM) { + this._inSystemVM = inSystemVM; + } + @Override public Answer executeRequest(Command cmd) { if (cmd instanceof DownloadProgressCommand) { @@ -215,6 +219,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S protected CopyCmdAnswer postProcessing(File destFile, String downloadPath, String destPath, DataTO srcData, DataTO destData) throws ConfigurationException { + if (destData.getObjectType() == DataObjectType.SNAPSHOT) { + SnapshotObjectTO snapshot = new SnapshotObjectTO(); + snapshot.setPath(destPath + File.separator + destFile.getName()); + + CopyCmdAnswer answer = new CopyCmdAnswer(snapshot); + return answer; + } // do post processing to unzip the file if it is compressed String scriptsDir = "scripts/storage/secondary"; String createTmpltScr = Script.findScript(scriptsDir, "createtmplt.sh"); @@ -238,7 +249,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String extension = null; if (srcData.getObjectType() == DataObjectType.TEMPLATE) { extension = ((TemplateObjectTO) srcData).getFormat().getFileExtension(); - } else { + } else if (srcData.getObjectType() == DataObjectType.VOLUME) { extension = ((VolumeObjectTO) srcData).getFormat().getFileExtension(); } @@ -403,11 +414,51 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S if (srcData.getHypervisorType() == HypervisorType.XenServer) { return copySnapshotToTemplateFromNfsToNfsXenserver(cmd, srcData, srcDataStore, destData, destDataStore); + } else if (srcData.getHypervisorType() == HypervisorType.KVM) { + File srcFile = getFile(srcData.getPath(), srcDataStore.getUrl()); + File destFile = getFile(destData.getPath(), destDataStore.getUrl()); + s_logger.debug("copy snapshot to template"); + Script.runSimpleBashScript("cp " + srcFile.getAbsolutePath() + " " + destFile.getAbsolutePath()); + QCOW2Processor processor = new QCOW2Processor(); + Map params = new HashMap(); + params.put(StorageLayer.InstanceConfigKey, _storage); + try { + processor.configure("qcow2 processor", params); + String destPath = destFile.getAbsolutePath(); + String templateName = srcFile.getName(); + FormatInfo info = processor.process(destPath, null, templateName); + TemplateLocation loc = new TemplateLocation(_storage, destPath); + loc.create(1, true, srcFile.getName()); + loc.addFormat(info); + loc.save(); + TemplateProp prop = loc.getTemplateInfo(); + TemplateObjectTO newTemplate = new TemplateObjectTO(); + newTemplate.setPath(destData.getPath() + File.separator + templateName); + newTemplate.setFormat(ImageFormat.VHD); + newTemplate.setSize(prop.getSize()); + return new CopyCmdAnswer(newTemplate); + } catch (ConfigurationException e) { + s_logger.debug("Failed to create template:" + e.toString()); + return new CopyCmdAnswer(e.toString()); + } catch (IOException e) { + s_logger.debug("Failed to create template:" + e.toString()); + return new CopyCmdAnswer(e.toString()); + } } return new CopyCmdAnswer(""); } + protected File getFile(String path, String nfsPath) { + String filePath = getRootDir(nfsPath) + File.separator + path; + File f = new File(filePath); + if (!f.exists()) { + _storage.mkdirs(filePath); + f = new File(filePath); + } + return f; + } + protected Answer createTemplateFromSnapshot(CopyCommand cmd) { DataTO srcData = cmd.getSrcTO(); DataTO destData = cmd.getDestTO(); @@ -422,8 +473,32 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S if (destDataStore instanceof NfsTO) { return copySnapshotToTemplateFromNfsToNfs(cmd, (SnapshotObjectTO) srcData, (NfsTO) srcDataStore, (TemplateObjectTO) destData, (NfsTO) destDataStore); - } + } else if (destDataStore instanceof SwiftTO) { + //create template on the same data store + CopyCmdAnswer answer = (CopyCmdAnswer)copySnapshotToTemplateFromNfsToNfs(cmd, (SnapshotObjectTO) srcData, (NfsTO) srcDataStore, + (TemplateObjectTO) destData, (NfsTO) srcDataStore); + if (!answer.getResult()) { + return answer; + } + s_logger.debug("starting copy template to swift"); + DataTO newTemplate = (DataTO)answer.getNewData(); + File templateFile = getFile(newTemplate.getPath(), ((NfsTO) srcDataStore).getUrl()); + SwiftTO swift = (SwiftTO)destDataStore; + String containterName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId()); + String swiftPath = SwiftUtil.putObject(swift, templateFile, containterName, templateFile.getName()); + //upload template.properties + File properties = new File(templateFile.getParent() + File.separator + _tmpltpp); + if (properties.exists()) { + SwiftUtil.putObject(swift, properties, containterName, _tmpltpp); + } + TemplateObjectTO template = new TemplateObjectTO(); + template.setPath(swiftPath); + template.setSize(templateFile.length()); + SnapshotObjectTO snapshot = (SnapshotObjectTO)srcData; + template.setFormat(snapshot.getVolume().getFormat()); + return new CopyCmdAnswer(template); + } } return new CopyCmdAnswer(""); } @@ -581,9 +656,29 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S File file = null; try { NfsTO nfsCacheStore = (NfsTO)cacheStore; - String fileName = UUID.randomUUID().toString() + "." + cmd.getFormat().getFileExtension(); + String fileName = cmd.getName() + "." + cmd.getFormat().getFileExtension(); file = downloadFromUrlToNfs(cmd.getUrl(), nfsCacheStore, path, fileName); - String swiftPath = SwiftUtil.putObject(swiftTO, file, "T-" + cmd.getId()); + String container = "T-" + cmd.getId(); + String swiftPath = SwiftUtil.putObject(swiftTO, file, container, null); + + //put metda file + File uniqDir = _storage.createUniqDir(); + String metaFileName = uniqDir.getAbsolutePath() + File.separator + "template.properties"; + _storage.create(uniqDir.getAbsolutePath(), "template.properties"); + File metaFile = new File(metaFileName); + FileWriter writer = new FileWriter(metaFile); + BufferedWriter bufferWriter = new BufferedWriter(writer); + bufferWriter.write("uniquename=" + cmd.getName()); + bufferWriter.write("\n"); + bufferWriter.write("filename=" + fileName); + bufferWriter.write("\n"); + bufferWriter.write("size=" + file.length()); + bufferWriter.close(); + writer.close(); + + SwiftUtil.putObject(swiftTO, metaFile, container, "template.properties"); + metaFile.delete(); + uniqDir.delete(); String md5sum = null; try { md5sum = DigestUtils.md5Hex(new FileInputStream(file)); @@ -1361,32 +1456,52 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } Map swiftListTemplate(SwiftTO swift) { - String[] containers = swiftList(swift, "", ""); + String[] containers = SwiftUtil.list(swift, "", null); if (containers == null) { return null; } Map tmpltInfos = new HashMap(); for (String container : containers) { if (container.startsWith("T-")) { - String ldir = _tmpltDir + "/" + UUID.randomUUID().toString(); - createLocalDir(ldir); - String lFullPath = ldir + "/" + _tmpltpp; - swiftDownload(swift, container, _tmpltpp, lFullPath); - TemplateLocation loc = new TemplateLocation(_storage, ldir); - try { - if (!loc.load()) { - s_logger.warn("Can not parse template.properties file for template " + container); - continue; - } - } catch (IOException e) { - s_logger.warn("Unable to load template location " + ldir + " due to " + e.toString(), e); + String[] files = SwiftUtil.list(swift, container, "template.properties"); + if (files.length != 1) { + continue; + } + try { + File tempFile = File.createTempFile("template", ".tmp"); + File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + "template.properties"); + if (tmpFile == null) { + continue; + } + FileReader fr = new FileReader(tmpFile); + BufferedReader brf = new BufferedReader(fr); + String line = null; + String uniqName = null; + Long size = null; + String name = null; + while ((line = brf.readLine()) != null) { + if (line.startsWith("uniquename=")) { + uniqName = line.split("=")[1]; + } else if (line.startsWith("size=")) { + size = Long.parseLong(line.split("=")[1]); + } else if (line.startsWith("filename=")) { + name = line.split("=")[1]; + } + } + brf.close(); + tempFile.delete(); + if (uniqName != null) { + TemplateProp prop = new TemplateProp(uniqName, container + File.separator + name, size, size, true, false); + tmpltInfos.put(uniqName, prop); + } + + } catch (IOException e) { + s_logger.debug("Failed to create templ file:" + e.toString()); + continue; + } catch (Exception e) { + s_logger.debug("Failed to get properties: " + e.toString()); continue; } - TemplateProp tInfo = loc.getTemplateInfo(); - tInfo.setInstallPath(container); - tmpltInfos.put(tInfo.getTemplateName(), tInfo); - loc.purge(); - deleteLocalDir(ldir); } } return tmpltInfos; @@ -1612,7 +1727,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } protected GetStorageStatsAnswer execute(final GetStorageStatsCommand cmd) { - String rootDir = getRootDir(cmd.getSecUrl()); + DataStoreTO store = cmd.getStore(); + if (store instanceof S3TO || store instanceof SwiftTO) { + long infinity = Integer.MAX_VALUE; + return new GetStorageStatsAnswer(cmd, infinity, 0L); + } + + String rootDir = getRootDir(((NfsTO) store).getUrl()); final long usedSize = getUsedSize(rootDir); final long totalSize = getTotalSize(rootDir); if (usedSize == -1 || totalSize == -1) { diff --git a/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java b/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java index 7723321060d..0c355ec884d 100644 --- a/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java +++ b/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java @@ -20,6 +20,8 @@ package org.apache.cloudstack.storage.resource; import com.cloud.agent.api.Answer; import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.agent.api.storage.ListTemplateAnswer; +import com.cloud.agent.api.storage.ListTemplateCommand; import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.SwiftTO; @@ -28,9 +30,11 @@ import com.cloud.storage.Storage; import com.cloud.utils.SwiftUtil; import junit.framework.Assert; import junit.framework.TestCase; +import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.junit.Before; import org.junit.Test; @@ -39,6 +43,7 @@ import org.mockito.Mockito; import javax.naming.ConfigurationException; import java.util.HashMap; +import java.util.UUID; public class LocalNfsSecondaryStorageResourceTest extends TestCase { LocalNfsSecondaryStorageResource resource; @@ -46,6 +51,8 @@ public class LocalNfsSecondaryStorageResourceTest extends TestCase { @Override public void setUp() throws ConfigurationException { resource = new LocalNfsSecondaryStorageResource(); + resource.setInSystemVM(true); + resource.setParentPath("/mnt"); System.setProperty("paths.script", "/Users/edison/develop/asf-master/script"); //resource.configure("test", new HashMap()); @@ -59,14 +66,15 @@ public class LocalNfsSecondaryStorageResourceTest extends TestCase { Mockito.when(swift.getEndPoint()).thenReturn("https://objects.dreamhost.com/auth"); Mockito.when(swift.getAccount()).thenReturn("cloudstack"); Mockito.when(swift.getUserName()).thenReturn("images"); - //Mockito.when(swift.getKey()).thenReturn("something"); + Mockito.when(swift.getKey()).thenReturn("oxvELQaOD1U5_VyosGfA-wpZ7uBWEff-CUBGCM0u"); Mockito.when(template.getDataStore()).thenReturn(swift); Mockito.when(template.getPath()).thenReturn("template/1/1/"); Mockito.when(template.isRequiresHvm()).thenReturn(true); Mockito.when(template.getId()).thenReturn(1L); Mockito.when(template.getFormat()).thenReturn(Storage.ImageFormat.VHD); - Mockito.when(template.getOrigUrl()).thenReturn("http://nfs1.lab.vmops.com/templates/ttylinux_pv.vhd"); + Mockito.when(template.getOrigUrl()).thenReturn("http://nfs1.lab.vmops.com/templates/test.bz2"); + Mockito.when(template.getName()).thenReturn(UUID.randomUUID().toString()); Mockito.when(template.getObjectType()).thenReturn(DataObjectType.TEMPLATE); DownloadCommand cmd = new DownloadCommand(template, 100000L); @@ -86,5 +94,11 @@ public class LocalNfsSecondaryStorageResourceTest extends TestCase { CopyCmdAnswer copyCmdAnswer = (CopyCmdAnswer)resource.executeRequest(cpyCmd); Assert.assertTrue(copyCmdAnswer.getResult()); + //list template + ListTemplateCommand listCmd = new ListTemplateCommand(swift); + ListTemplateAnswer listAnswer = (ListTemplateAnswer)resource.executeRequest(listCmd); + + Assert.assertTrue(listAnswer.getTemplateInfo().size() > 0); } + } diff --git a/setup/db/db/schema-410to420.sql b/setup/db/db/schema-410to420.sql index 7fc5ea2f44f..e109f9bcd4d 100644 --- a/setup/db/db/schema-410to420.sql +++ b/setup/db/db/schema-410to420.sql @@ -161,7 +161,7 @@ CREATE TABLE `cloud`.`template_store_ref` ( `destroyed` tinyint(1) COMMENT 'indicates whether the template_store entry was destroyed by the user or not', `is_copy` tinyint(1) NOT NULL DEFAULT 0 COMMENT 'indicates whether this was copied ', `update_count` bigint unsigned, - `ref_cnt` bigint unsigned, + `ref_cnt` bigint unsigned DEFAULT 0, `updated` datetime, PRIMARY KEY (`id`), -- CONSTRAINT `fk_template_store_ref__store_id` FOREIGN KEY `fk_template_store_ref__store_id` (`store_id`) REFERENCES `image_store` (`id`) ON DELETE CASCADE, diff --git a/utils/src/com/cloud/utils/SwiftUtil.java b/utils/src/com/cloud/utils/SwiftUtil.java index c01de86e948..85da505707a 100644 --- a/utils/src/com/cloud/utils/SwiftUtil.java +++ b/utils/src/com/cloud/utils/SwiftUtil.java @@ -24,6 +24,8 @@ import com.cloud.utils.script.Script; import org.apache.log4j.Logger; import java.io.File; +import java.util.HashMap; +import java.util.Map; public class SwiftUtil { @@ -45,8 +47,34 @@ public class SwiftUtil { return swiftCLI; } - public static String putObject(SwiftClientCfg cfg, File srcFile, String container) { + public static boolean postMeta(SwiftClientCfg cfg, String container, String object, Map metas) { String swiftCli = getSwiftCLIPath(); + StringBuilder cms = new StringBuilder(); + for(Map.Entry entry : metas.entrySet()) { + cms.append(" -m "); + cms.append(entry.getKey()); + cms.append(":"); + cms.append(entry.getValue()); + cms.append(" "); + } + Script command = new Script("/bin/bash", logger); + command.add("-c"); + command.add("/usr/bin/python " + swiftCli + " -A " + + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + + cfg.getKey() + " post " + container + " " + object + " " + cms.toString()); + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String result = command.execute(parser); + if (result != null) { + throw new CloudRuntimeException("Failed to post meta" + result); + } + return true; + } + + public static String putObject(SwiftClientCfg cfg, File srcFile, String container, String fileName) { + String swiftCli = getSwiftCLIPath(); + if (fileName == null) { + fileName = srcFile.getName(); + } String srcDirectory = srcFile.getParent(); Script command = new Script("/bin/bash", logger); long size = srcFile.length(); @@ -55,12 +83,12 @@ public class SwiftUtil { command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " - + cfg.getKey() + " upload " + container + " " + srcFile.getName()); + + cfg.getKey() + " upload " + container + " " + fileName); } else { command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " - + cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + srcFile.getName()); + + cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + fileName); } OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); String result = command.execute(parser); @@ -71,19 +99,74 @@ public class SwiftUtil { if (parser.getLines() != null) { String[] lines = parser.getLines().split("\\n"); for (String line : lines) { - if (line.contains("Errno") || line.contains("failed")) { + if (line.contains("Errno") || line.contains("failed") || line.contains("not found")) { throw new CloudRuntimeException("Failed to upload file: " + lines.toString()); } } } + return container + File.separator + srcFile.getName(); } + private static StringBuilder buildSwiftCmd(SwiftClientCfg swift) { + String swiftCli = getSwiftCLIPath(); + StringBuilder sb = new StringBuilder(); + sb.append(" /usr/bin/python "); + sb.append(swiftCli); + sb.append(" -A "); + sb.append(swift.getEndPoint()); + sb.append(" -U "); + sb.append(swift.getAccount()); + sb.append(":"); + sb.append(swift.getUserName()); + sb.append(" -K "); + sb.append(swift.getKey()); + sb.append(" "); + return sb; + } + + public static String[] list(SwiftClientCfg swift, String container, String rFilename) { + String swiftCli = getSwiftCLIPath(); + Script command = new Script("/bin/bash", logger); + command.add("-c"); + + StringBuilder swiftCmdBuilder = buildSwiftCmd(swift); + swiftCmdBuilder.append(" list "); + swiftCmdBuilder.append(container); + + if (rFilename != null) { + swiftCmdBuilder.append(" -p "); + swiftCmdBuilder.append(rFilename); + } + + command.add(swiftCmdBuilder.toString()); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = command.execute(parser); + if (result == null && parser.getLines() != null && !parser.getLines().equalsIgnoreCase("")) { + String[] lines = parser.getLines().split("\\n"); + return lines; + } else { + if (result != null) { + String errMsg = "swiftList failed , err=" + result; + logger.debug("Failed to list " + errMsg); + } else { + String errMsg = "swiftList failed, no lines returns"; + logger.debug("Failed to list " + errMsg); + } + } + return new String[0]; + } + public static File getObject(SwiftClientCfg cfg, File destDirectory, String swiftPath) { int firstIndexOfSeparator = swiftPath.indexOf(File.separator); String container = swiftPath.substring(0, firstIndexOfSeparator); String srcPath = swiftPath.substring(firstIndexOfSeparator + 1); - String destFilePath = destDirectory.getAbsolutePath() + File.separator + srcPath; + String destFilePath = null; + if (destDirectory.isDirectory()) { + destFilePath = destDirectory.getAbsolutePath() + File.separator + srcPath; + } else { + destFilePath = destDirectory.getAbsolutePath(); + } String swiftCli = getSwiftCLIPath(); Script command = new Script("/bin/bash", logger); command.add("-c"); @@ -109,4 +192,15 @@ public class SwiftUtil { } return new File(destFilePath); } + + public static String getContainerName(String type, Long id) { + if (type.startsWith("T")) { + return "T-" + id; + } else if (type.startsWith("S")) { + return "S-" + id; + } else if (type.startsWith("V")) { + return "V-" + id; + } + return null; + } }