apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix Sync of template.properties in Swift

Browse Source
This commit is contained in:
Syed 2015-12-14 17:37:28 -05:00
parent 7a0b37a29a
commit f5ac8ddded
6 changed files with 477 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java
View File

@ -156,6 +156,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager {
// template.properties
// there
}
ts.setInstallPath(installPath);
ts.setState(ObjectInDataStoreStateMachine.State.Allocated);
ts = templateDataStoreDao.persist(ts);

178
server/src/com/cloud/test/TestAppender.java Normal file
View File

@ -0,0 +1,178 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.test;
import com.google.common.base.Joiner;
import com.google.common.base.Objects;
import com.google.common.collect.ImmutableMap;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
import org.springframework.util.Assert;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.isNullOrEmpty;
import static java.lang.String.format;
import static org.apache.log4j.Level.ALL;
import static org.apache.log4j.Level.DEBUG;
import static org.apache.log4j.Level.ERROR;
import static org.apache.log4j.Level.FATAL;
import static org.apache.log4j.Level.INFO;
import static org.apache.log4j.Level.OFF;
/**
*
* Tracks one or more patterns to determine whether or not they have been
* logged. It uses a streaming approach to determine whether or not a message
* has a occurred to prevent unnecessary memory consumption. Instances of this
* of this class are created using the {@link TestAppenderBuilder}.
*
* To use this class, register a one or more expected patterns by level as part
* of the test setup and retain an reference to the appender instance. After the
* expected logging events have occurred in the test case, call
* {@link TestAppender#assertMessagesLogged()} which will fail the test if any of the
* expected patterns were not logged.
*
*/
public final class TestAppender extends AppenderSkeleton {
private final static String APPENDER_NAME = "test_appender";
private final ImmutableMap<Level, Set<PatternResult>> expectedPatternResults;
private TestAppender(final Map<Level, Set<PatternResult>> expectedPatterns) {
super();
expectedPatternResults = ImmutableMap.copyOf(expectedPatterns);
}
protected void append(LoggingEvent loggingEvent) {
checkArgument(loggingEvent != null, "append requires a non-null loggingEvent");
final Level level = loggingEvent.getLevel();
checkState(expectedPatternResults.containsKey(level), "level " + level + " not supported by append");
for (final PatternResult patternResult : expectedPatternResults.get(level)) {
if (patternResult.getPattern().matcher(loggingEvent.getRenderedMessage()).matches()) {
patternResult.markFound();
}
}
}
public void close() {
// Do nothing ...
}
public boolean requiresLayout() {
return false;
}
public void assertMessagesLogged() {
final List<String> unloggedPatterns = new ArrayList<>();
for (final Map.Entry<Level, Set<PatternResult>> expectedPatternResult : expectedPatternResults.entrySet()) {
for (final PatternResult patternResults : expectedPatternResult.getValue()) {
if (!patternResults.isFound()) {
unloggedPatterns.add(format("%1$s was not logged for level %2$s",
patternResults.getPattern().toString(), expectedPatternResult.getKey()));
}
}
}
if (!unloggedPatterns.isEmpty()) {
//Raise an assert
Assert.isTrue(false, Joiner.on(",").join(unloggedPatterns));
}
}
private static final class PatternResult {
private final Pattern pattern;
private boolean foundFlag = false;
private PatternResult(Pattern pattern) {
super();
this.pattern = pattern;
}
public Pattern getPattern() {
return pattern;
}
public void markFound() {
// This operation is thread-safe because the value will only ever be switched from false to true. Therefore,
// multiple threads mutating the value for a pattern will not corrupt the value ...
foundFlag = true;
}
public boolean isFound() {
return foundFlag;
}
@Override
public boolean equals(Object thatObject) {
if (this == thatObject) {
return true;
}
if (thatObject == null || getClass() != thatObject.getClass()) {
return false;
}
PatternResult thatPatternResult = (PatternResult) thatObject;
return foundFlag == thatPatternResult.foundFlag &&
Objects.equal(pattern, thatPatternResult.pattern);
}
@Override
public int hashCode() {
return Objects.hashCode(pattern, foundFlag);
}
@Override
public String toString() {
return format("Pattern Result [ pattern: %1$s, markFound: %2$s ]", pattern.toString(), foundFlag);
}
}
public static final class TestAppenderBuilder {
private final Map<Level, Set<PatternResult>> expectedPatterns;
public TestAppenderBuilder() {
super();
expectedPatterns = new HashMap<>();
expectedPatterns.put(ALL, new HashSet<PatternResult>());
expectedPatterns.put(DEBUG, new HashSet<PatternResult>());
expectedPatterns.put(ERROR, new HashSet<PatternResult>());
expectedPatterns.put(FATAL, new HashSet<PatternResult>());
expectedPatterns.put(INFO, new HashSet<PatternResult>());
expectedPatterns.put(OFF, new HashSet<PatternResult>());
}
public TestAppenderBuilder addExpectedPattern(final Level level, final String pattern) {
checkArgument(level != null, "addExpectedPattern requires a non-null level");
checkArgument(!isNullOrEmpty(pattern), "addExpectedPattern requires a non-blank pattern");
checkState(expectedPatterns.containsKey(level), "level " + level + " is not supported by " + getClass().getName());
expectedPatterns.get(level).add(new PatternResult(Pattern.compile(pattern)));
return this;
}
public TestAppender build() {
return new TestAppender(expectedPatterns);
}
}
/**
*
* Attaches a {@link TestAppender} to a {@link Logger} and ensures that it is the only
* test appender attached to the logger.
*
* @param logger The logger which will be monitored by the test
* @param testAppender The test appender to attach to {@code logger}
*/
public static void safeAddAppender(Logger logger, TestAppender testAppender) {
logger.removeAppender(APPENDER_NAME);
logger.addAppender(testAppender);
}
}

3
services/secondary-storage/server/pom.xml
View File

@ -26,9 +26,6 @@
<version>4.7.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<properties>
<skipTests>true</skipTests>
</properties>
<dependencies>
<dependency>
<groupId>log4j</groupId>

371
services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
View File

@ -16,91 +16,7 @@
// under the License.
package org.apache.cloudstack.storage.resource;
import static com.cloud.utils.storage.S3.S3Utils.putFile;
import static com.cloud.utils.StringUtils.join;
import static java.lang.String.format;
import static java.util.Arrays.asList;
import static org.apache.commons.lang.StringUtils.substringAfterLast;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.naming.ConfigurationException;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.storage.Storage;
import com.cloud.storage.template.TemplateConstants;
import com.cloud.utils.EncryptionUtil;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.http.HttpContentCompressor;
import io.netty.handler.codec.http.HttpRequestDecoder;
import io.netty.handler.codec.http.HttpResponseEncoder;
import io.netty.handler.logging.LogLevel;
import io.netty.handler.logging.LoggingHandler;
import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
import org.apache.cloudstack.storage.template.UploadEntity;
import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.log4j.Logger;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.DeleteCommand;
import org.apache.cloudstack.storage.command.DownloadCommand;
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
import org.apache.cloudstack.storage.command.UploadStatusAnswer;
import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
import org.apache.cloudstack.storage.command.UploadStatusCommand;
import org.apache.cloudstack.storage.template.DownloadManager;
import org.apache.cloudstack.storage.template.DownloadManagerImpl;
import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
import org.apache.cloudstack.storage.template.UploadManager;
import org.apache.cloudstack.storage.template.UploadManagerImpl;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.CheckHealthAnswer;
import com.cloud.agent.api.CheckHealthCommand;
@ -135,11 +51,13 @@ import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;
import com.cloud.agent.api.to.SwiftTO;
import com.cloud.exception.InternalErrorException;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.host.Host;
import com.cloud.host.Host.Type;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.resource.ServerResourceBase;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.StorageLayer;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
@ -149,24 +67,102 @@ import com.cloud.storage.template.Processor.FormatInfo;
import com.cloud.storage.template.QCOW2Processor;
import com.cloud.storage.template.RawImageProcessor;
import com.cloud.storage.template.TARProcessor;
import com.cloud.storage.template.TemplateConstants;
import com.cloud.storage.template.TemplateLocation;
import com.cloud.storage.template.TemplateProp;
import com.cloud.storage.template.VhdProcessor;
import com.cloud.storage.template.VmdkProcessor;
import com.cloud.utils.EncryptionUtil;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.storage.S3.S3Utils;
import com.cloud.utils.SwiftUtil;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.net.NetUtils;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
import com.cloud.utils.storage.S3.S3Utils;
import com.cloud.vm.SecondaryStorageVm;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.http.HttpContentCompressor;
import io.netty.handler.codec.http.HttpRequestDecoder;
import io.netty.handler.codec.http.HttpResponseEncoder;
import io.netty.handler.logging.LogLevel;
import io.netty.handler.logging.LoggingHandler;
import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.DeleteCommand;
import org.apache.cloudstack.storage.command.DownloadCommand;
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
import org.apache.cloudstack.storage.command.UploadStatusAnswer;
import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
import org.apache.cloudstack.storage.command.UploadStatusCommand;
import org.apache.cloudstack.storage.template.DownloadManager;
import org.apache.cloudstack.storage.template.DownloadManagerImpl;
import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
import org.apache.cloudstack.storage.template.UploadEntity;
import org.apache.cloudstack.storage.template.UploadManager;
import org.apache.cloudstack.storage.template.UploadManagerImpl;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.log4j.Logger;
import org.joda.time.DateTime;
import org.joda.time.format.ISODateTimeFormat;
import javax.naming.ConfigurationException;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import static com.cloud.utils.StringUtils.join;
import static com.cloud.utils.storage.S3.S3Utils.putFile;
import static java.lang.String.format;
import static java.util.Arrays.asList;
import static org.apache.commons.lang.StringUtils.substringAfterLast;
public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource {
private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
public static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
private static final String TEMPLATE_ROOT_DIR = "template/tmpl";
private static final String VOLUME_ROOT_DIR = "volumes";
@ -499,10 +495,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
String destFileFullPath = destFile.getAbsolutePath() + File.separator + fileName;
s_logger.debug("copy snapshot " + srcFile.getAbsolutePath() + " to template " + destFileFullPath);
Script.runSimpleBashScript("cp " + srcFile.getAbsolutePath() + " " + destFileFullPath);
String metaFileName = destFile.getAbsolutePath() + File.separator + "template.properties";
String metaFileName = destFile.getAbsolutePath() + File.separator + _tmpltpp;
File metaFile = new File(metaFileName);
try {
_storage.create(destFile.getAbsolutePath(), "template.properties");
_storage.create(destFile.getAbsolutePath(), _tmpltpp);
try ( // generate template.properties file
FileWriter writer = new FileWriter(metaFile);
BufferedWriter bufferWriter = new BufferedWriter(writer);
@ -597,32 +593,14 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
return answer;
}
s_logger.debug("starting copy template to swift");
DataTO newTemplate = answer.getNewData();
File templateFile = getFile(newTemplate.getPath(), ((NfsTO)srcDataStore).getUrl());
SwiftTO swift = (SwiftTO)destDataStore;
String containterName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId());
String swiftPath = SwiftUtil.putObject(swift, templateFile, containterName, templateFile.getName());
//upload template.properties
File properties = new File(templateFile.getParent() + File.separator + _tmpltpp);
if (properties.exists()) {
SwiftUtil.putObject(swift, properties, containterName, _tmpltpp);
}
TemplateObjectTO newTemplate = (TemplateObjectTO)answer.getNewData();
newTemplate.setDataStore(srcDataStore);
CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(), cmd.executeInSequence());
Answer result = copyFromNfsToSwift(newCpyCmd);
//clean up template data on staging area
try {
DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
execute(deleteCommand);
} catch (Exception e) {
s_logger.debug("Failed to clean up staging area:", e);
}
cleanupStagingNfs(newTemplate);
return result;
TemplateObjectTO template = new TemplateObjectTO();
template.setPath(swiftPath);
template.setSize(templateFile.length());
template.setPhysicalSize(template.getSize());
SnapshotObjectTO snapshot = (SnapshotObjectTO)srcData;
template.setFormat(snapshot.getVolume().getFormat());
return new CopyCmdAnswer(template);
} else if (destDataStore instanceof S3TO) {
//create template on the same data store
CopyCmdAnswer answer =
@ -635,18 +613,27 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
newTemplate.setDataStore(srcDataStore);
CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(), cmd.executeInSequence());
Answer result = copyFromNfsToS3(newCpyCmd);
//clean up template data on staging area
try {
DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
execute(deleteCommand);
} catch (Exception e) {
s_logger.debug("Failed to clean up staging area:", e);
}
cleanupStagingNfs(newTemplate);
return result;
}
}
s_logger.debug("Failed to create templat from snapshot");
return new CopyCmdAnswer("Unsupported prototcol");
s_logger.debug("Failed to create template from snapshot");
return new CopyCmdAnswer("Unsupported protocol");
}
/**
* clean up template data on staging area
* @param newTemplate: The template on the secondary storage that needs to be cleaned up
*/
protected void cleanupStagingNfs(TemplateObjectTO newTemplate) {
try {
DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
execute(deleteCommand);
} catch (Exception e) {
s_logger.debug("Failed to clean up staging area:", e);
}
}
protected Answer copyFromNfsToImage(CopyCommand cmd) {
@ -759,22 +746,18 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
String container = "T-" + cmd.getId();
String swiftPath = SwiftUtil.putObject(swiftTO, file, container, null);
long virtualSize = getVirtualSize(file, getTemplateFormat(file.getName()));
long size = file.length();
String uniqueName = cmd.getName();
//put metda file
File uniqDir = _storage.createUniqDir();
String metaFileName = uniqDir.getAbsolutePath() + File.separator + "template.properties";
_storage.create(uniqDir.getAbsolutePath(), "template.properties");
File metaFile = new File(metaFileName);
FileWriter writer = new FileWriter(metaFile);
BufferedWriter bufferWriter = new BufferedWriter(writer);
bufferWriter.write("uniquename=" + cmd.getName());
bufferWriter.write("\n");
bufferWriter.write("filename=" + fileName);
bufferWriter.write("\n");
bufferWriter.write("size=" + file.length());
bufferWriter.close();
writer.close();
String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp;
_storage.create(uniqDir.getAbsolutePath(), _tmpltpp);
SwiftUtil.putObject(swiftTO, metaFile, container, "template.properties");
File metaFile = swiftWriteMetadataFile(metaFileName, uniqueName, fileName, size, virtualSize);
SwiftUtil.putObject(swiftTO, metaFile, container, _tmpltpp);
metaFile.delete();
uniqDir.delete();
String md5sum = null;
@ -785,7 +768,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
DownloadAnswer answer =
new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, file.length(), file.length(), md5sum);
new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, virtualSize, file.length(), md5sum);
return answer;
} catch (IOException e) {
s_logger.debug("Failed to register template into swift", e);
@ -942,6 +925,118 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
}
/***
*This method will create a file using the filenName and metaFileName.
*That file will contain the given attributes (unique name, file name, size, and virtualSize).
*
* @param metaFileName : The path of the metadata file
* @param filename :attribute: Filename of the template
* @param uniqueName :attribute: Unique name of the template
* @param size :attribute: physical size of the template
* @param virtualSize :attribute: virtual size of the template
* @return File representing the metadata file
* @throws IOException
*/
protected File swiftWriteMetadataFile(String metaFileName, String uniqueName, String filename, long size, long virtualSize) throws IOException {
File metaFile = new File(metaFileName);
FileWriter writer = new FileWriter(metaFile);
BufferedWriter bufferWriter = new BufferedWriter(writer);
bufferWriter.write("uniquename=" + uniqueName);
bufferWriter.write("\n");
bufferWriter.write("filename=" + filename);
bufferWriter.write("\n");
bufferWriter.write("size=" + size);
bufferWriter.write("\n");
bufferWriter.write("virtualsize=" + virtualSize);
bufferWriter.close();
writer.close();
return metaFile;
}
/**
* Creates a template.properties for Swift with its correct unique name
*
* @param swift The swift object
* @param srcFile Source file on the staging NFS
* @param containerName Destination container
* @return true on successful write
*/
protected boolean swiftUploadMetadataFile(SwiftTO swift, File srcFile, String containerName) throws IOException {
String uniqueName = FilenameUtils.getBaseName(srcFile.getName());
File uniqDir = _storage.createUniqDir();
String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp;
_storage.create(uniqDir.getAbsolutePath(), _tmpltpp);
long virtualSize = getVirtualSize(srcFile, getTemplateFormat(srcFile.getName()));
File metaFile = swiftWriteMetadataFile(metaFileName,
uniqueName,
srcFile.getName(),
srcFile.length(),
virtualSize);
SwiftUtil.putObject(swift, metaFile, containerName, _tmpltpp);
metaFile.delete();
uniqDir.delete();
return true;
}
/**
* Copies data from NFS and uploads it into a Swift container
*
* @param cmd CopyComand
* @return CopyCmdAnswer
*/
protected Answer copyFromNfsToSwift(CopyCommand cmd) {
final DataTO srcData = cmd.getSrcTO();
final DataTO destData = cmd.getDestTO();
DataStoreTO srcDataStore = srcData.getDataStore();
NfsTO srcStore = (NfsTO)srcDataStore;
DataStoreTO destDataStore = destData.getDataStore();
File srcFile = getFile(srcData.getPath(), srcStore.getUrl());
SwiftTO swift = (SwiftTO)destDataStore;
try {
String containerName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId());
String swiftPath = SwiftUtil.putObject(swift, srcFile, containerName, srcFile.getName());
DataTO retObj = null;
if (destData.getObjectType() == DataObjectType.TEMPLATE) {
swiftUploadMetadataFile(swift, srcFile, containerName);
TemplateObjectTO newTemplate = new TemplateObjectTO();
newTemplate.setPath(swiftPath);
newTemplate.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())));
newTemplate.setPhysicalSize(srcFile.length());
newTemplate.setFormat(getTemplateFormat(srcFile.getName()));
retObj = newTemplate;
} else if (destData.getObjectType() == DataObjectType.VOLUME) {
VolumeObjectTO newVol = new VolumeObjectTO();
newVol.setPath(containerName);
newVol.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())));
retObj = newVol;
} else if (destData.getObjectType() == DataObjectType.SNAPSHOT) {
SnapshotObjectTO newSnapshot = new SnapshotObjectTO();
newSnapshot.setPath(containerName);
retObj = newSnapshot;
}
return new CopyCmdAnswer(retObj);
} catch (Exception e) {
s_logger.error("failed to upload " + srcData.getPath(), e);
return new CopyCmdAnswer("failed to upload " + srcData.getPath() + e.toString());
}
}
String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath) {
Script command = new Script("/bin/bash", s_logger);
command.add("-c");
@ -1458,13 +1553,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
Map<String, TemplateProp> tmpltInfos = new HashMap<String, TemplateProp>();
for (String container : containers) {
if (container.startsWith("T-")) {
String[] files = SwiftUtil.list(swift, container, "template.properties");
String[] files = SwiftUtil.list(swift, container, _tmpltpp);
if (files.length != 1) {
continue;
}
try {
File tempFile = File.createTempFile("template", ".tmp");
File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + "template.properties");
File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + _tmpltpp);
if (tmpFile == null) {
continue;
}
@ -1779,7 +1874,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
} else {
boolean found = false;
for (File f : tmpltFiles) {
if (!found && f.getName().equals("template.properties")) {
if (!found && f.getName().equals(_tmpltpp)) {
found = true;
}

44
services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
View File

@ -18,29 +18,6 @@
*/
package org.apache.cloudstack.storage.resource;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import javax.naming.ConfigurationException;
import junit.framework.Assert;
import junit.framework.TestCase;
import org.apache.log4j.Logger;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.DownloadCommand;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import com.cloud.agent.api.storage.DownloadAnswer;
import com.cloud.agent.api.storage.ListTemplateAnswer;
import com.cloud.agent.api.storage.ListTemplateCommand;
@ -51,7 +28,28 @@ import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage;
import com.cloud.utils.PropertiesUtil;
import com.cloud.utils.exception.CloudRuntimeException;
import junit.framework.Assert;
import junit.framework.TestCase;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.DownloadCommand;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.log4j.Logger;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.mockito.Mockito;
import javax.naming.ConfigurationException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
@Ignore
public class LocalNfsSecondaryStorageResourceTest extends TestCase {
private static Map<String, Object> testParams;

112
services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
View File

@ -18,91 +18,67 @@
*/
package org.apache.cloudstack.storage.resource;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.util.Map;
import java.util.Properties;
import javax.naming.ConfigurationException;
import junit.framework.Assert;
import junit.framework.TestCase;
import com.cloud.test.TestAppender;
import org.apache.cloudstack.storage.command.DeleteCommand;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import com.cloud.utils.PropertiesUtil;
import com.cloud.utils.exception.CloudRuntimeException;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.StringWriter;
public class NfsSecondaryStorageResourceTest extends TestCase {
private static Map<String, Object> testParams;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.spy;
private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResourceTest.class.getName());
@RunWith(PowerMockRunner.class)
public class NfsSecondaryStorageResourceTest {
NfsSecondaryStorageResource resource;
private NfsSecondaryStorageResource resource;
@Before
@Override
public void setUp() throws ConfigurationException {
s_logger.setLevel(Level.ALL);
public void setUp() {
resource = new NfsSecondaryStorageResource();
resource.setInSystemVM(true);
testParams = PropertiesUtil.toMap(loadProperties());
resource.configureStorageLayerClass(testParams);
Object testLocalRoot = testParams.get("testLocalRoot");
if (testLocalRoot != null) {
resource.setParentPath((String)testLocalRoot);
}
}
@Test
public void testMount() throws Exception {
String sampleUriStr = "cifs://192.168.1.128/CSHV3?user=administrator&password=1pass%40word1&foo=bar";
URI sampleUri = new URI(sampleUriStr);
@PrepareForTest(NfsSecondaryStorageResource.class)
public void testSwiftWriteMetadataFile() throws Exception {
String expected = "uniquename=test\nfilename=testfile\nsize=100\nvirtualsize=1000";
s_logger.info("Check HostIp parsing");
String hostIpStr = resource.getUriHostIp(sampleUri);
Assert.assertEquals("Expected host IP " + sampleUri.getHost() + " and actual host IP " + hostIpStr + " differ.", sampleUri.getHost(), hostIpStr);
StringWriter stringWriter = new StringWriter();
BufferedWriter bufferWriter = new BufferedWriter(stringWriter);
PowerMockito.whenNew(BufferedWriter.class).withArguments(any(FileWriter.class)).thenReturn(bufferWriter);
s_logger.info("Check option parsing");
String expected = "user=administrator,password=1pass@word1,foo=bar,";
String actualOpts = resource.parseCifsMountOptions(sampleUri);
Assert.assertEquals("Options should be " + expected + " and not " + actualOpts, expected, actualOpts);
resource.swiftWriteMetadataFile("testfile", "test", "testfile", 100, 1000);
// attempt a configured mount
final Map<String, Object> params = PropertiesUtil.toMap(loadProperties());
String sampleMount = (String)params.get("testCifsMount");
if (!sampleMount.isEmpty()) {
s_logger.info("functional test, mount " + sampleMount);
URI realMntUri = new URI(sampleMount);
String mntSubDir = resource.mountUri(realMntUri);
s_logger.info("functional test, umount " + mntSubDir);
resource.umount(resource.getMountingRoot() + mntSubDir, realMntUri);
} else {
s_logger.info("no entry for testCifsMount in " + "./conf/agent.properties - skip functional test");
}
Assert.assertEquals(expected, stringWriter.toString());
}
public static Properties loadProperties() throws ConfigurationException {
Properties properties = new Properties();
final File file = PropertiesUtil.findConfigFile("agent.properties");
if (file == null) {
throw new ConfigurationException("Unable to find agent.properties.");
}
s_logger.info("agent.properties found at " + file.getAbsolutePath());
try(FileInputStream fs = new FileInputStream(file);) {
properties.load(fs);
} catch (final FileNotFoundException ex) {
throw new CloudRuntimeException("Cannot find the file: " + file.getAbsolutePath(), ex);
} catch (final IOException ex) {
throw new CloudRuntimeException("IOException in reading " + file.getAbsolutePath(), ex);
}
return properties;
}
@Test
public void testCleanupStagingNfs() throws Exception{
NfsSecondaryStorageResource spyResource = spy(resource);
RuntimeException exception = new RuntimeException();
doThrow(exception).when(spyResource).execute(any(DeleteCommand.class));
TemplateObjectTO mockTemplate = Mockito.mock(TemplateObjectTO.class);
TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder();
appenderBuilder.addExpectedPattern(Level.DEBUG, "Failed to clean up staging area:");
TestAppender testLogAppender = appenderBuilder.build();
TestAppender.safeAddAppender(NfsSecondaryStorageResource.s_logger, testLogAppender);
spyResource.cleanupStagingNfs(mockTemplate);
testLogAppender.assertMessagesLogged();
}
}