CLOUDSTACK-6940, CLOUDSTACK-7312, CLOUDSTACK-5512: Template/ISO/Volume

upload rejects some valid URL formats. Also consolidate URL format check
into one util routine.
This commit is contained in:
Min Chen 2014-08-12 11:42:17 -07:00
parent 83bd4d60f1
commit e3564658be
5 changed files with 106 additions and 122 deletions

View File

@ -24,6 +24,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Date;
@ -251,7 +252,14 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
out.seek(offset);
totalBytes += bytes;
if (!verifiedFormat && (offset >= 1048576 || offset >= remoteSize)) { //let's check format after we get 1MB or full file
String unsupportedFormat = TemplateUtils.checkTemplateFormat(file.getAbsolutePath(), getDownloadUrl());
String uripath = null;
try {
URI str = new URI(getDownloadUrl());
uripath = str.getPath();
} catch (URISyntaxException e) {
s_logger.warn("Invalid download url: " + getDownloadUrl() + ", This should not happen since we have validated the url before!!");
}
String unsupportedFormat = TemplateUtils.checkTemplateFormat(file.getAbsolutePath(), uripath);
if (unsupportedFormat == null || !unsupportedFormat.isEmpty()) {
try {
request.abort();

View File

@ -295,26 +295,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
if (userSpecifiedName == null) {
userSpecifiedName = getRandomVolumeName();
}
if ((!url.toLowerCase().endsWith("vhd")) && (!url.toLowerCase().endsWith("vhd.zip")) && (!url.toLowerCase().endsWith("vhd.bz2")) && (!url.toLowerCase().endsWith("vhdx"))
&& (!url.toLowerCase().endsWith("vhdx.zip")) && (!url.toLowerCase().endsWith("vhdx.gz")) && (!url.toLowerCase().endsWith("vhdx.bz2"))
&& (!url.toLowerCase().endsWith("vhd.gz")) && (!url.toLowerCase().endsWith("qcow2")) && (!url.toLowerCase().endsWith("qcow2.zip"))
&& (!url.toLowerCase().endsWith("qcow2.bz2")) && (!url.toLowerCase().endsWith("qcow2.gz")) && (!url.toLowerCase().endsWith("ova"))
&& (!url.toLowerCase().endsWith("ova.zip")) && (!url.toLowerCase().endsWith("ova.bz2")) && (!url.toLowerCase().endsWith("ova.gz"))
&& (!url.toLowerCase().endsWith("img")) && (!url.toLowerCase().endsWith("raw"))) {
throw new InvalidParameterValueException("Please specify a valid " + format.toLowerCase());
}
if ((format.equalsIgnoreCase("vhd") && (!url.toLowerCase().endsWith(".vhd") && !url.toLowerCase().endsWith("vhd.zip") && !url.toLowerCase().endsWith("vhd.bz2") && !url
.toLowerCase().endsWith("vhd.gz")))
|| (format.equalsIgnoreCase("vhdx") && (!url.toLowerCase().endsWith(".vhdx") && !url.toLowerCase().endsWith("vhdx.zip") && !url.toLowerCase().endsWith("vhdx.bz2") && !url
.toLowerCase().endsWith("vhdx.gz")))
|| (format.equalsIgnoreCase("qcow2") && (!url.toLowerCase().endsWith(".qcow2") && !url.toLowerCase().endsWith("qcow2.zip")
&& !url.toLowerCase().endsWith("qcow2.bz2") && !url.toLowerCase().endsWith("qcow2.gz")))
|| (format.equalsIgnoreCase("ova") && (!url.toLowerCase().endsWith(".ova") && !url.toLowerCase().endsWith("ova.zip") && !url.toLowerCase().endsWith("ova.bz2") && !url
.toLowerCase().endsWith("ova.gz"))) || (format.equalsIgnoreCase("raw") && (!url.toLowerCase().endsWith(".img") && !url.toLowerCase().endsWith("raw")))) {
throw new InvalidParameterValueException("Please specify a valid URL. URL:" + url + " is an invalid for the format " + format.toLowerCase());
}
UriUtils.validateUrl(url);
UriUtils.validateUrl(format, url);
// Check that the resource limit for secondary storage won't be exceeded

View File

@ -16,8 +16,6 @@
// under the License.
package com.cloud.template;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@ -113,13 +111,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
public TemplateProfile prepare(RegisterIsoCmd cmd) throws ResourceAllocationException {
TemplateProfile profile = super.prepare(cmd);
String url = profile.getUrl();
if ((!url.toLowerCase().endsWith("iso")) && (!url.toLowerCase().endsWith("iso.zip")) && (!url.toLowerCase().endsWith("iso.bz2")) &&
(!url.toLowerCase().endsWith("iso.gz"))) {
throw new InvalidParameterValueException("Please specify a valid iso");
}
UriUtils.validateUrl(url);
UriUtils.validateUrl(ImageFormat.ISO.getFileExtension(), url);
profile.setUrl(url);
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
@ -130,88 +122,13 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocationException {
TemplateProfile profile = super.prepare(cmd);
String url = profile.getUrl();
String path = null;
try {
URL str = new URL(url);
path = str.getPath();
} catch (MalformedURLException ex) {
throw new InvalidParameterValueException("Please specify a valid URL. URL:" + url + " is invalid");
}
try {
checkFormat(cmd.getFormat(), url);
} catch (InvalidParameterValueException ex) {
checkFormat(cmd.getFormat(), path);
}
UriUtils.validateUrl(url);
UriUtils.validateUrl(cmd.getFormat(), url);
profile.setUrl(url);
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
return profile;
}
private void checkFormat(String format, String url) {
if ((!url.toLowerCase().endsWith("vhd")) && (!url.toLowerCase().endsWith("vhd.zip")) && (!url.toLowerCase().endsWith("vhd.bz2")) &&
(!url.toLowerCase().endsWith("vhdx")) && (!url.toLowerCase().endsWith("vhdx.gz")) &&
(!url.toLowerCase().endsWith("vhdx.bz2")) && (!url.toLowerCase().endsWith("vhdx.zip")) &&
(!url.toLowerCase().endsWith("vhd.gz")) && (!url.toLowerCase().endsWith("qcow2")) && (!url.toLowerCase().endsWith("qcow2.zip")) &&
(!url.toLowerCase().endsWith("qcow2.bz2")) && (!url.toLowerCase().endsWith("qcow2.gz")) && (!url.toLowerCase().endsWith("ova")) &&
(!url.toLowerCase().endsWith("ova.zip")) && (!url.toLowerCase().endsWith("ova.bz2")) && (!url.toLowerCase().endsWith("ova.gz")) &&
(!url.toLowerCase().endsWith("tar")) && (!url.toLowerCase().endsWith("tar.zip")) && (!url.toLowerCase().endsWith("tar.bz2")) &&
(!url.toLowerCase().endsWith("tar.gz")) && (!url.toLowerCase().endsWith("vmdk")) && (!url.toLowerCase().endsWith("vmdk.gz")) &&
(!url.toLowerCase().endsWith("vmdk.zip")) && (!url.toLowerCase().endsWith("vmdk.bz2")) && (!url.toLowerCase().endsWith("img")) &&
(!url.toLowerCase().endsWith("img.gz")) && (!url.toLowerCase().endsWith("img.zip")) && (!url.toLowerCase().endsWith("img.bz2")) &&
(!url.toLowerCase().endsWith("raw")) && (!url.toLowerCase().endsWith("raw.gz")) && (!url.toLowerCase().endsWith("raw.bz2")) &&
(!url.toLowerCase().endsWith("raw.zip"))) {
throw new InvalidParameterValueException("Please specify a valid " + format.toLowerCase());
}
if ((format.equalsIgnoreCase("vhd")
&& (!url.toLowerCase().endsWith("vhd")
&& !url.toLowerCase().endsWith("vhd.zip")
&& !url.toLowerCase().endsWith("vhd.bz2")
&& !url.toLowerCase().endsWith("vhd.gz")))
|| (format.equalsIgnoreCase("vhdx")
&& (!url.toLowerCase().endsWith("vhdx")
&& !url.toLowerCase().endsWith("vhdx.zip")
&& !url.toLowerCase().endsWith("vhdx.bz2")
&& !url.toLowerCase().endsWith("vhdx.gz")))
|| (format.equalsIgnoreCase("qcow2")
&& (!url.toLowerCase().endsWith("qcow2")
&& !url.toLowerCase().endsWith("qcow2.zip")
&& !url.toLowerCase().endsWith("qcow2.bz2")
&& !url.toLowerCase().endsWith("qcow2.gz")))
|| (format.equalsIgnoreCase("ova")
&& (!url.toLowerCase().endsWith("ova")
&& !url.toLowerCase().endsWith("ova.zip")
&& !url.toLowerCase().endsWith("ova.bz2")
&& !url.toLowerCase().endsWith("ova.gz")))
|| (format.equalsIgnoreCase("tar")
&& (!url.toLowerCase().endsWith("tar")
&& !url.toLowerCase().endsWith("tar.zip")
&& !url.toLowerCase().endsWith("tar.bz2")
&& !url.toLowerCase().endsWith("tar.gz")))
|| (format.equalsIgnoreCase("raw")
&& (!url.toLowerCase().endsWith("img")
&& !url.toLowerCase().endsWith("img.zip")
&& !url.toLowerCase().endsWith("img.bz2")
&& !url.toLowerCase().endsWith("img.gz")
&& !url.toLowerCase().endsWith("raw")
&& !url.toLowerCase().endsWith("raw.bz2")
&& !url.toLowerCase().endsWith("raw.zip")
&& !url.toLowerCase().endsWith("raw.gz")))
|| (format.equalsIgnoreCase("vmdk")
&& (!url.toLowerCase().endsWith("vmdk")
&& !url.toLowerCase().endsWith("vmdk.zip")
&& !url.toLowerCase().endsWith("vmdk.bz2")
&& !url.toLowerCase().endsWith("vmdk.gz")))
) {
throw new InvalidParameterValueException("Please specify a valid URL. URL:" + url + " is an invalid for the format " + format.toLowerCase());
}
}
@Override
public VMTemplateVO create(TemplateProfile profile) {
// persist entry in vm_template, vm_template_details and template_zone_ref tables, not that entry at template_store_ref is not created here, and created in createTemplateAsync.

View File

@ -45,9 +45,9 @@ import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.util.URIUtil;
import org.apache.http.NameValuePair;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.message.BasicNameValuePair;
import org.apache.log4j.Logger;
import com.cloud.utils.crypt.DBEncryptionUtil;
@ -233,10 +233,14 @@ public class UriUtils {
}
public static Pair<String, Integer> validateUrl(String url) throws IllegalArgumentException {
return validateUrl(null, url);
}
public static Pair<String, Integer> validateUrl(String format, String url) throws IllegalArgumentException {
try {
URI uri = new URI(url);
if ((uri.getScheme() == null) ||
(!uri.getScheme().equalsIgnoreCase("http") && !uri.getScheme().equalsIgnoreCase("https") && !uri.getScheme().equalsIgnoreCase("file"))) {
(!uri.getScheme().equalsIgnoreCase("http") && !uri.getScheme().equalsIgnoreCase("https") && !uri.getScheme().equalsIgnoreCase("file"))) {
throw new IllegalArgumentException("Unsupported scheme for url: " + url);
}
int port = uri.getPort();
@ -259,15 +263,88 @@ public class UriUtils {
if (hostAddr instanceof Inet6Address) {
throw new IllegalArgumentException("IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")");
}
return new Pair<String, Integer>(host, port);
} catch (UnknownHostException uhe) {
throw new IllegalArgumentException("Unable to resolve " + host);
}
// verify format
if (format != null) {
String uripath = uri.getPath();
checkFormat(format, uripath);
}
return new Pair<String, Integer>(host, port);
} catch (URISyntaxException use) {
throw new IllegalArgumentException("Invalid URL: " + url);
}
}
// verify if a URI path is compliance with the file format given
private static void checkFormat(String format, String uripath) {
if ((!uripath.toLowerCase().endsWith("vhd")) && (!uripath.toLowerCase().endsWith("vhd.zip")) && (!uripath.toLowerCase().endsWith("vhd.bz2")) &&
(!uripath.toLowerCase().endsWith("vhdx")) && (!uripath.toLowerCase().endsWith("vhdx.gz")) &&
(!uripath.toLowerCase().endsWith("vhdx.bz2")) && (!uripath.toLowerCase().endsWith("vhdx.zip")) &&
(!uripath.toLowerCase().endsWith("vhd.gz")) && (!uripath.toLowerCase().endsWith("qcow2")) && (!uripath.toLowerCase().endsWith("qcow2.zip")) &&
(!uripath.toLowerCase().endsWith("qcow2.bz2")) && (!uripath.toLowerCase().endsWith("qcow2.gz")) && (!uripath.toLowerCase().endsWith("ova")) &&
(!uripath.toLowerCase().endsWith("ova.zip")) && (!uripath.toLowerCase().endsWith("ova.bz2")) && (!uripath.toLowerCase().endsWith("ova.gz")) &&
(!uripath.toLowerCase().endsWith("tar")) && (!uripath.toLowerCase().endsWith("tar.zip")) && (!uripath.toLowerCase().endsWith("tar.bz2")) &&
(!uripath.toLowerCase().endsWith("tar.gz")) && (!uripath.toLowerCase().endsWith("vmdk")) && (!uripath.toLowerCase().endsWith("vmdk.gz")) &&
(!uripath.toLowerCase().endsWith("vmdk.zip")) && (!uripath.toLowerCase().endsWith("vmdk.bz2")) && (!uripath.toLowerCase().endsWith("img")) &&
(!uripath.toLowerCase().endsWith("img.gz")) && (!uripath.toLowerCase().endsWith("img.zip")) && (!uripath.toLowerCase().endsWith("img.bz2")) &&
(!uripath.toLowerCase().endsWith("raw")) && (!uripath.toLowerCase().endsWith("raw.gz")) && (!uripath.toLowerCase().endsWith("raw.bz2")) &&
(!uripath.toLowerCase().endsWith("raw.zip")) && (!uripath.toLowerCase().endsWith("iso")) && (!uripath.toLowerCase().endsWith("iso.zip"))
&& (!uripath.toLowerCase().endsWith("iso.bz2")) && (!uripath.toLowerCase().endsWith("iso.gz"))) {
throw new IllegalArgumentException("Please specify a valid " + format.toLowerCase());
}
if ((format.equalsIgnoreCase("vhd")
&& (!uripath.toLowerCase().endsWith("vhd")
&& !uripath.toLowerCase().endsWith("vhd.zip")
&& !uripath.toLowerCase().endsWith("vhd.bz2")
&& !uripath.toLowerCase().endsWith("vhd.gz")))
|| (format.equalsIgnoreCase("vhdx")
&& (!uripath.toLowerCase().endsWith("vhdx")
&& !uripath.toLowerCase().endsWith("vhdx.zip")
&& !uripath.toLowerCase().endsWith("vhdx.bz2")
&& !uripath.toLowerCase().endsWith("vhdx.gz")))
|| (format.equalsIgnoreCase("qcow2")
&& (!uripath.toLowerCase().endsWith("qcow2")
&& !uripath.toLowerCase().endsWith("qcow2.zip")
&& !uripath.toLowerCase().endsWith("qcow2.bz2")
&& !uripath.toLowerCase().endsWith("qcow2.gz")))
|| (format.equalsIgnoreCase("ova")
&& (!uripath.toLowerCase().endsWith("ova")
&& !uripath.toLowerCase().endsWith("ova.zip")
&& !uripath.toLowerCase().endsWith("ova.bz2")
&& !uripath.toLowerCase().endsWith("ova.gz")))
|| (format.equalsIgnoreCase("tar")
&& (!uripath.toLowerCase().endsWith("tar")
&& !uripath.toLowerCase().endsWith("tar.zip")
&& !uripath.toLowerCase().endsWith("tar.bz2")
&& !uripath.toLowerCase().endsWith("tar.gz")))
|| (format.equalsIgnoreCase("raw")
&& (!uripath.toLowerCase().endsWith("img")
&& !uripath.toLowerCase().endsWith("img.zip")
&& !uripath.toLowerCase().endsWith("img.bz2")
&& !uripath.toLowerCase().endsWith("img.gz")
&& !uripath.toLowerCase().endsWith("raw")
&& !uripath.toLowerCase().endsWith("raw.bz2")
&& !uripath.toLowerCase().endsWith("raw.zip")
&& !uripath.toLowerCase().endsWith("raw.gz")))
|| (format.equalsIgnoreCase("vmdk")
&& (!uripath.toLowerCase().endsWith("vmdk")
&& !uripath.toLowerCase().endsWith("vmdk.zip")
&& !uripath.toLowerCase().endsWith("vmdk.bz2")
&& !uripath.toLowerCase().endsWith("vmdk.gz")))
|| (format.equalsIgnoreCase("iso")
&& (!uripath.toLowerCase().endsWith("iso.zip")
&& !uripath.toLowerCase().endsWith("iso.bz2")
&& !uripath.toLowerCase().endsWith("iso.gz")))) {
throw new IllegalArgumentException("Please specify a valid URL. URL:" + uripath + " is an invalid for the format " + format.toLowerCase());
}
}
public static InputStream getInputStreamFromUrl(String url, String user, String password) {
try {

View File

@ -28,58 +28,58 @@ public class TemplateUtils {
// given a path, returns empty if path is supported image, and the file type if unsupported
// this is meant to catch things like accidental upload of ASCII text .vmdk descriptor
public static String checkTemplateFormat(String path, String url) {
public static String checkTemplateFormat(String path, String uripath) {
// note 'path' was generated by us so it should be safe on the cmdline, be wary of 'url'
String command = "file ";
if (isCompressedExtension(url)) {
if (isCompressedExtension(uripath)) {
command = "file -z ";
}
String output = Script.runSimpleBashScript(command + path + " | cut -d: -f2", 60000);
// vmdk
if ((output.contains("VMware") || output.contains("data")) && isCorrectExtension(url, "vmdk")) {
if ((output.contains("VMware") || output.contains("data")) && isCorrectExtension(uripath, "vmdk")) {
s_logger.debug("File at path " + path + " looks like a vmware image :" + output);
return "";
}
// raw
if ((output.contains("x86 boot") || output.contains("data")) && (isCorrectExtension(url, "raw") || isCorrectExtension(url, "img"))) {
if ((output.contains("x86 boot") || output.contains("data")) && (isCorrectExtension(uripath, "raw") || isCorrectExtension(uripath, "img"))) {
s_logger.debug("File at path " + path + " looks like a raw image :" + output);
return "";
}
// qcow2
if (output.contains("QEMU QCOW") && isCorrectExtension(url, "qcow2")) {
if (output.contains("QEMU QCOW") && isCorrectExtension(uripath, "qcow2")) {
s_logger.debug("File at path " + path + " looks like QCOW2 : " + output);
return "";
}
// vhd
if (output.contains("Microsoft Disk Image") && (isCorrectExtension(url, "vhd") || isCorrectExtension(url, "vhdx"))) {
if (output.contains("Microsoft Disk Image") && (isCorrectExtension(uripath, "vhd") || isCorrectExtension(uripath, "vhdx"))) {
s_logger.debug("File at path " + path + " looks like vhd : " + output);
return "";
}
// ova
if (output.contains("POSIX tar") && isCorrectExtension(url, "ova")) {
if (output.contains("POSIX tar") && isCorrectExtension(uripath, "ova")) {
s_logger.debug("File at path " + path + " looks like ova : " + output);
return "";
}
//lxc
if (output.contains("POSIX tar") && isCorrectExtension(url, "tar")) {
if (output.contains("POSIX tar") && isCorrectExtension(uripath, "tar")) {
s_logger.debug("File at path " + path + " looks like just tar : " + output);
return "";
}
if (output.contains("ISO 9660") && isCorrectExtension(url, "iso")) {
if (output.contains("ISO 9660") && isCorrectExtension(uripath, "iso")) {
s_logger.debug("File at path " + path + " looks like an iso : " + output);
return "";
}
return output;
}
public static boolean isCorrectExtension(String url, String ext) {
if (url.toLowerCase().endsWith(ext)
|| url.toLowerCase().endsWith(ext + ".gz")
|| url.toLowerCase().endsWith(ext + ".bz2")
|| url.toLowerCase().endsWith(ext + ".zip")) {
public static boolean isCorrectExtension(String path, String ext) {
if (path.toLowerCase().endsWith(ext)
|| path.toLowerCase().endsWith(ext + ".gz")
|| path.toLowerCase().endsWith(ext + ".bz2")
|| path.toLowerCase().endsWith(ext + ".zip")) {
return true;
}
return false;