apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge release branch 4.18 to main

Browse Source 
* 4.18:
  test,refactor: fix test_project_resources cleanup (#8097)
  kvm: fix direct download template size (#8093)
This commit is contained in:
Daan Hoogland 2023-10-16 16:30:00 +02:00
commit c9d230369a
6 changed files with 81 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
View File

@ -19,29 +19,6 @@
package org.apache.cloudstack.direct.download;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import com.cloud.utils.storage.QCOW2Utils;
import org.apache.cloudstack.utils.security.SSLUtils;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.io.IOUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.commons.collections.MapUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
@ -60,6 +37,32 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import org.apache.cloudstack.utils.security.SSLUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.io.IOUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import com.cloud.utils.storage.QCOW2Utils;
public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected CloseableHttpClient httpsClient;
@ -183,8 +186,7 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
SSLContext context = getSSLContext();
urlConnection.setSSLSocketFactory(context.getSocketFactory());
urlConnection.connect();
boolean isCompressed = !url.endsWith("qcow2");
return QCOW2Utils.getVirtualSize(urlObj.openStream(), isCompressed);
return QCOW2Utils.getVirtualSize(urlObj.openStream(), UriUtils.isUrlForCompressedFile(url));
} catch (IOException e) {
throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e);
}

80
test/integration/component/test_project_resources.py
View File

@ -551,66 +551,59 @@ class TestTemplates(cloudstackTestCase):
cls.hypervisor = cls.testClient.getHypervisorInfo()
if cls.hypervisor.lower() in ['lxc']:
raise unittest.SkipTest("create template from volume is not supported on %s" % cls.hypervisor.lower())
cls._cleanup = []
cls.template = get_template(
cls.api_client,
cls.zone.id,
cls.services["ostype"]
)
cls.api_client,
cls.zone.id,
cls.services["ostype"]
)
cls.services["server"]["zoneid"] = cls.zone.id
# Create Domains, Account etc
cls.domain = Domain.create(
cls.api_client,
cls.services["domain"]
)
cls.api_client,
cls.services["domain"]
)
cls._cleanup.append(cls.domain)
cls.account = Account.create(
cls.api_client,
cls.services["account"],
domainid=cls.domain.id
)
cls.api_client,
cls.services["account"],
domainid=cls.domain.id
)
cls._cleanup.append(cls.account)
cls.user = Account.create(
cls.api_client,
cls.services["account"],
domainid=cls.domain.id
)
cls.api_client,
cls.services["account"],
domainid=cls.domain.id
)
cls._cleanup.append(cls.user)
# Create project as a domain admin
cls.project = Project.create(
cls.api_client,
cls.services["project"],
account=cls.account.name,
domainid=cls.account.domainid
)
cls.api_client,
cls.services["project"],
account=cls.account.name,
domainid=cls.account.domainid
)
cls._cleanup.append(cls.project)
cls.services["account"] = cls.account.name
# Create Service offering and disk offerings etc
cls.service_offering = ServiceOffering.create(
cls.api_client,
cls.services["service_offering"]
)
cls.api_client,
cls.services["service_offering"]
)
cls._cleanup.append(cls.service_offering)
cls.userapiclient = cls.testClient.getUserApiClient(
UserName=cls.account.name,
DomainName=cls.domain.name
)
cls._cleanup = [
cls.project,
cls.service_offering,
cls.account,
cls.user,
cls.domain
]
UserName=cls.account.name,
DomainName=cls.domain.name
)
return
@classmethod
def tearDownClass(cls):
try:
#Cleanup resources used
cleanup_resources(cls.api_client, cls._cleanup)
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
return
super(TestTemplates, cls).tearDownClass()
def setUp(self):
self.apiclient = self.testClient.getApiClient()
@ -619,12 +612,7 @@ class TestTemplates(cloudstackTestCase):
return
def tearDown(self):
try:
#Clean up, terminate the created instance, volumes and snapshots
cleanup_resources(self.apiclient, self.cleanup)
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
return
super(TestTemplates, self).tearDown()
@attr(tags=["advanced", "basic", "sg", "eip", "advancedns"], required_hardware="false")
def test_04_public_private_template_use_in_project(self):

8
utils/src/main/java/com/cloud/utils/UriUtils.java
View File

@ -408,14 +408,14 @@ public class UriUtils {
return urls;
}
public static final Set<String> COMMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz");
public static final Set<String> COMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz");
public static final Set<String> buildExtensionSet(boolean metalink, String... baseExtensions) {
final ImmutableSet.Builder<String> builder = ImmutableSet.builder();
for (String baseExtension : baseExtensions) {
builder.add("." + baseExtension);
for (String format : COMMPRESSION_FORMATS) {
for (String format : COMPRESSION_FORMATS) {
builder.add("." + baseExtension + "." + format);
}
}
@ -647,4 +647,8 @@ public class UriUtils {
throw new CloudRuntimeException(url + " is not a valid uri for RBD");
}
}
public static boolean isUrlForCompressedFile(String url) {
return UriUtils.COMPRESSION_FORMATS.stream().anyMatch(f -> url.toLowerCase().endsWith(f));
}
}

3
utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
View File

@ -32,6 +32,7 @@ import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.log4j.Logger;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.UriUtils;
public final class QCOW2Utils {
public static final Logger LOGGER = Logger.getLogger(QCOW2Utils.class.getName());
@ -114,7 +115,7 @@ public final class QCOW2Utils {
public static long getVirtualSize(String urlStr) {
try {
URL url = new URL(urlStr);
return getVirtualSizeFromInputStream(url.openStream());
return getVirtualSize(url.openStream(), UriUtils.isUrlForCompressedFile(urlStr));
} catch (MalformedURLException e) {
LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
throw new IllegalArgumentException("Invalid URL: " + urlStr);

2
utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java
View File

@ -101,7 +101,7 @@ public class ImageStoreUtil {
public static boolean isCompressedExtension(String path) {
final String lowerCasePath = path.toLowerCase();
return UriUtils.COMMPRESSION_FORMATS
return UriUtils.COMPRESSION_FORMATS
.stream()
.map(extension -> "." + extension)
.anyMatch(lowerCasePath::endsWith);

14
utils/src/test/java/com/cloud/utils/UriUtilsTest.java
View File

@ -19,13 +19,13 @@
package com.cloud.utils;
import org.junit.Assert;
import org.junit.Test;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
public class UriUtilsTest {
@Test
public void encodeURIComponent() {
@ -265,4 +265,12 @@ public class UriUtilsTest {
testGetUriInfoInternal(url11, host);
testGetUriInfoInternal(url12, host);
}
@Test
public void testIsUrlForCompressedFile() {
Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.bz2"));
Assert.assertTrue(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.zip"));
Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.gz"));
Assert.assertFalse(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.qcow2"));
}
}