apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge release branch 4.18 to main

Browse Source 
* 4.18:
  test,refactor: fix test_project_resources cleanup (#8097)
  kvm: fix direct download template size (#8093)
This commit is contained in:
Daan Hoogland 2023-10-16 16:30:00 +02:00
commit c9d230369a
6 changed files with 81 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
View File

@ -19,29 +19,6 @@
package org.apache.cloudstack.direct.download; package org.apache.cloudstack.direct.download;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import com.cloud.utils.storage.QCOW2Utils;
import org.apache.cloudstack.utils.security.SSLUtils;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.io.IOUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.commons.collections.MapUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
@ -60,6 +37,32 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import org.apache.cloudstack.utils.security.SSLUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.io.IOUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import com.cloud.utils.storage.QCOW2Utils;
public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected CloseableHttpClient httpsClient; protected CloseableHttpClient httpsClient;
@ -183,8 +186,7 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
SSLContext context = getSSLContext(); SSLContext context = getSSLContext();
urlConnection.setSSLSocketFactory(context.getSocketFactory()); urlConnection.setSSLSocketFactory(context.getSocketFactory());
urlConnection.connect(); urlConnection.connect();
boolean isCompressed = !url.endsWith("qcow2"); return QCOW2Utils.getVirtualSize(urlObj.openStream(), UriUtils.isUrlForCompressedFile(url));
return QCOW2Utils.getVirtualSize(urlObj.openStream(), isCompressed);
} catch (IOException e) { } catch (IOException e) {
throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e); throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e);
} }

80
test/integration/component/test_project_resources.py
View File

@ -551,66 +551,59 @@ class TestTemplates(cloudstackTestCase):
cls.hypervisor = cls.testClient.getHypervisorInfo() cls.hypervisor = cls.testClient.getHypervisorInfo()
if cls.hypervisor.lower() in ['lxc']: if cls.hypervisor.lower() in ['lxc']:
raise unittest.SkipTest("create template from volume is not supported on %s" % cls.hypervisor.lower()) raise unittest.SkipTest("create template from volume is not supported on %s" % cls.hypervisor.lower())
cls._cleanup = []
cls.template = get_template( cls.template = get_template(
cls.api_client, cls.api_client,
cls.zone.id, cls.zone.id,
cls.services["ostype"] cls.services["ostype"]
) )
cls.services["server"]["zoneid"] = cls.zone.id cls.services["server"]["zoneid"] = cls.zone.id
# Create Domains, Account etc # Create Domains, Account etc
cls.domain = Domain.create( cls.domain = Domain.create(
cls.api_client, cls.api_client,
cls.services["domain"] cls.services["domain"]
) )
cls._cleanup.append(cls.domain)
cls.account = Account.create( cls.account = Account.create(
cls.api_client, cls.api_client,
cls.services["account"], cls.services["account"],
domainid=cls.domain.id domainid=cls.domain.id
) )
cls._cleanup.append(cls.account)
cls.user = Account.create( cls.user = Account.create(
cls.api_client, cls.api_client,
cls.services["account"], cls.services["account"],
domainid=cls.domain.id domainid=cls.domain.id
) )
cls._cleanup.append(cls.user)
# Create project as a domain admin # Create project as a domain admin
cls.project = Project.create( cls.project = Project.create(
cls.api_client, cls.api_client,
cls.services["project"], cls.services["project"],
account=cls.account.name, account=cls.account.name,
domainid=cls.account.domainid domainid=cls.account.domainid
) )
cls._cleanup.append(cls.project)
cls.services["account"] = cls.account.name cls.services["account"] = cls.account.name
# Create Service offering and disk offerings etc # Create Service offering and disk offerings etc
cls.service_offering = ServiceOffering.create( cls.service_offering = ServiceOffering.create(
cls.api_client, cls.api_client,
cls.services["service_offering"] cls.services["service_offering"]
) )
cls._cleanup.append(cls.service_offering)
cls.userapiclient = cls.testClient.getUserApiClient( cls.userapiclient = cls.testClient.getUserApiClient(
UserName=cls.account.name, UserName=cls.account.name,
DomainName=cls.domain.name DomainName=cls.domain.name
) )
cls._cleanup = [
cls.project,
cls.service_offering,
cls.account,
cls.user,
cls.domain
]
return return
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
try: super(TestTemplates, cls).tearDownClass()
#Cleanup resources used
cleanup_resources(cls.api_client, cls._cleanup)
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
return
def setUp(self): def setUp(self):
self.apiclient = self.testClient.getApiClient() self.apiclient = self.testClient.getApiClient()
@ -619,12 +612,7 @@ class TestTemplates(cloudstackTestCase):
return return
def tearDown(self): def tearDown(self):
try: super(TestTemplates, self).tearDown()
#Clean up, terminate the created instance, volumes and snapshots
cleanup_resources(self.apiclient, self.cleanup)
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
return
@attr(tags=["advanced", "basic", "sg", "eip", "advancedns"], required_hardware="false") @attr(tags=["advanced", "basic", "sg", "eip", "advancedns"], required_hardware="false")
def test_04_public_private_template_use_in_project(self): def test_04_public_private_template_use_in_project(self):

8
utils/src/main/java/com/cloud/utils/UriUtils.java
View File

@ -408,14 +408,14 @@ public class UriUtils {
return urls; return urls;
} }
public static final Set<String> COMMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz"); public static final Set<String> COMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz");
public static final Set<String> buildExtensionSet(boolean metalink, String... baseExtensions) { public static final Set<String> buildExtensionSet(boolean metalink, String... baseExtensions) {
final ImmutableSet.Builder<String> builder = ImmutableSet.builder(); final ImmutableSet.Builder<String> builder = ImmutableSet.builder();
for (String baseExtension : baseExtensions) { for (String baseExtension : baseExtensions) {
builder.add("." + baseExtension); builder.add("." + baseExtension);
for (String format : COMMPRESSION_FORMATS) { for (String format : COMPRESSION_FORMATS) {
builder.add("." + baseExtension + "." + format); builder.add("." + baseExtension + "." + format);
} }
} }
@ -647,4 +647,8 @@ public class UriUtils {
throw new CloudRuntimeException(url + " is not a valid uri for RBD"); throw new CloudRuntimeException(url + " is not a valid uri for RBD");
} }
} }
public static boolean isUrlForCompressedFile(String url) {
return UriUtils.COMPRESSION_FORMATS.stream().anyMatch(f -> url.toLowerCase().endsWith(f));
}
} }

3
utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
View File

@ -32,6 +32,7 @@ import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import com.cloud.utils.NumbersUtil; import com.cloud.utils.NumbersUtil;
import com.cloud.utils.UriUtils;
public final class QCOW2Utils { public final class QCOW2Utils {
public static final Logger LOGGER = Logger.getLogger(QCOW2Utils.class.getName()); public static final Logger LOGGER = Logger.getLogger(QCOW2Utils.class.getName());
@ -114,7 +115,7 @@ public final class QCOW2Utils {
public static long getVirtualSize(String urlStr) { public static long getVirtualSize(String urlStr) {
try { try {
URL url = new URL(urlStr); URL url = new URL(urlStr);
return getVirtualSizeFromInputStream(url.openStream()); return getVirtualSize(url.openStream(), UriUtils.isUrlForCompressedFile(urlStr));
} catch (MalformedURLException e) { } catch (MalformedURLException e) {
LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage()); LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
throw new IllegalArgumentException("Invalid URL: " + urlStr); throw new IllegalArgumentException("Invalid URL: " + urlStr);

2
utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java
View File

@ -101,7 +101,7 @@ public class ImageStoreUtil {
public static boolean isCompressedExtension(String path) { public static boolean isCompressedExtension(String path) {
final String lowerCasePath = path.toLowerCase(); final String lowerCasePath = path.toLowerCase();
return UriUtils.COMMPRESSION_FORMATS return UriUtils.COMPRESSION_FORMATS
.stream() .stream()
.map(extension -> "." + extension) .map(extension -> "." + extension)
.anyMatch(lowerCasePath::endsWith); .anyMatch(lowerCasePath::endsWith);

14
utils/src/test/java/com/cloud/utils/UriUtilsTest.java
View File

@ -19,13 +19,13 @@
package com.cloud.utils; package com.cloud.utils;
import org.junit.Assert;
import org.junit.Test;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import org.junit.Assert;
import org.junit.Test;
public class UriUtilsTest { public class UriUtilsTest {
@Test @Test
public void encodeURIComponent() { public void encodeURIComponent() {
@ -265,4 +265,12 @@ public class UriUtilsTest {
testGetUriInfoInternal(url11, host); testGetUriInfoInternal(url11, host);
testGetUriInfoInternal(url12, host); testGetUriInfoInternal(url12, host);
} }
@Test
public void testIsUrlForCompressedFile() {
Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.bz2"));
Assert.assertTrue(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.zip"));
Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.gz"));
Assert.assertFalse(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.qcow2"));
}
} }