apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-11-02 20:02:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '4.20' of https://github.com/apache/cloudstack into fix-assign-vm-1

Browse Source
This commit is contained in:
Pearl Dsilva 2025-05-13 13:49:40 +05:30
commit 8fbe8b410b
3 changed files with 122 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java
View File

@ -27,6 +27,7 @@ import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
@ -80,6 +81,18 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
private ResourceType resourceType = ResourceType.TEMPLATE;
private final HttpMethodRetryHandler myretryhandler;
private boolean followRedirects = false;
private boolean isChunkedTransfer;
protected static final List<String> CUSTOM_HEADERS_FOR_CHUNKED_TRANSFER_SIZE = Arrays.asList(
"x-goog-stored-content-length",
"x-goog-meta-size",
"x-amz-meta-size",
"x-amz-meta-content-length",
"x-object-meta-size",
"x-original-content-length",
"x-oss-meta-content-length",
"x-file-size");
private static final long MIN_FORMAT_VERIFICATION_SIZE = 1024 * 1024;
public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes,
String user, String password, Proxy proxy, ResourceType resourceType) {
@ -205,13 +218,11 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
RandomAccessFile out = new RandomAccessFile(file, "rw");
) {
out.seek(localFileSize);
logger.info("Starting download from " + downloadUrl + " to " + toFile + " remoteSize=" + toHumanReadableSize(remoteSize) + " , max size=" + toHumanReadableSize(maxTemplateSizeInBytes));
if (copyBytes(file, in, out)) return 0;
logger.info("Starting download from {} to {} remoteSize={} , max size={}",downloadUrl, toFile,
toHumanReadableSize(remoteSize), toHumanReadableSize(maxTemplateSizeInBytes));
boolean eof = copyBytes(file, in, out);
Date finish = new Date();
checkDowloadCompletion();
checkDownloadCompletion(eof);
downloadTime += finish.getTime() - start.getTime();
} finally { /* in.close() and out.close() */ }
return totalBytes;
@ -237,28 +248,32 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
}
private boolean copyBytes(File file, InputStream in, RandomAccessFile out) throws IOException {
int bytes;
byte[] block = new byte[CHUNK_SIZE];
byte[] buffer = new byte[CHUNK_SIZE];
long offset = 0;
boolean done = false;
VerifyFormat verifyFormat = new VerifyFormat(file);
status = Status.IN_PROGRESS;
while (!done && status != Status.ABORTED && offset <= remoteSize) {
if ((bytes = in.read(block, 0, CHUNK_SIZE)) > -1) {
offset = writeBlock(bytes, out, block, offset);
if (!ResourceType.SNAPSHOT.equals(resourceType) &&
!verifyFormat.isVerifiedFormat() &&
(offset >= 1048576 || offset >= remoteSize)) { //let's check format after we get 1MB or full file
verifyFormat.invoke();
}
} else {
done = true;
while (status != Status.ABORTED) {
int bytesRead = in.read(buffer, 0, CHUNK_SIZE);
if (bytesRead == -1) {
logger.debug("Reached EOF on input stream");
break;
}
offset = writeBlock(bytesRead, out, buffer, offset);
if (!ResourceType.SNAPSHOT.equals(resourceType)
&& !verifyFormat.isVerifiedFormat()
&& (offset >= MIN_FORMAT_VERIFICATION_SIZE || offset >= remoteSize)) {
verifyFormat.invoke();
}
if (offset >= remoteSize) {
logger.debug("Reached expected remote size limit: {} bytes", remoteSize);
break;
}
}
out.getFD().sync();
return false;
return !Status.ABORTED.equals(status);
}
private long writeBlock(int bytes, RandomAccessFile out, byte[] block, long offset) throws IOException {
out.write(block, 0, bytes);
offset += bytes;
@ -267,11 +282,13 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
return offset;
}
private void checkDowloadCompletion() {
private void checkDownloadCompletion(boolean eof) {
String downloaded = "(incomplete download)";
if (totalBytes >= remoteSize) {
if (eof && ((totalBytes >= remoteSize) || (isChunkedTransfer && remoteSize == maxTemplateSizeInBytes))) {
status = Status.DOWNLOAD_FINISHED;
downloaded = "(download complete remote=" + toHumanReadableSize(remoteSize) + " bytes)";
downloaded = "(download complete remote=" +
(remoteSize == maxTemplateSizeInBytes ? toHumanReadableSize(remoteSize) : "unknown") +
" bytes)";
}
errorString = "Downloaded " + toHumanReadableSize(totalBytes) + " bytes " + downloaded;
}
@ -293,18 +310,42 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
}
}
protected long getRemoteSizeForChunkedTransfer() {
for (String headerKey : CUSTOM_HEADERS_FOR_CHUNKED_TRANSFER_SIZE) {
Header header = request.getResponseHeader(headerKey);
if (header == null) {
continue;
}
try {
return Long.parseLong(header.getValue());
} catch (NumberFormatException ignored) {}
}
Header contentRangeHeader = request.getResponseHeader("Content-Range");
if (contentRangeHeader != null) {
String contentRange = contentRangeHeader.getValue();
if (contentRange != null && contentRange.contains("/")) {
String totalSize = contentRange.substring(contentRange.indexOf('/') + 1).trim();
return Long.parseLong(totalSize);
}
}
return 0;
}
private boolean tryAndGetRemoteSize() {
Header contentLengthHeader = request.getResponseHeader("content-length");
boolean chunked = false;
isChunkedTransfer = false;
long reportedRemoteSize = 0;
if (contentLengthHeader == null) {
Header chunkedHeader = request.getResponseHeader("Transfer-Encoding");
if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
if (chunkedHeader != null && "chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
isChunkedTransfer = true;
reportedRemoteSize = getRemoteSizeForChunkedTransfer();
logger.debug("{} is using chunked transfer encoding, possible remote size: {}", downloadUrl,
reportedRemoteSize);
} else {
status = Status.UNRECOVERABLE_ERROR;
errorString = " Failed to receive length of download ";
return false;
} else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
chunked = true;
}
} else {
reportedRemoteSize = Long.parseLong(contentLengthHeader.getValue());
@ -316,9 +357,11 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
return false;
}
}
if (remoteSize == 0) {
remoteSize = reportedRemoteSize;
if (remoteSize != 0) {
logger.debug("Remote size for {} found to be {}", downloadUrl, toHumanReadableSize(remoteSize));
}
}
return true;
}

43
engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java
View File

@ -126,7 +126,26 @@ public class ConfigDriveBuilder {
File openStackFolder = new File(tempDirName + ConfigDrive.openStackConfigDriveName);
writeVendorEmptyJsonFile(openStackFolder);
/*
Try to find VM password in the vmData.
If it is found, then write it into vendor-data.json
*/
String vmPassword = "";
for (String[] item : vmData) {
String dataType = item[CONFIGDATA_DIR];
String fileName = item[CONFIGDATA_FILE];
String content = item[CONFIGDATA_CONTENT];
if (PASSWORD_FILE.equals(fileName)) {
vmPassword = content;
break;
}
}
if (vmPassword.equals("")) {
writeVendorDataJsonFile(openStackFolder);
} else {
writeVendorDataJsonFile(openStackFolder, vmPassword);
}
writeNetworkData(nics, supportedServices, openStackFolder);
for (NicProfile nic: nics) {
if (supportedServices.get(nic.getId()).contains(Network.Service.UserData)) {
@ -253,7 +272,7 @@ public class ConfigDriveBuilder {
*
* If the folder does not exist, and we cannot create it, we throw a {@link CloudRuntimeException}.
*/
static void writeVendorEmptyJsonFile(File openStackFolder) {
static void writeVendorDataJsonFile(File openStackFolder) {
if (openStackFolder.exists() || openStackFolder.mkdirs()) {
writeFile(openStackFolder, "vendor_data.json", "{}");
} else {
@ -261,6 +280,26 @@ public class ConfigDriveBuilder {
}
}
/**
* Writes vendor data containing Cloudstack-generated password into vendor-data.json
*
* If the folder does not exist, and we cannot create it, we throw a {@link CloudRuntimeException}.
*/
static void writeVendorDataJsonFile(File openStackFolder, String password) {
if (openStackFolder.exists() || openStackFolder.mkdirs()) {
writeFile(
openStackFolder,
"vendor_data.json",
String.format(
"{\"cloud-init\": \"#cloud-config\\npassword: %s\\nchpasswd:\\n expire: False\"}",
password
)
);
} else {
throw new CloudRuntimeException("Failed to create folder " + openStackFolder);
}
}
/**
* Creates the {@link JsonObject} with VM's metadata. The vmData is a list of arrays; we expect this list to have the following entries:
* <ul>

20
engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java
View File

@ -134,7 +134,7 @@ public class ConfigDriveBuilderTest {
@Test(expected = CloudRuntimeException.class)
public void buildConfigDriveTestIoException() {
try (MockedStatic<ConfigDriveBuilder> configDriveBuilderMocked = Mockito.mockStatic(ConfigDriveBuilder.class)) {
configDriveBuilderMocked.when(() -> ConfigDriveBuilder.writeVendorEmptyJsonFile(nullable(File.class))).thenThrow(CloudRuntimeException.class);
configDriveBuilderMocked.when(() -> ConfigDriveBuilder.writeVendorDataJsonFile(nullable(File.class))).thenThrow(CloudRuntimeException.class);
Mockito.when(ConfigDriveBuilder.buildConfigDrive(null, new ArrayList<>(), "teste", "C:", null, supportedServices)).thenCallRealMethod();
ConfigDriveBuilder.buildConfigDrive(null, new ArrayList<>(), "teste", "C:", null, supportedServices);
}
@ -144,7 +144,7 @@ public class ConfigDriveBuilderTest {
public void buildConfigDriveTest() {
try (MockedStatic<ConfigDriveBuilder> configDriveBuilderMocked = Mockito.mockStatic(ConfigDriveBuilder.class)) {
configDriveBuilderMocked.when(() -> ConfigDriveBuilder.writeVendorEmptyJsonFile(Mockito.any(File.class))).then(invocationOnMock -> null);
configDriveBuilderMocked.when(() -> ConfigDriveBuilder.writeVendorDataJsonFile(Mockito.any(File.class))).then(invocationOnMock -> null);
configDriveBuilderMocked.when(() -> ConfigDriveBuilder.writeVmMetadata(Mockito.anyList(), Mockito.anyString(), Mockito.any(File.class), anyMap())).then(invocationOnMock -> null);
@ -163,7 +163,7 @@ public class ConfigDriveBuilderTest {
Assert.assertEquals("mockIsoDataBase64", returnedIsoData);
configDriveBuilderMocked.verify(() -> {
ConfigDriveBuilder.writeVendorEmptyJsonFile(Mockito.any(File.class));
ConfigDriveBuilder.writeVendorDataJsonFile(Mockito.any(File.class));
ConfigDriveBuilder.writeVmMetadata(Mockito.anyList(), Mockito.anyString(), Mockito.any(File.class), anyMap());
ConfigDriveBuilder.linkUserData(Mockito.anyString());
ConfigDriveBuilder.generateAndRetrieveIsoAsBase64Iso(Mockito.anyString(), Mockito.anyString(), Mockito.anyString());
@ -172,23 +172,23 @@ public class ConfigDriveBuilderTest {
}
@Test(expected = CloudRuntimeException.class)
public void writeVendorEmptyJsonFileTestCannotCreateOpenStackFolder() {
public void writeVendorDataJsonFileTestCannotCreateOpenStackFolder() {
File folderFileMock = Mockito.mock(File.class);
Mockito.doReturn(false).when(folderFileMock).mkdirs();
ConfigDriveBuilder.writeVendorEmptyJsonFile(folderFileMock);
ConfigDriveBuilder.writeVendorDataJsonFile(folderFileMock);
}
@Test(expected = CloudRuntimeException.class)
public void writeVendorEmptyJsonFileTest() {
public void writeVendorDataJsonFileTest() {
File folderFileMock = Mockito.mock(File.class);
Mockito.doReturn(false).when(folderFileMock).mkdirs();
ConfigDriveBuilder.writeVendorEmptyJsonFile(folderFileMock);
ConfigDriveBuilder.writeVendorDataJsonFile(folderFileMock);
}
@Test
public void writeVendorEmptyJsonFileTestCreatingFolder() {
public void writeVendorDataJsonFileTestCreatingFolder() {
try (MockedStatic<ConfigDriveBuilder> configDriveBuilderMocked = Mockito.mockStatic(ConfigDriveBuilder.class)) {
File folderFileMock = Mockito.mock(File.class);
@ -196,9 +196,9 @@ public class ConfigDriveBuilderTest {
Mockito.doReturn(true).when(folderFileMock).mkdirs();
//force execution of real method
configDriveBuilderMocked.when(() -> ConfigDriveBuilder.writeVendorEmptyJsonFile(folderFileMock)).thenCallRealMethod();
configDriveBuilderMocked.when(() -> ConfigDriveBuilder.writeVendorDataJsonFile(folderFileMock)).thenCallRealMethod();
ConfigDriveBuilder.writeVendorEmptyJsonFile(folderFileMock);
ConfigDriveBuilder.writeVendorDataJsonFile(folderFileMock);
Mockito.verify(folderFileMock).exists();
Mockito.verify(folderFileMock).mkdirs();