Fix userdata append header restrictions (#9575)

engine/userdata/cloud-init/src/main/java/org/apache/cloudstack/userdata/CloudInitUserDataProvider.java

@@ -88,14 +88,20 @@ public class CloudInitUserDataProvider extends AdapterBase implements UserDataPr
                 .filter(x -> (x.startsWith("#") && !x.startsWith("##")) || (x.startsWith("Content-Type:")))
                 .collect(Collectors.toList());
         if (CollectionUtils.isEmpty(lines)) {
-            throw new CloudRuntimeException("Failed to detect the user data format type as it " +
+            LOGGER.debug("Failed to detect the user data format type as it does not contain a header");
-                    "does not contain a header");
+            return null;
         }
         return lines.get(0);
     }
 
-    protected FormatType mapUserDataHeaderToFormatType(String header) {
+    protected FormatType mapUserDataHeaderToFormatType(String header, FormatType defaultFormatType) {
-        if (header.equalsIgnoreCase("#cloud-config")) {
+        if (StringUtils.isBlank(header)) {
+            if (defaultFormatType == null) {
+                throw new CloudRuntimeException("Failed to detect the user data format type as it does not contain a header");
+            }
+            LOGGER.debug(String.format("Empty header for userdata, using the default format type: %s", defaultFormatType.name()));
+            return defaultFormatType;
+        } else if (header.equalsIgnoreCase("#cloud-config")) {
             return FormatType.CLOUD_CONFIG;
         } else if (header.startsWith("#!")) {
             return FormatType.BASH_SCRIPT;
@@ -115,9 +121,11 @@ public class CloudInitUserDataProvider extends AdapterBase implements UserDataPr
 
     /**
      * Detect the user data type
+     * @param userdata the userdata string to detect the type
+     * @param defaultFormatType if not null, then use it in case the header does not exist in the userdata, otherwise fail
      * Reference: <a href="https://canonical-cloud-init.readthedocs-hosted.com/en/latest/explanation/format.html#user-data-formats" />
      */
-    protected FormatType getUserDataFormatType(String userdata) {
+    protected FormatType getUserDataFormatType(String userdata, FormatType defaultFormatType) {
         if (StringUtils.isBlank(userdata)) {
             String msg = "User data expected but provided empty user data";
             LOGGER.error(msg);
@@ -125,7 +133,7 @@ public class CloudInitUserDataProvider extends AdapterBase implements UserDataPr
         }
 
         String header = extractUserDataHeader(userdata);
-        return mapUserDataHeaderToFormatType(header);
+        return mapUserDataHeaderToFormatType(header, defaultFormatType);
     }
 
     private String getContentType(String userData, FormatType formatType) throws MessagingException {
@@ -234,7 +242,9 @@ public class CloudInitUserDataProvider extends AdapterBase implements UserDataPr
     }
 
     private String simpleAppendSameFormatTypeUserData(String userData1, String userData2) {
-        return String.format("%s\n\n%s", userData1, userData2.substring(userData2.indexOf('\n')+1));
+        String userdata2Header = extractUserDataHeader(userData2);
+        int beginIndex = StringUtils.isNotBlank(userdata2Header) ? userData2.indexOf('\n')+1 : 0;
+        return String.format("%s\n\n%s", userData1, userData2.substring(beginIndex));
     }
 
     private void checkGzipAppend(String encodedUserData1, String encodedUserData2) {
@@ -249,8 +259,8 @@ public class CloudInitUserDataProvider extends AdapterBase implements UserDataPr
             checkGzipAppend(encodedUserData1, encodedUserData2);
             String userData1 = new String(Base64.decodeBase64(encodedUserData1));
             String userData2 = new String(Base64.decodeBase64(encodedUserData2));
-            FormatType formatType1 = getUserDataFormatType(userData1);
+            FormatType formatType1 = getUserDataFormatType(userData1, null);
-            FormatType formatType2 = getUserDataFormatType(userData2);
+            FormatType formatType2 = getUserDataFormatType(userData2, formatType1);
             if (formatType1.equals(formatType2) && List.of(FormatType.CLOUD_CONFIG, FormatType.BASH_SCRIPT).contains(formatType1)) {
                 return simpleAppendSameFormatTypeUserData(userData1, userData2);
             }

engine/userdata/cloud-init/src/test/java/org/apache/cloudstack/userdata/CloudInitUserDataProviderTest.java

@@ -73,21 +73,28 @@ public class CloudInitUserDataProviderTest {
 
     @Test
     public void testGetUserDataFormatType() {
-        CloudInitUserDataProvider.FormatType type = provider.getUserDataFormatType(CLOUD_CONFIG_USERDATA);
+        CloudInitUserDataProvider.FormatType type = provider.getUserDataFormatType(CLOUD_CONFIG_USERDATA, null);
         Assert.assertEquals(CloudInitUserDataProvider.FormatType.CLOUD_CONFIG, type);
     }
 
     @Test(expected = CloudRuntimeException.class)
     public void testGetUserDataFormatTypeNoHeader() {
         String userdata = "password: password\nchpasswd: { expire: False }\nssh_pwauth: True";
-        provider.getUserDataFormatType(userdata);
+        provider.getUserDataFormatType(userdata, null);
+    }
+
+    @Test
+    public void testGetUserDataFormatTypeNoHeaderDefaultFormat() {
+        String userdata = "password: password\nchpasswd: { expire: False }\nssh_pwauth: True";
+        CloudInitUserDataProvider.FormatType defaultFormatType = CloudInitUserDataProvider.FormatType.CLOUD_CONFIG;
+        Assert.assertEquals(defaultFormatType, provider.getUserDataFormatType(userdata, defaultFormatType));
     }
 
     @Test(expected = CloudRuntimeException.class)
     public void testGetUserDataFormatTypeInvalidType() {
         String userdata = "#invalid-type\n" +
                 "password: password\nchpasswd: { expire: False }\nssh_pwauth: True";
-        provider.getUserDataFormatType(userdata);
+        provider.getUserDataFormatType(userdata, null);
     }
 
     private MimeMultipart getCheckedMultipartFromMultipartData(String multipartUserData, int count) {
@@ -111,6 +118,16 @@ public class CloudInitUserDataProviderTest {
         getCheckedMultipartFromMultipartData(multipartUserData, 2);
     }
 
+    @Test
+    public void testAppendUserDataSecondWithoutHeader() {
+        String userdataWithHeader = Base64.encodeBase64String(SHELL_SCRIPT_USERDATA1.getBytes());
+        String bashScriptWithoutHeader = "echo \"without header\"";
+        String userdataWithoutHeader = Base64.encodeBase64String(bashScriptWithoutHeader.getBytes());
+        String appended = provider.appendUserData(userdataWithHeader, userdataWithoutHeader);
+        String expected = String.format("%s\n\n%s", SHELL_SCRIPT_USERDATA1, bashScriptWithoutHeader);
+        Assert.assertEquals(expected, appended);
+    }
+
     @Test
     public void testAppendSameShellScriptTypeUserData() {
         String result = SHELL_SCRIPT_USERDATA + "\n\n" +
@@ -129,6 +146,22 @@ public class CloudInitUserDataProviderTest {
         Assert.assertEquals(result, appendUserData);
     }
 
+    @Test
+    public void testAppendCloudConfig() {
+        String userdata1 = "#cloud-config\n" +
+                "chpasswd:\n" +
+                "  list: |\n" +
+                "    root:password\n" +
+                "  expire: False";
+        String userdata2 = "write_files:\n" +
+                "- path: /root/CLOUD_INIT_WAS_HERE";
+        String userdataWithHeader = Base64.encodeBase64String(userdata1.getBytes());
+        String userdataWithoutHeader = Base64.encodeBase64String(userdata2.getBytes());
+        String appended = provider.appendUserData(userdataWithHeader, userdataWithoutHeader);
+        String expected = String.format("%s\n\n%s", userdata1, userdata2);
+        Assert.assertEquals(expected, appended);
+    }
+
     @Test
     public void testAppendUserDataMIMETemplateData() {
         String multipartUserData = provider.appendUserData(

ui/src/utils/plugins.js

@@ -506,7 +506,7 @@ export const genericUtilPlugin = {
       if (isBase64(text)) {
         return text
       }
-      return encodeURIComponent(btoa(unescape(encodeURIComponent(text))))
+      return encodeURI(btoa(unescape(encodeURIComponent(text))))
     }
   }
 }