CLOUDSTACK-10290: allow config drives on primary storage for KVM (#2651)

This introduces a new global setting `vm.configdrive.primarypool.enabled` to toggle creation/hosting of config drive iso files on primary storage, the default will be false causing them to be hosted on secondary storage. The current support is limited from hypervisor resource side and in current implementation limited to `KVM` only. The next big change is that config drive is created at a temporary location by management server and shipped to either KVM or SSVM agent via cmd-answer pattern, the data of which is not logged in logs. This saves us from adding genisoimage dependency on cloudstack-agent pkg.

The APIs to reset ssh public key, password and user-data (via update VM API) requires that VM should be shutdown. Therefore, in the refactoring I removed the case of updation of existing ISO. If there are objections I'll re-put the strategy to detach+attach new config iso as a way of updation. In the refactored implementation, the folder name is changed to lower-cased configdrive. And during VM start, migration or shutdown/removal if primary storage is enable for use, the KVM agent will handle cleanup tasks otherwise SSVM agent will handle them.

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

api/src/com/cloud/network/element/UserDataServiceProvider.java

@@ -26,7 +26,7 @@ import com.cloud.vm.ReservationContext;
 import com.cloud.vm.VirtualMachineProfile;
 
 public interface UserDataServiceProvider extends NetworkElement {
-    public boolean addPasswordAndUserdata(Network network, NicProfile nic, VirtualMachineProfile vm, DeployDestination dest, ReservationContext context)
+    boolean addPasswordAndUserdata(Network network, NicProfile nic, VirtualMachineProfile vm, DeployDestination dest, ReservationContext context)
         throws ConcurrentOperationException, InsufficientCapacityException, ResourceUnavailableException;
 
     boolean savePassword(Network network, NicProfile nic, VirtualMachineProfile vm) throws ResourceUnavailableException;

client/pom.xml

@@ -378,6 +378,11 @@
       <artifactId>cloud-engine-storage-cache</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-engine-storage-configdrive</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.cloudstack</groupId>
       <artifactId>cloud-controller-secondary-storage</artifactId>

core/src/com/cloud/agent/api/HandleConfigDriveIsoCommand.java

@@ -19,28 +19,22 @@
 
 package com.cloud.agent.api;
 
-import java.util.List;
-
 import com.cloud.agent.api.to.DataStoreTO;
 
 public class HandleConfigDriveIsoCommand extends Command {
 
-    String isoFile;
+    @LogLevel(LogLevel.Log4jLevel.Off)
-    List<String[]> vmData;
+    private String isoData;
-    String configDriveLabel;
+
-    boolean create = false;
+    private String isoFile;
-    private boolean update = false;
+    private boolean create = false;
     private DataStoreTO destStore;
 
-    public HandleConfigDriveIsoCommand(List<String[]> vmData, String label, DataStoreTO destStore, String isoFile, boolean create, boolean update) {
+    public HandleConfigDriveIsoCommand(String isoFile, String isoData, DataStoreTO destStore, boolean create) {
-        this.vmData = vmData;
-        this.configDriveLabel = label;
-        this.create = create;
-        this.update = update;
-        this.destStore = destStore;
-
-
         this.isoFile = isoFile;
+        this.isoData = isoData;
+        this.destStore = destStore;
+        this.create = create;
     }
 
     @Override
@@ -48,22 +42,14 @@ public class HandleConfigDriveIsoCommand extends Command {
         return false;
     }
 
-    public List<String[]> getVmData() {
+    public String getIsoData() {
-        return vmData;
+        return isoData;
-    }
-
-    public void setVmData(List<String[]> vmData) {
-        this.vmData = vmData;
     }
 
     public boolean isCreate() {
         return create;
     }
 
-    public String getConfigDriveLabel() {
-        return configDriveLabel;
-    }
-
     public DataStoreTO getDestStore() {
         return destStore;
     }
@@ -71,8 +57,4 @@ public class HandleConfigDriveIsoCommand extends Command {
     public String getIsoFile() {
         return isoFile;
     }
-
-    public boolean isUpdate() {
-        return update;
-    }
 }

engine/api/src/com/cloud/vm/VirtualMachineManager.java

@@ -49,14 +49,17 @@ import com.cloud.utils.fsm.NoTransitionException;
  */
 public interface VirtualMachineManager extends Manager {
 
-    static final ConfigKey<Boolean> ExecuteInSequence = new ConfigKey<Boolean>("Advanced", Boolean.class, "execute.in.sequence.hypervisor.commands", "false",
+    ConfigKey<Boolean> ExecuteInSequence = new ConfigKey<>("Advanced", Boolean.class, "execute.in.sequence.hypervisor.commands", "false",
             "If set to true, start, stop, reboot, copy and migrate commands will be serialized on the agent side. If set to false the commands are executed in parallel. Default value is false.", false);
 
-    static final ConfigKey<String> VmConfigDriveLabel = new ConfigKey<String>("Hidden", String.class, "vm.configdrive.label", "config-2",
+    ConfigKey<String> VmConfigDriveLabel = new ConfigKey<>("Hidden", String.class, "vm.configdrive.label", "config-2",
             "The default label name for the config drive", false);
 
-    public interface Topics {
+    ConfigKey<Boolean> VmConfigDriveOnPrimaryPool = new ConfigKey<>("Advanced", Boolean.class, "vm.configdrive.primarypool.enabled", "false",
-        public static final String VM_POWER_STATE = "vm.powerstate";
+            "If config drive need to be created and hosted on primary storage pool. Currently only supported for KVM.", true);
+
+    interface Topics {
+        String VM_POWER_STATE = "vm.powerstate";
     }
 
     /**

engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -1108,10 +1108,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                 }
 
                 try {
-                    _networkMgr.prepare(vmProfile, new DeployDestination(dest.getDataCenter(), dest.getPod(), null, null), ctx);
+                    _networkMgr.prepare(vmProfile, new DeployDestination(dest.getDataCenter(), dest.getPod(), null, null, dest.getStorageForDisks()), ctx);
                     if (vm.getHypervisorType() != HypervisorType.BareMetal) {
                         volumeMgr.prepare(vmProfile, dest);
                     }
+
                     //since StorageMgr succeeded in volume creation, reuse Volume for further tries until current cluster has capacity
                     if (!reuseVolume) {
                         reuseVolume = true;
@@ -4018,7 +4019,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {ClusterDeltaSyncInterval, StartRetry, VmDestroyForcestop, VmOpCancelInterval, VmOpCleanupInterval, VmOpCleanupWait,
                 VmOpLockStateRetry,
-                VmOpWaitInterval, ExecuteInSequence, VmJobCheckInterval, VmJobTimeout, VmJobStateReportInterval, VmConfigDriveLabel, HaVmRestartHostUp};
+                VmOpWaitInterval, ExecuteInSequence, VmJobCheckInterval, VmJobTimeout, VmJobStateReportInterval, VmConfigDriveLabel, VmConfigDriveOnPrimaryPool, HaVmRestartHostUp};
     }
 
     public List<StoragePoolAllocator> getStoragePoolAllocators() {

engine/pom.xml

@@ -53,6 +53,7 @@
     <module>storage/datamotion</module>
     <module>storage/cache</module>
     <module>storage/snapshot</module>
+    <module>storage/configdrive</module>
     <module>components-api</module>
     <module>network</module>
     <module>service</module>

engine/storage/configdrive/pom.xml

@@ -0,0 +1,43 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements. See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership. The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>cloud-engine-storage-configdrive</artifactId>
+  <name>Apache CloudStack Framework - Storage Config Drive Component</name>
+  <parent>
+    <groupId>org.apache.cloudstack</groupId>
+    <artifactId>cloud-engine</artifactId>
+    <version>4.11.1.0-SNAPSHOT</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-api</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-core</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+  </dependencies>
+
+</project>

engine/storage/configdrive/src/org/apache/cloudstack/storage/configdrive/ConfigDrive.java

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.storage.configdrive;
+
+public class ConfigDrive {
+
+    public final static String CONFIGDRIVEFILENAME = "configdrive.iso";
+    public final static String CONFIGDRIVEDIR = "configdrive";
+
+    public static final String cloudStackConfigDriveName = "/cloudstack/";
+    public static final String openStackConfigDriveName = "/openstack/latest/";
+
+    /**
+     * This is the path to iso file relative to mount point
+     * @return config drive iso file path
+     */
+    public static String createConfigDrivePath(final String instanceName) {
+        return ConfigDrive.CONFIGDRIVEDIR + "/" + instanceName + "/"  + ConfigDrive.CONFIGDRIVEFILENAME;
+    }
+
+}

engine/storage/configdrive/src/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java

@@ -0,0 +1,222 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.storage.configdrive;
+
+import static com.cloud.network.NetworkModel.CONFIGDATA_CONTENT;
+import static com.cloud.network.NetworkModel.CONFIGDATA_DIR;
+import static com.cloud.network.NetworkModel.CONFIGDATA_FILE;
+import static com.cloud.network.NetworkModel.PASSWORD_FILE;
+import static com.cloud.network.NetworkModel.USERDATA_FILE;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.joda.time.Duration;
+
+import com.cloud.network.NetworkModel;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+import com.google.common.base.Strings;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+public class ConfigDriveBuilder {
+
+    public static final Logger LOG = Logger.getLogger(ConfigDriveBuilder.class);
+
+    private static void writeFile(final File folder, final String file, final String content) {
+        if (folder == null || Strings.isNullOrEmpty(file)) {
+            return;
+        }
+        final File vendorDataFile = new File(folder, file);
+        try (final FileWriter fw = new FileWriter(vendorDataFile); final BufferedWriter bw = new BufferedWriter(fw)) {
+            bw.write(content);
+        } catch (IOException ex) {
+            throw new CloudRuntimeException("Failed to create config drive file " + file, ex);
+        }
+    }
+
+    public static String fileToBase64String(final File isoFile) throws IOException {
+        byte[] encoded = Base64.encodeBase64(FileUtils.readFileToByteArray(isoFile));
+        return new String(encoded, StandardCharsets.US_ASCII);
+    }
+
+    public static File base64StringToFile(final String encodedIsoData, final String folder, final String fileName) throws IOException {
+        byte[] decoded = Base64.decodeBase64(encodedIsoData.getBytes(StandardCharsets.US_ASCII));
+        Path destPath = Paths.get(folder, fileName);
+        return Files.write(destPath, decoded).toFile();
+    }
+
+    public static String buildConfigDrive(final List<String[]> vmData, final String isoFileName, final String driveLabel) {
+        if (vmData == null) {
+            throw new CloudRuntimeException("No VM metadata provided");
+        }
+
+        Path tempDir = null;
+        String tempDirName = null;
+        try {
+            tempDir = Files.createTempDirectory(ConfigDrive.CONFIGDRIVEDIR);
+            tempDirName = tempDir.toString();
+
+            File openStackFolder = new File(tempDirName + ConfigDrive.openStackConfigDriveName);
+            if (openStackFolder.exists() || openStackFolder.mkdirs()) {
+                writeFile(openStackFolder, "vendor_data.json", "{}");
+                writeFile(openStackFolder, "network_data.json", "{}");
+            } else {
+                throw new CloudRuntimeException("Failed to create folder " + openStackFolder);
+            }
+
+            JsonObject metaData = new JsonObject();
+            for (String[] item : vmData) {
+                String dataType = item[CONFIGDATA_DIR];
+                String fileName = item[CONFIGDATA_FILE];
+                String content = item[CONFIGDATA_CONTENT];
+                LOG.debug(String.format("[createConfigDriveIsoForVM] dataType=%s, filename=%s, content=%s",
+                        dataType, fileName, (fileName.equals(PASSWORD_FILE)?"********":content)));
+
+                // create file with content in folder
+                if (dataType != null && !dataType.isEmpty()) {
+                    //create folder
+                    File typeFolder = new File(tempDirName + ConfigDrive.cloudStackConfigDriveName + dataType);
+                    if (typeFolder.exists() || typeFolder.mkdirs()) {
+                        if (StringUtils.isNotEmpty(content)) {
+                            File file = new File(typeFolder, fileName + ".txt");
+                            try  {
+                                if (fileName.equals(USERDATA_FILE)) {
+                                    // User Data is passed as a base64 encoded string
+                                    FileUtils.writeByteArrayToFile(file, Base64.decodeBase64(content));
+                                } else {
+                                    FileUtils.write(file, content, com.cloud.utils.StringUtils.getPreferredCharset());
+                                }
+                            } catch (IOException ex) {
+                                throw new CloudRuntimeException("Failed to create file ", ex);
+                            }
+                        }
+                    } else {
+                        throw new CloudRuntimeException("Failed to create folder: " + typeFolder);
+                    }
+
+                    //now write the file to the OpenStack directory
+                    metaData = buildOpenStackMetaData(metaData, dataType, fileName, content);
+                }
+            }
+            writeFile(openStackFolder, "meta_data.json", metaData.toString());
+
+            String linkResult = linkUserData(tempDirName);
+            if (linkResult != null) {
+                String errMsg = "Unable to create user_data link due to " + linkResult;
+                throw new CloudRuntimeException(errMsg);
+            }
+
+            File tmpIsoStore = new File(tempDirName, new File(isoFileName).getName());
+            Script command = new Script("/usr/bin/genisoimage", Duration.standardSeconds(300), LOG);
+            command.add("-o", tmpIsoStore.getAbsolutePath());
+            command.add("-ldots");
+            command.add("-allow-lowercase");
+            command.add("-allow-multidot");
+            command.add("-cache-inodes"); // Enable caching inode and device numbers to find hard links to files.
+            command.add("-l");
+            command.add("-quiet");
+            command.add("-J");
+            command.add("-r");
+            command.add("-V", driveLabel);
+            command.add(tempDirName);
+            LOG.debug("Executing config drive creation command: " + command.toString());
+            String result = command.execute();
+            if (result != null) {
+                String errMsg = "Unable to create iso file: " + isoFileName + " due to " + result;
+                LOG.warn(errMsg);
+                throw new CloudRuntimeException(errMsg);
+            }
+            File tmpIsoFile = new File(tmpIsoStore.getAbsolutePath());
+            // Max allowed file size of config drive is 64MB: https://docs.openstack.org/project-install-guide/baremetal/draft/configdrive.html
+            if (tmpIsoFile.length() > (64L * 1024L * 1024L)) {
+                throw new CloudRuntimeException("Config drive file exceeds maximum allowed size of 64MB");
+            }
+            return fileToBase64String(tmpIsoFile);
+        } catch (IOException e) {
+            throw new CloudRuntimeException("Failed due to", e);
+        } finally {
+            try {
+                FileUtils.deleteDirectory(tempDir.toFile());
+            } catch (IOException ioe) {
+                LOG.warn("Failed to delete ConfigDrive temporary directory: " + tempDirName, ioe);
+            }
+        }
+    }
+
+    private static String linkUserData(String tempDirName) {
+        //Hard link the user_data.txt file with the user_data file in the OpenStack directory.
+        String userDataFilePath = tempDirName + ConfigDrive.cloudStackConfigDriveName + "userdata/user_data.txt";
+        if ((new File(userDataFilePath).exists())) {
+            Script hardLink = new Script("ln", Duration.standardSeconds(300), LOG);
+            hardLink.add(userDataFilePath);
+            hardLink.add(tempDirName + ConfigDrive.openStackConfigDriveName + "user_data");
+            LOG.debug("execute command: " + hardLink.toString());
+            return hardLink.execute();
+        }
+        return null;
+    }
+
+    private static JsonArray arrayOf(JsonElement... elements) {
+        JsonArray array = new JsonArray();
+        for (JsonElement element : elements) {
+            array.add(element);
+        }
+        return array;
+    }
+
+    private static JsonObject buildOpenStackMetaData(JsonObject metaData, String dataType, String fileName, String content) {
+        if (dataType.equals(NetworkModel.METATDATA_DIR) &&  StringUtils.isNotEmpty(content)) {
+            //keys are a special case in OpenStack format
+            if (NetworkModel.PUBLIC_KEYS_FILE.equals(fileName)) {
+                String[] keyArray = content.replace("\\n", "").split(" ");
+                String keyName = "key";
+                if (keyArray.length > 3 && StringUtils.isNotEmpty(keyArray[2])){
+                    keyName = keyArray[2];
+                }
+
+                JsonObject keyLegacy = new JsonObject();
+                keyLegacy.addProperty("type", "ssh");
+                keyLegacy.addProperty("data", content.replace("\\n", ""));
+                keyLegacy.addProperty("name", keyName);
+                metaData.add("keys", arrayOf(keyLegacy));
+
+                JsonObject key = new JsonObject();
+                key.addProperty(keyName, content);
+                metaData.add("public_keys", key);
+            } else if (NetworkModel.openStackFileMapping.get(fileName) != null) {
+                metaData.addProperty(NetworkModel.openStackFileMapping.get(fileName), content);
+            }
+        }
+        return metaData;
+    }
+
+}

engine/storage/configdrive/test/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java

@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.storage.configdrive;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ConfigDriveBuilderTest {
+
+    @Test
+    public void testConfigDriveIsoPath() throws IOException {
+        Assert.assertEquals(ConfigDrive.createConfigDrivePath("i-x-y"), "configdrive/i-x-y/configdrive.iso");
+    }
+
+    @Test
+    public void testConfigDriveBuild() throws IOException {
+        List<String[]> actualVmData = Arrays.asList(
+                new String[]{"userdata", "user_data", "c29tZSB1c2VyIGRhdGE="},
+                new String[]{"metadata", "service-offering", "offering"},
+                new String[]{"metadata", "availability-zone", "zone1"},
+                new String[]{"metadata", "local-hostname", "hostname"},
+                new String[]{"metadata", "local-ipv4", "192.168.111.111"},
+                new String[]{"metadata", "public-hostname", "7.7.7.7"},
+                new String[]{"metadata", "public-ipv4", "7.7.7.7"},
+                new String[]{"metadata", "vm-id", "uuid"},
+                new String[]{"metadata", "instance-id", "i-x-y"},
+                new String[]{"metadata", "public-keys", "ssh-rsa some-key"},
+                new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", "uuid")},
+                new String[]{"password", "vm_password", "password123"}
+        );
+
+        final Path tempDir = Files.createTempDirectory(ConfigDrive.CONFIGDRIVEDIR);
+        final String isoData = ConfigDriveBuilder.buildConfigDrive(actualVmData, "i-x-y.iso", "config-2");
+        final File isoFile = ConfigDriveBuilder.base64StringToFile(isoData, tempDir.toAbsolutePath().toString(), ConfigDrive.CONFIGDRIVEFILENAME);
+
+        Assert.assertTrue(isoFile.exists());
+        Assert.assertTrue(isoFile.isFile());
+        Assert.assertTrue(isoFile.length() > 0L);
+
+        FileUtils.deleteDirectory(tempDir.toFile());
+    }
+}

packaging/centos7/cloud.spec

@@ -75,7 +75,7 @@ Requires: sudo
 Requires: /sbin/service
 Requires: /sbin/chkconfig
 Requires: /usr/bin/ssh-keygen
-Requires: mkisofs
+Requires: genisoimage
 Requires: mysql-connector-python
 Requires: ipmitool
 Requires: %{name}-common = %{_ver}

plugins/hypervisors/kvm/pom.xml

@@ -39,6 +39,11 @@
       <artifactId>cloud-plugin-network-ovs</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-engine-storage-configdrive</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.ceph</groupId>
       <artifactId>rados</artifactId>

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -2200,9 +2200,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         final DataTO data = volume.getData();
         final DataStoreTO store = data.getDataStore();
 
-        if (volume.getType() == Volume.Type.ISO && data.getPath() != null) {
+        if (volume.getType() == Volume.Type.ISO && data.getPath() != null && (store instanceof NfsTO || store instanceof PrimaryDataStoreTO)) {
-            final NfsTO nfsStore = (NfsTO)store;
+            final String isoPath = store.getUrl().split("\\?")[0] + File.separator + data.getPath();
-            final String isoPath = nfsStore.getUrl() + File.separator + data.getPath();
             final int index = isoPath.lastIndexOf("/");
             final String path = isoPath.substring(0, index);
             final String name = isoPath.substring(index + 1);

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtHandleConfigDriveCommandWrapper.java

@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.hypervisor.kvm.resource.wrapper;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.apache.cloudstack.storage.configdrive.ConfigDriveBuilder;
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.HandleConfigDriveIsoCommand;
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
+import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+import com.cloud.storage.Storage;
+
+@ResourceWrapper(handles =  HandleConfigDriveIsoCommand.class)
+public final class LibvirtHandleConfigDriveCommandWrapper extends CommandWrapper<HandleConfigDriveIsoCommand, Answer, LibvirtComputingResource> {
+    private static final Logger LOG = Logger.getLogger(LibvirtHandleConfigDriveCommandWrapper.class);
+
+    @Override
+    public Answer execute(final HandleConfigDriveIsoCommand command, final LibvirtComputingResource libvirtComputingResource) {
+        final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
+        final KVMStoragePool pool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, command.getDestStore().getUuid());
+        if (pool == null) {
+            return new Answer(command, false, "Pool not found, config drive for KVM is only supported for NFS");
+        }
+
+        final String mountPoint = pool.getLocalPath();
+        final Path isoPath = Paths.get(mountPoint, command.getIsoFile());
+        final File isoFile = new File(mountPoint, command.getIsoFile());
+        if (command.isCreate()) {
+            LOG.debug("Creating config drive: " + command.getIsoFile());
+            if (command.getIsoData() == null) {
+                return new Answer(command, false, "Invalid config drive ISO data received");
+            }
+            if (isoFile.exists()) {
+                LOG.debug("An old config drive iso already exists");
+            }
+            try {
+                Files.createDirectories(isoPath.getParent());
+                ConfigDriveBuilder.base64StringToFile(command.getIsoData(), mountPoint, command.getIsoFile());
+            } catch (IOException e) {
+                return new Answer(command, false, "Failed due to exception: " + e.getMessage());
+            }
+        } else {
+            try {
+                FileUtils.deleteDirectory(isoPath.getParent().toFile());
+            } catch (IOException e) {
+                LOG.warn("Failed to delete config drive: " + isoPath.toAbsolutePath().toString());
+                return new Answer(command, false, "Failed due to exception: " + e.getMessage());
+            }
+        }
+
+        return new Answer(command);
+    }
+}

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java

@@ -265,7 +265,7 @@ public class KVMStoragePoolManager {
         String uuid = null;
         String sourceHost = "";
         StoragePoolType protocol = null;
-        if (storageUri.getScheme().equalsIgnoreCase("nfs")) {
+        if (storageUri.getScheme().equalsIgnoreCase("nfs") || storageUri.getScheme().equalsIgnoreCase("NetworkFilesystem")) {
             sourcePath = storageUri.getPath();
             sourcePath = sourcePath.replace("//", "/");
             sourceHost = storageUri.getHost();

server/pom.xml

@@ -141,6 +141,11 @@
       <artifactId>cloud-framework-agent-lb</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-engine-storage-configdrive</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.opensaml</groupId>
       <artifactId>opensaml</artifactId>

server/src/com/cloud/network/element/ConfigDriveNetworkElement.java

@@ -20,20 +20,20 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+
 import javax.inject.Inject;
 
-import org.apache.log4j.Logger;
-
-import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
+import org.apache.cloudstack.storage.configdrive.ConfigDrive;
+import org.apache.cloudstack.storage.configdrive.ConfigDriveBuilder;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
+import org.apache.log4j.Logger;
+
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
-import com.cloud.agent.api.AttachIsoAnswer;
-import com.cloud.agent.api.AttachIsoCommand;
 import com.cloud.agent.api.HandleConfigDriveIsoCommand;
 import com.cloud.agent.api.to.DiskTO;
 import com.cloud.configuration.ConfigurationManager;
@@ -41,10 +41,8 @@ import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.deploy.DeployDestination;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
-import com.cloud.exception.OperationTimedoutException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.exception.UnsupportedServiceException;
-import com.cloud.host.Host;
 import com.cloud.host.dao.HostDao;
 import com.cloud.network.Network;
 import com.cloud.network.Network.Capability;
@@ -54,54 +52,48 @@ import com.cloud.network.NetworkMigrationResponder;
 import com.cloud.network.NetworkModel;
 import com.cloud.network.Networks.TrafficType;
 import com.cloud.network.PhysicalNetworkServiceProvider;
-import com.cloud.network.dao.NetworkDao;
 import com.cloud.offering.NetworkOffering;
 import com.cloud.service.dao.ServiceOfferingDao;
+import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.Storage;
+import com.cloud.storage.StoragePool;
 import com.cloud.storage.Volume;
+import com.cloud.storage.VolumeVO;
 import com.cloud.storage.dao.GuestOSCategoryDao;
 import com.cloud.storage.dao.GuestOSDao;
+import com.cloud.storage.dao.VolumeDao;
 import com.cloud.utils.component.AdapterBase;
+import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.fsm.StateListener;
 import com.cloud.utils.fsm.StateMachine2;
 import com.cloud.vm.Nic;
 import com.cloud.vm.NicProfile;
 import com.cloud.vm.ReservationContext;
 import com.cloud.vm.UserVmDetailVO;
-import com.cloud.vm.UserVmManager;
 import com.cloud.vm.UserVmVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineManager;
 import com.cloud.vm.VirtualMachineProfile;
-import com.cloud.vm.dao.DomainRouterDao;
 import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.UserVmDetailsDao;
 
 public class ConfigDriveNetworkElement extends AdapterBase implements NetworkElement, UserDataServiceProvider,
         StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualMachine>, NetworkMigrationResponder {
-    private static final Logger s_logger = Logger.getLogger(ConfigDriveNetworkElement.class);
+    private static final Logger LOG = Logger.getLogger(ConfigDriveNetworkElement.class);
 
     private static final Map<Service, Map<Capability, String>> capabilities = setCapabilities();
 
-    @Inject
-    NetworkDao _networkConfigDao;
     @Inject
     NetworkModel _networkMgr;
     @Inject
-    UserVmManager _userVmMgr;
-    @Inject
     UserVmDao _userVmDao;
     @Inject
     UserVmDetailsDao _userVmDetailsDao;
     @Inject
-    DomainRouterDao _routerDao;
-    @Inject
     ConfigurationManager _configMgr;
     @Inject
     DataCenterDao _dcDao;
     @Inject
-    AgentManager _agentManager;
-    @Inject
     ServiceOfferingDao _serviceOfferingDao;
     @Inject
     NetworkModel _networkModel;
@@ -110,16 +102,16 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
     @Inject
     GuestOSDao _guestOSDao;
     @Inject
+    VolumeDao _volumeDao;
+    @Inject
     HostDao _hostDao;
     @Inject
+    AgentManager agentManager;
+    @Inject
     DataStoreManager _dataStoreMgr;
     @Inject
     EndPointSelector _ep;
-    @Inject
-    VolumeOrchestrationService _volumeMgr;
 
-    private final static String CONFIGDRIVEFILENAME = "configdrive.iso";
-    private final static String CONFIGDRIVEDIR = "ConfigDrive";
     private final static Integer CONFIGDRIVEDISKSEQ = 4;
 
     private boolean canHandle(TrafficType trafficType) {
@@ -149,20 +141,13 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
         if (!nic.isDefaultNic()) {
             return true;
         }
-        // Remove form secondary storage
-        DataStore secondaryStore = _dataStoreMgr.getImageStore(network.getDataCenterId());
 
-        String isoFile =  "/" + CONFIGDRIVEDIR + "/" + vm.getInstanceName()+ "/" + CONFIGDRIVEFILENAME;
+        try {
-        HandleConfigDriveIsoCommand deleteCommand = new HandleConfigDriveIsoCommand(vm.getVmData(),
+            return deleteConfigDriveIso(vm.getVirtualMachine());
-                vm.getConfigDriveLabel(), secondaryStore.getTO(), isoFile, false, false);
+        } catch (ResourceUnavailableException e) {
-        // Delete the ISO on the secondary store
+            LOG.error("Failed to delete config drive due to: ", e);
-        EndPoint endpoint = _ep.select(secondaryStore);
-        if (endpoint == null) {
-            s_logger.error(String.format("Secondary store: %s not available", secondaryStore.getName()));
             return false;
         }
-        Answer answer = endpoint.sendMessage(deleteCommand);
-        return answer.getResult();
     }
 
     @Override
@@ -207,37 +192,49 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
     }
 
     private String getSshKey(VirtualMachineProfile profile) {
-        UserVmDetailVO vmDetailSshKey = _userVmDetailsDao.findDetail(profile.getId(), "SSH.PublicKey");
+        final UserVmDetailVO vmDetailSshKey = _userVmDetailsDao.findDetail(profile.getId(), "SSH.PublicKey");
         return (vmDetailSshKey!=null ? vmDetailSshKey.getValue() : null);
     }
 
     @Override
     public boolean addPasswordAndUserdata(Network network, NicProfile nic, VirtualMachineProfile profile, DeployDestination dest, ReservationContext context)
             throws ConcurrentOperationException, InsufficientCapacityException, ResourceUnavailableException {
-        String sshPublicKey = getSshKey(profile);
         return (canHandle(network.getTrafficType())
-                && updateConfigDrive(profile, sshPublicKey, nic))
+                && configureConfigDriveData(profile, nic))
-                && updateConfigDriveIso(network, profile, dest.getHost(), false);
+                && createConfigDriveIso(profile, dest);
     }
 
     @Override
-    public boolean savePassword(Network network, NicProfile nic, VirtualMachineProfile profile) throws ResourceUnavailableException {
+    public boolean savePassword(final Network network, final NicProfile nic, final VirtualMachineProfile vm) throws ResourceUnavailableException {
-        String sshPublicKey = getSshKey(profile);
+        // savePassword is called by resetPasswordForVirtualMachine API which requires VM to be shutdown
-        if (!(canHandle(network.getTrafficType()) && updateConfigDrive(profile, sshPublicKey, nic))) return false;
+        // Upper layers should save password in db, we do not need to update/create config drive iso at this point
-        return updateConfigDriveIso(network, profile, true);
+        // Config drive will be created with updated password when VM starts in future
+        if (vm != null && vm.getVirtualMachine().getState().equals(VirtualMachine.State.Running)) {
+            throw new CloudRuntimeException("VM should to stopped to reset password");
+        }
+        return canHandle(network.getTrafficType());
     }
 
     @Override
-    public boolean saveSSHKey(Network network, NicProfile nic, VirtualMachineProfile vm, String sshPublicKey) throws ResourceUnavailableException {
+    public boolean saveSSHKey(final Network network, final NicProfile nic, final VirtualMachineProfile vm, final String sshPublicKey) throws ResourceUnavailableException {
-        if (!(canHandle(network.getTrafficType()) && updateConfigDrive(vm, sshPublicKey, nic))) return false;
+        // saveSSHKey is called by resetSSHKeyForVirtualMachine API which requires VM to be shutdown
-        return updateConfigDriveIso(network, vm, true);
+        // Upper layers should save ssh public key in db, we do not need to update/create config drive iso at this point
+        // Config drive will be created with updated password when VM starts in future
+        if (vm != null && vm.getVirtualMachine().getState().equals(VirtualMachine.State.Running)) {
+            throw new CloudRuntimeException("VM should to stopped to reset password");
+        }
+        return canHandle(network.getTrafficType());
     }
 
     @Override
-    public boolean saveUserData(Network network, NicProfile nic, VirtualMachineProfile profile) throws ResourceUnavailableException {
+    public boolean saveUserData(final Network network, final NicProfile nic, final VirtualMachineProfile vm) throws ResourceUnavailableException {
-        String sshPublicKey = getSshKey(profile);
+        // saveUserData is called by updateVirtualMachine API which requires VM to be shutdown
-        if (!(canHandle(network.getTrafficType()) && updateConfigDrive(profile, sshPublicKey, nic))) return false;
+        // Upper layers should save userdata in db, we do not need to update/create config drive iso at this point
-        return updateConfigDriveIso(network, profile, true);
+        // Config drive will be created with updated password when VM starts in future
+        if (vm != null && vm.getVirtualMachine().getState().equals(VirtualMachine.State.Running)) {
+            throw new CloudRuntimeException("VM should to stopped to reset password");
+        }
+        return canHandle(network.getTrafficType());
     }
 
     @Override
@@ -251,31 +248,23 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
     }
 
     @Override
-    public boolean postStateTransitionEvent(StateMachine2.Transition<VirtualMachine.State, VirtualMachine.Event> transition, VirtualMachine vo, boolean status, Object opaque) {
+    public boolean postStateTransitionEvent(StateMachine2.Transition<VirtualMachine.State, VirtualMachine.Event> transition, VirtualMachine vm, boolean status, Object opaque) {
         if (transition.getToState().equals(VirtualMachine.State.Expunging) && transition.getEvent().equals(VirtualMachine.Event.ExpungeOperation)) {
-            Nic nic = _networkModel.getDefaultNic(vo.getId());
+            Nic nic = _networkModel.getDefaultNic(vm.getId());
+            if (nic == null) {
+                return true;
+            }
             try {
-                if (nic != null) {
                 final Network network = _networkMgr.getNetwork(nic.getNetworkId());
                 final UserDataServiceProvider userDataUpdateProvider = _networkModel.getUserDataUpdateProvider(network);
                 final Provider provider = userDataUpdateProvider.getProvider();
                 if (provider.equals(Provider.ConfigDrive)) {
-                        // Delete config drive ISO on destroy
+                    try {
-                        DataStore secondaryStore = _dataStoreMgr.getImageStore(vo.getDataCenterId());
+                        return deleteConfigDriveIso(vm);
-                        String isoFile = "/" + CONFIGDRIVEDIR + "/" + vo.getInstanceName() + "/" + CONFIGDRIVEFILENAME;
+                    } catch (ResourceUnavailableException e) {
-                        HandleConfigDriveIsoCommand deleteCommand = new HandleConfigDriveIsoCommand(null,
+                        LOG.error("Failed to delete config drive due to: ", e);
-                                null, secondaryStore.getTO(), isoFile, false, false);
-                        EndPoint endpoint = _ep.select(secondaryStore);
-                        if (endpoint == null) {
-                            s_logger.error(String.format("Secondary store: %s not available", secondaryStore.getName()));
                         return false;
                     }
-                        Answer answer = endpoint.sendMessage(deleteCommand);
-                        if (!answer.getResult()) {
-                            s_logger.error(String.format("Update ISO failed, details: %s", answer.getDetails()));
-                            return false;
-                        }
-                    }
                 }
             } catch (UnsupportedServiceException usse) {}
         }
@@ -285,9 +274,9 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
     @Override
     public boolean prepareMigration(NicProfile nic, Network network, VirtualMachineProfile vm, DeployDestination dest, ReservationContext context) {
         if (nic.isDefaultNic() && _networkModel.getUserDataUpdateProvider(network).getProvider().equals(Provider.ConfigDrive)) {
-            s_logger.trace(String.format("[prepareMigration] for vm: %s", vm.getInstanceName()));
+            LOG.trace(String.format("[prepareMigration] for vm: %s", vm.getInstanceName()));
-            DataStore secondaryStore = _dataStoreMgr.getImageStore(network.getDataCenterId());
+            final DataStore dataStore = findDataStore(vm, dest);
-            configureConfigDriveDisk(vm, secondaryStore);
+            addConfigDriveDisk(vm, dataStore);
             return false;
         }
         else return  true;
@@ -295,64 +284,114 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
 
     @Override
     public void rollbackMigration(NicProfile nic, Network network, VirtualMachineProfile vm, ReservationContext src, ReservationContext dst) {
-
     }
 
     @Override
     public void commitMigration(NicProfile nic, Network network, VirtualMachineProfile vm, ReservationContext src, ReservationContext dst) {
-
     }
 
-    private boolean updateConfigDriveIso(Network network, VirtualMachineProfile profile, boolean update) throws ResourceUnavailableException {
+    private DataStore findDataStore(VirtualMachineProfile profile, DeployDestination dest) {
-        return updateConfigDriveIso(network, profile, null, update);
+        DataStore dataStore = null;
+        if (VirtualMachineManager.VmConfigDriveOnPrimaryPool.value()) {
+            if (dest.getStorageForDisks() != null) {
+                for (final Volume volume : dest.getStorageForDisks().keySet()) {
+                    if (volume.getVolumeType() == Volume.Type.ROOT) {
+                        final StoragePool primaryPool = dest.getStorageForDisks().get(volume);
+                        dataStore = _dataStoreMgr.getDataStore(primaryPool.getId(), DataStoreRole.Primary);
+                        break;
+                    }
+                }
+            }
+            if (dataStore == null) {
+                final List<VolumeVO> volumes = _volumeDao.findByInstanceAndType(profile.getVirtualMachine().getId(), Volume.Type.ROOT);
+                if (volumes != null && volumes.size() > 0) {
+                    dataStore = _dataStoreMgr.getDataStore(volumes.get(0).getPoolId(), DataStoreRole.Primary);
+                }
             }
-
-    private boolean updateConfigDriveIso(Network network, VirtualMachineProfile profile, Host host, boolean update) throws ResourceUnavailableException {
-        Integer deviceKey = null;
-        Long hostId;
-        if (host == null) {
-            hostId = (profile.getVirtualMachine().getHostId() == null ? profile.getVirtualMachine().getLastHostId(): profile.getVirtualMachine().getHostId());
         } else {
-            hostId = host.getId();
+            dataStore = _dataStoreMgr.getImageStore(dest.getDataCenter().getId());
+        }
+        return dataStore;
     }
 
-        DataStore secondaryStore = _dataStoreMgr.getImageStore(network.getDataCenterId());
+    private Long findAgentIdForImageStore(final DataStore dataStore) throws ResourceUnavailableException {
-        // Detach the existing ISO file if the machine is running
+        EndPoint endpoint = _ep.select(dataStore);
-        if (update && profile.getVirtualMachine().getState().equals(VirtualMachine.State.Running)) {
-            s_logger.debug("Detach config drive ISO for  vm " + profile.getInstanceName() + " in host " + _hostDao.findById(hostId));
-            deviceKey = detachIso(secondaryStore, profile.getInstanceName(), hostId);
-        }
-
-        // Create/Update the iso on the secondary store
-        s_logger.debug(String.format("%s config drive ISO for  vm %s in host %s",
-                (update?"update":"create"), profile.getInstanceName(), _hostDao.findById(hostId).getName()));
-        EndPoint endpoint = _ep.select(secondaryStore);
         if (endpoint == null) {
-            throw new ResourceUnavailableException(String.format("%s failed, secondary store not available", (update ? "Update" : "Create")), secondaryStore.getClass(),
+            throw new ResourceUnavailableException("Config drive creation failed, secondary store not available",
-                                                   secondaryStore.getId());
+                    dataStore.getClass(), dataStore.getId());
         }
-        String isoPath = CONFIGDRIVEDIR + "/" + profile.getInstanceName() + "/"  + CONFIGDRIVEFILENAME;
+        return endpoint.getId();
-        HandleConfigDriveIsoCommand configDriveIsoCommand = new HandleConfigDriveIsoCommand(profile.getVmData(),
-                profile.getConfigDriveLabel(), secondaryStore.getTO(), isoPath, true, update);
-        Answer createIsoAnswer = endpoint.sendMessage(configDriveIsoCommand);
-        if (!createIsoAnswer.getResult()) {
-            throw new ResourceUnavailableException(String.format("%s ISO failed, details: %s",
-                    (update?"Update":"Create"), createIsoAnswer.getDetails()), ConfigDriveNetworkElement.class, 0L);
     }
-        configureConfigDriveDisk(profile, secondaryStore);
 
-        // Re-attach the ISO if the machine is running
+    private Long findAgentId(VirtualMachineProfile profile, DeployDestination dest, DataStore dataStore) throws ResourceUnavailableException {
-        if (update && profile.getVirtualMachine().getState().equals(VirtualMachine.State.Running)) {
+        Long agentId;
-            s_logger.debug("Re-attach config drive ISO for  vm " + profile.getInstanceName() + " in host " + _hostDao.findById(hostId));
+        if (dest.getHost() == null) {
-            attachIso(secondaryStore, profile.getInstanceName(), hostId, deviceKey);
+            agentId = (profile.getVirtualMachine().getHostId() == null ? profile.getVirtualMachine().getLastHostId() : profile.getVirtualMachine().getHostId());
+        } else {
+            agentId = dest.getHost().getId();
+        }
+        if (!VirtualMachineManager.VmConfigDriveOnPrimaryPool.value()) {
+            agentId = findAgentIdForImageStore(dataStore);
+        }
+        return agentId;
+    }
+
+    private boolean createConfigDriveIso(VirtualMachineProfile profile, DeployDestination dest) throws ResourceUnavailableException {
+        final DataStore dataStore = findDataStore(profile, dest);
+        final Long agentId = findAgentId(profile, dest, dataStore);
+        if (agentId == null || dataStore == null) {
+            throw new ResourceUnavailableException("Config drive iso creation failed, agent or datastore not available",
+                    ConfigDriveNetworkElement.class, 0L);
+        }
+
+        LOG.debug("Creating config drive ISO for vm: " + profile.getInstanceName());
+
+        final String isoPath = ConfigDrive.createConfigDrivePath(profile.getInstanceName());
+        final String isoData = ConfigDriveBuilder.buildConfigDrive(profile.getVmData(), ConfigDrive.CONFIGDRIVEFILENAME, profile.getConfigDriveLabel());
+        final HandleConfigDriveIsoCommand configDriveIsoCommand = new HandleConfigDriveIsoCommand(isoPath, isoData, dataStore.getTO(), true);
+
+        final Answer answer = agentManager.easySend(agentId, configDriveIsoCommand);
+        if (!answer.getResult()) {
+            throw new ResourceUnavailableException(String.format("Config drive iso creation failed, details: %s",
+                    answer.getDetails()), ConfigDriveNetworkElement.class, 0L);
+        }
+        addConfigDriveDisk(profile, dataStore);
+        return true;
+    }
+
+    private boolean deleteConfigDriveIso(final VirtualMachine vm) throws ResourceUnavailableException {
+        DataStore dataStore = _dataStoreMgr.getImageStore(vm.getDataCenterId());
+        Long agentId = findAgentIdForImageStore(dataStore);
+
+        if (VirtualMachineManager.VmConfigDriveOnPrimaryPool.value()) {
+            List<VolumeVO> volumes = _volumeDao.findByInstanceAndType(vm.getId(), Volume.Type.ROOT);
+            if (volumes != null && volumes.size() > 0) {
+                dataStore = _dataStoreMgr.getDataStore(volumes.get(0).getPoolId(), DataStoreRole.Primary);
+            }
+            agentId = (vm.getHostId() != null) ? vm.getHostId() : vm.getLastHostId();
+        }
+
+        if (agentId == null || dataStore == null) {
+            throw new ResourceUnavailableException("Config drive iso creation failed, agent or datastore not available",
+                    ConfigDriveNetworkElement.class, 0L);
+        }
+
+        LOG.debug("Deleting config drive ISO for vm: " + vm.getInstanceName());
+
+        final String isoPath = ConfigDrive.createConfigDrivePath(vm.getInstanceName());
+        final HandleConfigDriveIsoCommand configDriveIsoCommand = new HandleConfigDriveIsoCommand(isoPath, null, dataStore.getTO(), false);
+
+        final Answer answer = agentManager.easySend(agentId, configDriveIsoCommand);
+        if (!answer.getResult()) {
+            LOG.error("Failed to remove config drive for instance: " + vm.getInstanceName());
+            return false;
         }
         return true;
-
     }
 
-    private void configureConfigDriveDisk(VirtualMachineProfile profile, DataStore secondaryStore) {
+    private void addConfigDriveDisk(final VirtualMachineProfile profile, final DataStore dataStore) {
         boolean isoAvailable = false;
-        String isoPath = CONFIGDRIVEDIR + "/" + profile.getInstanceName() + "/"  + CONFIGDRIVEFILENAME;
+        final String isoPath = ConfigDrive.createConfigDrivePath(profile.getInstanceName());
         for (DiskTO dataTo : profile.getDisks()) {
             if (dataTo.getPath().equals(isoPath)) {
                 isoAvailable = true;
@@ -361,71 +400,34 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
         }
         if (!isoAvailable) {
             TemplateObjectTO dataTO = new TemplateObjectTO();
-            dataTO.setDataStore(secondaryStore.getTO());
+            dataTO.setDataStore(dataStore.getTO());
             dataTO.setUuid(profile.getUuid());
             dataTO.setPath(isoPath);
             dataTO.setFormat(Storage.ImageFormat.ISO);
 
             profile.addDisk(new DiskTO(dataTO, CONFIGDRIVEDISKSEQ.longValue(), isoPath, Volume.Type.ISO));
+        } else {
+            LOG.warn("Config drive iso already is in VM profile.");
         }
     }
 
-    private boolean updateConfigDrive(VirtualMachineProfile profile, String publicKey, NicProfile nic) {
+    private boolean configureConfigDriveData(final VirtualMachineProfile profile, final NicProfile nic) {
-        UserVmVO vm = _userVmDao.findById(profile.getId());
+        final UserVmVO vm = _userVmDao.findById(profile.getId());
         if (vm.getType() != VirtualMachine.Type.User) {
             return false;
         }
-        // add/update userdata and/or password info into vm profile
+        final Nic defaultNic = _networkModel.getDefaultNic(vm.getId());
-        Nic defaultNic = _networkModel.getDefaultNic(vm.getId());
         if (defaultNic != null) {
+            final String sshPublicKey = getSshKey(profile);
             final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText();
             boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
 
-            List<String[]> vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getHostName(), vm.getId(),
+            final List<String[]> vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getHostName(), vm.getId(),
-                    vm.getUuid(), nic.getIPv4Address(), publicKey, (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
+                    vm.getUuid(), nic.getIPv4Address(), sshPublicKey, (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
             profile.setVmData(vmData);
             profile.setConfigDriveLabel(VirtualMachineManager.VmConfigDriveLabel.value());
         }
         return true;
     }
 
-    private Integer detachIso (DataStore secondaryStore, String instanceName, Long hostId) throws ResourceUnavailableException {
-        String isoPath = CONFIGDRIVEDIR + "/" + instanceName + "/"  + CONFIGDRIVEFILENAME;
-        AttachIsoCommand isoCommand = new AttachIsoCommand(instanceName, secondaryStore.getUri() + "/" + isoPath, false, CONFIGDRIVEDISKSEQ, true);
-        isoCommand.setStoreUrl(secondaryStore.getUri());
-        Answer attachIsoAnswer = null;
-
-        try {
-            attachIsoAnswer = _agentManager.send(hostId, isoCommand);
-        } catch (OperationTimedoutException e) {
-            throw new ResourceUnavailableException("Detach ISO failed: " + e.getMessage(), ConfigDriveNetworkElement.class, 0L);
-        }
-
-        if (!attachIsoAnswer.getResult()) {
-            throw new ResourceUnavailableException("Detach ISO failed: " + attachIsoAnswer.getDetails(), ConfigDriveNetworkElement.class, 0L);
-        }
-
-        if (attachIsoAnswer instanceof  AttachIsoAnswer) {
-            return ((AttachIsoAnswer)attachIsoAnswer).getDeviceKey();
-        } else {
-            return CONFIGDRIVEDISKSEQ;
-        }
-    }
-
-    private void attachIso (DataStore secondaryStore, String instanceName, Long hostId, Integer deviceKey) throws ResourceUnavailableException {
-        String isoPath = CONFIGDRIVEDIR + "/" + instanceName + "/"  + CONFIGDRIVEFILENAME;
-        AttachIsoCommand isoCommand = new AttachIsoCommand(instanceName, secondaryStore.getUri() + "/" + isoPath, true);
-        isoCommand.setStoreUrl(secondaryStore.getUri());
-        isoCommand.setDeviceKey(deviceKey);
-        Answer attachIsoAnswer = null;
-        try {
-            attachIsoAnswer = _agentManager.send(hostId, isoCommand);
-        } catch (OperationTimedoutException e) {
-            throw new ResourceUnavailableException("Attach ISO failed: " + e.getMessage() ,ConfigDriveNetworkElement.class,0L);
-        }
-        if (!attachIsoAnswer.getResult()) {
-            throw new ResourceUnavailableException("Attach ISO failed: " + attachIsoAnswer.getDetails(),ConfigDriveNetworkElement.class,0L);
-        }
-    }
-
 }

server/src/com/cloud/vm/UserVmManagerImpl.java

@@ -653,10 +653,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         }
     }
 
-
-
-
-
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_VM_RESETPASSWORD, eventDescription = "resetting Vm password", async = true)
     public UserVm resetVMPassword(ResetVMPasswordCmd cmd, String password) throws ResourceUnavailableException, InsufficientCapacityException {
@@ -681,6 +677,11 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             throw new InvalidParameterValueException("Vm with id " + vmId + " is not in the right state");
         }
 
+        if (userVm.getState() != State.Stopped) {
+            s_logger.error("vm is not in the right state: " + vmId);
+            throw new InvalidParameterValueException("Vm " + userVm + " should be stopped to do password reset");
+        }
+
         _accountMgr.checkAccess(caller, null, true, userVm);
 
         boolean result = resetVMPasswordInternal(vmId, password);
@@ -733,6 +734,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                 s_logger.debug("Failed to reset password for the virtual machine; no need to reboot the vm");
                 return false;
             } else {
+                final UserVmVO userVm = _vmDao.findById(vmId);
+                _vmDao.loadDetails(userVm);
+                userVm.setPassword(password);
+                // update the password in vm_details table too
+                // Check if an SSH key pair was selected for the instance and if so
+                // use it to encrypt & save the vm password
+                encryptAndStorePassword(userVm, password);
+
                 if (vmInstance.getState() == State.Stopped) {
                     s_logger.debug("Vm " + vmInstance + " is stopped, not rebooting it as a part of password reset");
                     return true;
@@ -796,15 +805,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
 
         boolean result = resetVMSSHKeyInternal(vmId, sshPublicKey, password);
 
-        if (result) {
+        if (!result) {
-            userVm.setDetail("SSH.PublicKey", sshPublicKey);
-            if (template != null && template.getEnablePassword()) {
-                userVm.setPassword(password);
-                //update the encrypted password in vm_details table too
-                encryptAndStorePassword(userVm, password);
-            }
-            _vmDao.saveDetails(userVm);
-        } else {
             throw new CloudRuntimeException("Failed to reset SSH Key for the virtual machine ");
         }
         return userVm;
@@ -842,6 +843,16 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             s_logger.debug("Failed to reset SSH Key for the virtual machine; no need to reboot the vm");
             return false;
         } else {
+            final UserVmVO userVm = _vmDao.findById(vmId);
+            _vmDao.loadDetails(userVm);
+            userVm.setDetail("SSH.PublicKey", sshPublicKey);
+            if (template.getEnablePassword()) {
+                userVm.setPassword(password);
+                //update the encrypted password in vm_details table too
+                encryptAndStorePassword(userVm, password);
+            }
+            _vmDao.saveDetails(userVm);
+
             if (vmInstance.getState() == State.Stopped) {
                 s_logger.debug("Vm " + vmInstance + " is stopped, not rebooting it as a part of SSH Key reset");
                 return true;
@@ -4099,8 +4110,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             }
         }
 
-
-
         _templateMgr.prepareIsoForVmProfile(profile, dest);
         return true;
     }
@@ -6135,14 +6144,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             if (template.getEnablePassword()) {
                 password = _mgr.generateRandomPassword();
                 boolean result = resetVMPasswordInternal(vmId, password);
-                if (result) {
+                if (!result) {
-                    vm.setPassword(password);
-                    _vmDao.loadDetails(vm);
-                    // update the password in vm_details table too
-                    // Check if an SSH key pair was selected for the instance and if so
-                    // use it to encrypt & save the vm password
-                    encryptAndStorePassword(vm, password);
-                } else {
                     throw new CloudRuntimeException("VM reset is completed but failed to reset password for the virtual machine ");
                 }
             }

server/test/com/cloud/network/element/ConfigDriveNetworkElementTest.java

@@ -16,7 +16,6 @@
 // under the License.
 package com.cloud.network.element;
 
-import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertFalse;
@@ -36,6 +35,12 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
+import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.storage.configdrive.ConfigDrive;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 import org.mockito.InjectMocks;
@@ -43,16 +48,10 @@ import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 import org.mockito.Spy;
 
-import com.google.common.collect.Maps;
+import com.cloud.agent.AgentManager;
-
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
-import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
-import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
-import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
-
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.HandleConfigDriveIsoCommand;
+import com.cloud.dc.DataCenter;
 import com.cloud.dc.DataCenterVO;
 import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.deploy.DeployDestination;
@@ -89,6 +88,7 @@ import com.cloud.vm.VirtualMachineProfileImpl;
 import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.UserVmDetailsDao;
 import com.cloud.vm.dao.VMInstanceDao;
+import com.google.common.collect.Maps;
 
 public class ConfigDriveNetworkElementTest {
 
@@ -106,6 +106,7 @@ public class ConfigDriveNetworkElementTest {
     private final long SOID = 31L;
     private final long HOSTID = NETWORK_ID;
 
+    @Mock private DataCenter dataCenter;
     @Mock private ConfigurationDao _configDao;
     @Mock private DataCenterDao _dcDao;
     @Mock private DataStoreManager _dataStoreMgr;
@@ -134,6 +135,7 @@ public class ConfigDriveNetworkElementTest {
     @Mock private ServiceOfferingVO serviceOfferingVO;
     @Mock private UserVmVO virtualMachine;
     @Mock private IPAddressVO publicIp;
+    @Mock private AgentManager agentManager;
 
     @InjectMocks private final ConfigDriveNetworkElement _configDrivesNetworkElement = new ConfigDriveNetworkElement();
     @InjectMocks @Spy private NetworkModelImpl _networkModel = new NetworkModelImpl();
@@ -145,6 +147,7 @@ public class ConfigDriveNetworkElementTest {
         _configDrivesNetworkElement._networkModel = _networkModel;
 
         when(_dataStoreMgr.getImageStore(DATACENTERID)).thenReturn(dataStore);
+
         when(_ep.select(dataStore)).thenReturn(endpoint);
         when(_vmDao.findById(VMID)).thenReturn(virtualMachine);
         when(_dcDao.findById(DATACENTERID)).thenReturn(dataCenterVO);
@@ -167,7 +170,9 @@ public class ConfigDriveNetworkElementTest {
         when(virtualMachine.getInstanceName()).thenReturn(VMINSTANCENAME);
         when(virtualMachine.getUserData()).thenReturn(VMUSERDATA);
         when(virtualMachine.getHostName()).thenReturn(VMHOSTNAME);
+        when(dataCenter.getId()).thenReturn(DATACENTERID);
         when(deployDestination.getHost()).thenReturn(hostVO);
+        when(deployDestination.getDataCenter()).thenReturn(dataCenter);
         when(hostVO.getId()).thenReturn(HOSTID);
         when(nic.isDefaultNic()).thenReturn(true);
         when(nic.getNetworkId()).thenReturn(NETWORK_ID);
@@ -210,25 +215,23 @@ public class ConfigDriveNetworkElementTest {
         when(_vmInstanceDao.updateState(VirtualMachine.State.Stopped, VirtualMachine.Event.ExpungeOperation, VirtualMachine.State.Expunging, virtualMachine, null)).thenReturn(true);
 
         final Answer answer = mock(Answer.class);
-        when(endpoint.sendMessage(any(HandleConfigDriveIsoCommand.class))).thenReturn(answer);
+        when(agentManager.easySend(anyLong(), any(HandleConfigDriveIsoCommand.class))).thenReturn(answer);
         when(answer.getResult()).thenReturn(true);
 
         stateMachine.transitTo(virtualMachine, VirtualMachine.Event.ExpungeOperation, null, _vmInstanceDao);
 
         ArgumentCaptor<HandleConfigDriveIsoCommand> commandCaptor = ArgumentCaptor.forClass(HandleConfigDriveIsoCommand.class);
-        verify(endpoint, times(1)).sendMessage(commandCaptor.capture());
+        verify(agentManager, times(1)).easySend(anyLong(), commandCaptor.capture());
         HandleConfigDriveIsoCommand deleteCommand = commandCaptor.getValue();
 
         assertThat(deleteCommand.isCreate(), is(false));
-        assertThat(deleteCommand.isUpdate(), is(false));
-
 
     }
 
     @Test
     public void testRelease() {
         final Answer answer = mock(Answer.class);
-        when(endpoint.sendMessage(any(HandleConfigDriveIsoCommand.class))).thenReturn(answer);
+        when(agentManager.easySend(anyLong(), any(HandleConfigDriveIsoCommand.class))).thenReturn(answer);
         when(answer.getResult()).thenReturn(true);
         VirtualMachineProfile profile = new VirtualMachineProfileImpl(virtualMachine, null, serviceOfferingVO, null, null);
         assertTrue(_configDrivesNetworkElement.release(network, nicp, profile, null));
@@ -240,81 +243,20 @@ public class ConfigDriveNetworkElementTest {
     }
 
     @Test
-    public void testAddPasswordAndUserdata() throws InsufficientCapacityException, ResourceUnavailableException {
+    public void testAddPasswordAndUserData() throws Exception {
-        List<String[]> actualVmData = getVmData();
-
-        assertThat(actualVmData, containsInAnyOrder(
-                new String[]{"userdata", "user_data", VMUSERDATA},
-                new String[]{"metadata", "service-offering", VMOFFERING},
-                new String[]{"metadata", "availability-zone", ZONENAME},
-                new String[]{"metadata", "local-hostname", VMHOSTNAME},
-                new String[]{"metadata", "local-ipv4", "192.168.111.111"},
-                new String[]{"metadata", "public-hostname", null},
-                new String[]{"metadata", "public-ipv4", "192.168.111.111"},
-                new String[]{"metadata", "vm-id", String.valueOf(VMID)},
-                new String[]{"metadata", "instance-id", VMINSTANCENAME},
-                new String[]{"metadata", "public-keys", PUBLIC_KEY},
-                new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)},
-                new String[]{PASSWORD, "vm_password", PASSWORD}
-        ));
-    }
-
-    @Test
-    public void testAddPasswordAndUserdataStaticNat() throws InsufficientCapacityException, ResourceUnavailableException {
-        when(_ipAddressDao.findByAssociatedVmId(VMID)).thenReturn(publicIp);
-        when(publicIp.getAddress()).thenReturn(new Ip("7.7.7.7"));
-
-        List<String[]> actualVmData = getVmData();
-
-        assertThat(actualVmData, containsInAnyOrder(
-                new String[]{"userdata", "user_data", VMUSERDATA},
-                new String[]{"metadata", "service-offering", VMOFFERING},
-                new String[]{"metadata", "availability-zone", ZONENAME},
-                new String[]{"metadata", "local-hostname", VMHOSTNAME},
-                new String[]{"metadata", "local-ipv4", "192.168.111.111"},
-                new String[]{"metadata", "public-hostname", "7.7.7.7"},
-                new String[]{"metadata", "public-ipv4", "7.7.7.7"},
-                new String[]{"metadata", "vm-id", String.valueOf(VMID)},
-                new String[]{"metadata", "instance-id", VMINSTANCENAME},
-                new String[]{"metadata", "public-keys", PUBLIC_KEY},
-                new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)},
-                new String[]{PASSWORD, "vm_password", PASSWORD}
-        ));
-    }
-
-
-    @Test
-    public void testAddPasswordAndUserdataUuid() throws InsufficientCapacityException, ResourceUnavailableException {
-        when(virtualMachine.getUuid()).thenReturn("vm-uuid");
-
-        List<String[]> actualVmData = getVmData();
-
-        assertThat(actualVmData, containsInAnyOrder(
-                new String[]{"userdata", "user_data", VMUSERDATA},
-                new String[]{"metadata", "service-offering", VMOFFERING},
-                new String[]{"metadata", "availability-zone", ZONENAME},
-                new String[]{"metadata", "local-hostname", VMHOSTNAME},
-                new String[]{"metadata", "local-ipv4", "192.168.111.111"},
-                new String[]{"metadata", "public-hostname", null},
-                new String[]{"metadata", "public-ipv4", "192.168.111.111"},
-                new String[]{"metadata", "vm-id", "vm-uuid"},
-                new String[]{"metadata", "instance-id", "vm-uuid"},
-                new String[]{"metadata", "public-keys", PUBLIC_KEY},
-                new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)},
-                new String[]{PASSWORD, "vm_password", PASSWORD}
-        ));
-    }
-
-    private List<String[]> getVmData() throws InsufficientCapacityException, ResourceUnavailableException {
         final Answer answer = mock(Answer.class);
         final UserVmDetailVO userVmDetailVO = mock(UserVmDetailVO.class);
-        when(endpoint.sendMessage(any(HandleConfigDriveIsoCommand.class))).thenReturn(answer);
+        when(agentManager.easySend(anyLong(), any(HandleConfigDriveIsoCommand.class))).thenReturn(answer);
         when(answer.getResult()).thenReturn(true);
         when(network.getTrafficType()).thenReturn(Networks.TrafficType.Guest);
         when(virtualMachine.getState()).thenReturn(VirtualMachine.State.Stopped);
+        when(virtualMachine.getUuid()).thenReturn("vm-uuid");
         when(userVmDetailVO.getValue()).thenReturn(PUBLIC_KEY);
         when(nicp.getIPv4Address()).thenReturn("192.168.111.111");
         when(_userVmDetailsDao.findDetail(anyLong(), anyString())).thenReturn(userVmDetailVO);
+        when(_ipAddressDao.findByAssociatedVmId(VMID)).thenReturn(publicIp);
+        when(publicIp.getAddress()).thenReturn(new Ip("7.7.7.7"));
+
         Map<VirtualMachineProfile.Param, Object> parms = Maps.newHashMap();
         parms.put(VirtualMachineProfile.Param.VmPassword, PASSWORD);
         parms.put(VirtualMachineProfile.Param.VmSshPubKey, PUBLIC_KEY);
@@ -323,8 +265,11 @@ public class ConfigDriveNetworkElementTest {
                 network, nicp, profile, deployDestination, null));
 
         ArgumentCaptor<HandleConfigDriveIsoCommand> commandCaptor = ArgumentCaptor.forClass(HandleConfigDriveIsoCommand.class);
-        verify(endpoint, times(1)).sendMessage(commandCaptor.capture());
+        verify(agentManager, times(1)).easySend(anyLong(), commandCaptor.capture());
-        HandleConfigDriveIsoCommand result = commandCaptor.getValue();
+        HandleConfigDriveIsoCommand createCommand = commandCaptor.getValue();
-        return result.getVmData();
+
+        assertTrue(createCommand.isCreate());
+        assertTrue(createCommand.getIsoData().length() > 0);
+        assertTrue(createCommand.getIsoFile().equals(ConfigDrive.createConfigDrivePath(profile.getInstanceName())));
     }
 }

services/secondary-storage/server/pom.xml

@@ -50,6 +50,11 @@
       <artifactId>cloud-server</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-engine-storage-configdrive</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <!-- dependencies for starting a post upload server on ssvm -->
       <dependency>
         <groupId>io.netty</groupId>

services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java

@@ -16,9 +16,6 @@
 // under the License.
 package org.apache.cloudstack.storage.resource;
 
-import static com.cloud.network.NetworkModel.CONFIGDATA_CONTENT;
-import static com.cloud.network.NetworkModel.CONFIGDATA_DIR;
-import static com.cloud.network.NetworkModel.CONFIGDATA_FILE;
 import static com.cloud.network.NetworkModel.METATDATA_DIR;
 import static com.cloud.network.NetworkModel.PASSWORD_DIR;
 import static com.cloud.network.NetworkModel.PASSWORD_FILE;
@@ -45,7 +42,6 @@ import java.io.UnsupportedEncodingException;
 import java.net.InetAddress;
 import java.net.URI;
 import java.net.UnknownHostException;
-import java.nio.charset.Charset;
 import java.nio.file.Path;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
@@ -56,21 +52,29 @@ import java.util.UUID;
 
 import javax.naming.ConfigurationException;
 
-import io.netty.bootstrap.ServerBootstrap;
+import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
-import io.netty.channel.Channel;
+import org.apache.cloudstack.storage.command.CopyCmdAnswer;
-import io.netty.channel.ChannelInitializer;
+import org.apache.cloudstack.storage.command.CopyCommand;
-import io.netty.channel.ChannelPipeline;
+import org.apache.cloudstack.storage.command.DeleteCommand;
-import io.netty.channel.EventLoopGroup;
+import org.apache.cloudstack.storage.command.DownloadCommand;
-import io.netty.channel.nio.NioEventLoopGroup;
+import org.apache.cloudstack.storage.command.DownloadProgressCommand;
-import io.netty.channel.socket.SocketChannel;
+import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
+import org.apache.cloudstack.storage.command.UploadStatusAnswer;
-import io.netty.handler.codec.http.HttpContentCompressor;
+import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
-import io.netty.handler.codec.http.HttpRequestDecoder;
+import org.apache.cloudstack.storage.command.UploadStatusCommand;
-import io.netty.handler.codec.http.HttpResponseEncoder;
+import org.apache.cloudstack.storage.configdrive.ConfigDrive;
-import io.netty.handler.logging.LogLevel;
+import org.apache.cloudstack.storage.configdrive.ConfigDriveBuilder;
-import io.netty.handler.logging.LoggingHandler;
+import org.apache.cloudstack.storage.template.DownloadManager;
-
+import org.apache.cloudstack.storage.template.DownloadManagerImpl;
-import org.apache.commons.codec.binary.Base64;
+import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
+import org.apache.cloudstack.storage.template.UploadEntity;
+import org.apache.cloudstack.storage.template.UploadManager;
+import org.apache.cloudstack.storage.template.UploadManagerImpl;
+import org.apache.cloudstack.storage.to.SnapshotObjectTO;
+import org.apache.cloudstack.storage.to.TemplateObjectTO;
+import org.apache.cloudstack.storage.to.VolumeObjectTO;
+import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
+import org.apache.cloudstack.utils.security.DigestHelper;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
@@ -87,37 +91,6 @@ import org.joda.time.DateTime;
 import org.joda.time.format.ISODateTimeFormat;
 
 import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.io.Files;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-
-import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
-import org.apache.cloudstack.storage.command.CopyCmdAnswer;
-import org.apache.cloudstack.storage.command.CopyCommand;
-import org.apache.cloudstack.storage.command.DeleteCommand;
-import org.apache.cloudstack.storage.command.DownloadCommand;
-import org.apache.cloudstack.storage.command.DownloadProgressCommand;
-import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
-import org.apache.cloudstack.storage.command.UploadStatusAnswer;
-import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
-import org.apache.cloudstack.storage.command.UploadStatusCommand;
-import org.apache.cloudstack.storage.template.DownloadManager;
-import org.apache.cloudstack.storage.template.DownloadManagerImpl;
-import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
-import org.apache.cloudstack.storage.template.UploadEntity;
-import org.apache.cloudstack.storage.template.UploadManager;
-import org.apache.cloudstack.storage.template.UploadManagerImpl;
-import org.apache.cloudstack.storage.to.SnapshotObjectTO;
-import org.apache.cloudstack.storage.to.TemplateObjectTO;
-import org.apache.cloudstack.storage.to.VolumeObjectTO;
-import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
-import org.apache.cloudstack.utils.security.DigestHelper;
-
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.CheckHealthAnswer;
 import com.cloud.agent.api.CheckHealthCommand;
@@ -164,7 +137,6 @@ import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.host.Host;
 import com.cloud.host.Host.Type;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
-import com.cloud.network.NetworkModel;
 import com.cloud.resource.ServerResourceBase;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.Storage;
@@ -192,6 +164,23 @@ import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.Script;
 import com.cloud.utils.storage.S3.S3Utils;
 import com.cloud.vm.SecondaryStorageVm;
+import com.google.common.collect.Maps;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.handler.codec.http.HttpContentCompressor;
+import io.netty.handler.codec.http.HttpRequestDecoder;
+import io.netty.handler.codec.http.HttpResponseEncoder;
+import io.netty.handler.logging.LogLevel;
+import io.netty.handler.logging.LoggingHandler;
 
 public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource {
 
@@ -200,8 +189,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
     private static final String TEMPLATE_ROOT_DIR = "template/tmpl";
     private static final String VOLUME_ROOT_DIR = "volumes";
     private static final String POST_UPLOAD_KEY_LOCATION = "/etc/cloudstack/agent/ms-psk";
-    private static final String cloudStackConfigDriveName = "/cloudstack/";
-    private static final String openStackConfigDriveName = "/openstack/latest/";
 
     private static final Map<String, String> updatableConfigData = Maps.newHashMap();
     static {
@@ -338,32 +325,32 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
     }
 
     private Answer execute(HandleConfigDriveIsoCommand cmd) {
-
         if (cmd.isCreate()) {
-            s_logger.debug(String.format("VMdata %s, attach = %s", cmd.getVmData(), cmd.isCreate()));
+            if (cmd.getIsoData() == null) {
-            if(cmd.getVmData() == null) return new Answer(cmd, false, "No Vmdata available");
+                return new Answer(cmd, false, "Invalid config drive ISO data");
+            }
             String nfsMountPoint = getRootDir(cmd.getDestStore().getUrl(), _nfsVersion);
             File isoFile = new File(nfsMountPoint, cmd.getIsoFile());
             if(isoFile.exists()) {
-                if (!cmd.isUpdate()) {
+                s_logger.debug("config drive iso already exists");
-                    return new Answer(cmd, true, "ISO already available");
+            }
-                } else {
+            Path tempDir = null;
-                    // Find out if we have to recover the password/ssh-key from the already available ISO.
             try {
-                        List<String[]> recoveredVmData = recoverVmData(isoFile);
+                tempDir = java.nio.file.Files.createTempDirectory(ConfigDrive.CONFIGDRIVEDIR);
-                        for (String[] vmDataEntry : cmd.getVmData()) {
+                File tmpIsoFile = ConfigDriveBuilder.base64StringToFile(cmd.getIsoData(), tempDir.toAbsolutePath().toString(), ConfigDrive.CONFIGDRIVEFILENAME);
-                            if (updatableConfigData.containsKey(vmDataEntry[CONFIGDATA_FILE])
+                copyLocalToNfs(tmpIsoFile, new File(cmd.getIsoFile()), cmd.getDestStore());
-                                    && updatableConfigData.get(vmDataEntry[CONFIGDATA_FILE]).equals(vmDataEntry[CONFIGDATA_DIR])) {
+            } catch (IOException | ConfigurationException e) {
-                                   updateVmData(recoveredVmData, vmDataEntry);
+                return new Answer(cmd, false, "Failed due to exception: " + e.getMessage());
+            } finally {
+                try {
+                    if (tempDir != null) {
+                        FileUtils.deleteDirectory(tempDir.toFile());
+                    }
+                } catch (IOException ioe) {
+                    s_logger.warn("Failed to delete ConfigDrive temporary directory: " + tempDir.toString(), ioe);
                 }
             }
-                        cmd.setVmData(recoveredVmData);
+            return new Answer(cmd, true, "Successfully saved config drive at secondary storage");
-                    } catch (IOException e) {
-                        return new Answer(cmd, e);
-                    }
-                }
-            }
-            return createConfigDriveIsoForVM(cmd);
         } else {
             DataStoreTO dstore = cmd.getDestStore();
             if (dstore instanceof NfsTO) {
@@ -383,237 +370,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         }
     }
 
-    private void updateVmData(List<String[]> recoveredVmData, String[] vmDataEntry) {
-        for (String[] recoveredEntry : recoveredVmData) {
-            if (recoveredEntry[CONFIGDATA_DIR].equals(vmDataEntry[CONFIGDATA_DIR])
-                    && recoveredEntry[CONFIGDATA_FILE].equals(vmDataEntry[CONFIGDATA_FILE])) {
-                recoveredEntry[CONFIGDATA_CONTENT] = vmDataEntry[CONFIGDATA_CONTENT];
-                return;
-            }
-        }
-        recoveredVmData.add(vmDataEntry);
-    }
-
-    private List<String[]> recoverVmData(File isoFile) throws IOException {
-        String tempDirName = null;
-        List<String[]> recoveredVmData = Lists.newArrayList();
-        boolean mounted = false;
-        try {
-            Path tempDir = java.nio.file.Files.createTempDirectory("ConfigDrive");
-            tempDirName = tempDir.toString();
-
-            // Unpack the current config drive file
-            Script command = new Script(!_inSystemVM, "mount", _timeout, s_logger);
-            command.add("-o", "loop");
-            command.add(isoFile.getAbsolutePath());
-            command.add(tempDirName);
-            String result = command.execute();
-
-            if (result != null) {
-                String errMsg = "Unable to mount " + isoFile.getAbsolutePath() + " at " + tempDirName + " due to " + result;
-                s_logger.error(errMsg);
-                throw new IOException(errMsg);
-            }
-            mounted = true;
-
-
-            // Scan directory structure
-            for (File configDirectory: (new File(tempDirName, "cloudstack")).listFiles()){
-                for (File configFile: configDirectory.listFiles()) {
-                    recoveredVmData.add(new String[]{configDirectory.getName(),
-                            Files.getNameWithoutExtension(configFile.getName()),
-                            Files.readFirstLine(configFile, Charset.defaultCharset())});
-                }
-            }
-
-        } finally {
-            if (mounted) {
-                Script command = new Script(!_inSystemVM, "umount", _timeout, s_logger);
-                command.add(tempDirName);
-                String result = command.execute();
-                if (result != null) {
-                    s_logger.warn("Unable to umount " + tempDirName + " due to " + result);
-                }
-            }
-            try {
-                FileUtils.deleteDirectory(new File(tempDirName));
-            } catch (IOException ioe) {
-                s_logger.warn("Failed to delete ConfigDrive temporary directory: " + tempDirName, ioe);
-            }
-        }
-        return  recoveredVmData;
-    }
-
-    public Answer createConfigDriveIsoForVM(HandleConfigDriveIsoCommand cmd) {
-        //create folder for the VM
-        if (cmd.getVmData() != null) {
-
-            Path tempDir = null;
-            String tempDirName = null;
-            try {
-                tempDir = java.nio.file.Files.createTempDirectory("ConfigDrive");
-                tempDirName = tempDir.toString();
-
-                //create OpenStack files
-                //create folder with empty files
-                File openStackFolder = new File(tempDirName + openStackConfigDriveName);
-                if (openStackFolder.exists() || openStackFolder.mkdirs()) {
-                    File vendorDataFile = new File(openStackFolder,"vendor_data.json");
-                    try (FileWriter fw = new FileWriter(vendorDataFile); BufferedWriter bw = new BufferedWriter(fw)) {
-                        bw.write("{}");
-                    } catch (IOException ex) {
-                        s_logger.error("Failed to create file ", ex);
-                        return new Answer(cmd, ex);
-                    }
-                    File networkDataFile = new File(openStackFolder, "network_data.json");
-                    try (FileWriter fw = new FileWriter(networkDataFile); BufferedWriter bw = new BufferedWriter(fw)) {
-                        bw.write("{}");
-                    } catch (IOException ex) {
-                        s_logger.error("Failed to create file ", ex);
-                        return new Answer(cmd, ex);
-                    }
-                } else {
-                    s_logger.error("Failed to create folder " + openStackFolder);
-                    return new Answer(cmd, false, "Failed to create folder " + openStackFolder);
-                }
-
-                JsonObject metaData = new JsonObject();
-                for (String[] item : cmd.getVmData()) {
-                    String dataType = item[CONFIGDATA_DIR];
-                    String fileName = item[CONFIGDATA_FILE];
-                    String content = item[CONFIGDATA_CONTENT];
-                    s_logger.debug(String.format("[createConfigDriveIsoForVM] dataType=%s, filename=%s, content=%s",
-                            dataType, fileName, (fileName.equals(PASSWORD_FILE)?"********":content)));
-
-                    // create file with content in folder
-                    if (dataType != null && !dataType.isEmpty()) {
-                        //create folder
-                        File typeFolder = new File(tempDirName + cloudStackConfigDriveName + dataType);
-                        if (typeFolder.exists() || typeFolder.mkdirs()) {
-                            if (StringUtils.isNotEmpty(content)) {
-                                File file = new File(typeFolder, fileName + ".txt");
-                                try  {
-                                    if (fileName.equals(USERDATA_FILE)) {
-                                        // User Data is passed as a base64 encoded string
-                                        FileUtils.writeByteArrayToFile(file, Base64.decodeBase64(content));
-                                    } else {
-                                        FileUtils.write(file, content, com.cloud.utils.StringUtils.getPreferredCharset());
-                                    }
-                                } catch (IOException ex) {
-                                    s_logger.error("Failed to create file ", ex);
-                                    return new Answer(cmd, ex);
-                                }
-                            }
-                        } else {
-                            s_logger.error("Failed to create folder " + typeFolder);
-                            return new Answer(cmd, false, "Failed to create folder " + typeFolder);
-                        }
-
-                        //now write the file to the OpenStack directory
-                        metaData = constructOpenStackMetaData(metaData, dataType, fileName, content);
-                    }
-                }
-
-                File metaDataFile = new File(openStackFolder, "meta_data.json");
-                try (FileWriter fw = new FileWriter(metaDataFile); BufferedWriter bw = new BufferedWriter(fw)) {
-                    bw.write(metaData.toString());
-                } catch (IOException ex) {
-                    s_logger.error("Failed to create file ", ex);
-                    return new Answer(cmd, ex);
-                }
-
-                String linkResult = linkUserData(tempDirName);
-                if (linkResult != null) {
-                    String errMsg = "Unable to create user_data link due to " + linkResult;
-                    s_logger.warn(errMsg);
-                    return new Answer(cmd, false, errMsg);
-                }
-
-                File tmpIsoStore = new File(tempDirName, new File(cmd.getIsoFile()).getName());
-                Script command = new Script(!_inSystemVM, "/usr/bin/genisoimage", _timeout, s_logger);
-                command.add("-o", tmpIsoStore.getAbsolutePath());
-                command.add("-ldots");
-                command.add("-allow-lowercase");
-                command.add("-allow-multidot");
-                command.add("-cache-inodes"); // Enable caching inode and device numbers to find hard links to files.
-                command.add("-l");
-                command.add("-quiet");
-                command.add("-J");
-                command.add("-r");
-                command.add("-V", cmd.getConfigDriveLabel());
-                command.add(tempDirName);
-                s_logger.debug("execute command: " + command.toString());
-                String result = command.execute();
-                if (result != null) {
-                    String errMsg = "Unable to create iso file: " + cmd.getIsoFile() + " due to " + result;
-                    s_logger.warn(errMsg);
-                    return new Answer(cmd, false, errMsg);
-                }
-                copyLocalToNfs(tmpIsoStore, new File(cmd.getIsoFile()), cmd.getDestStore());
-
-            } catch (IOException e) {
-                return new Answer(cmd, e);
-            } catch (ConfigurationException e) {
-                s_logger.warn("SecondStorageException ", e);
-                return new Answer(cmd, e);
-            } finally {
-                try {
-                    FileUtils.deleteDirectory(tempDir.toFile());
-                } catch (IOException ioe) {
-                    s_logger.warn("Failed to delete ConfigDrive temporary directory: " + tempDirName, ioe);
-                }
-            }
-        }
-        return new Answer(cmd);
-    }
-
-    JsonObject constructOpenStackMetaData(JsonObject metaData, String dataType, String fileName, String content) {
-        if (dataType.equals(NetworkModel.METATDATA_DIR) &&  StringUtils.isNotEmpty(content)) {
-            //keys are a special case in OpenStack format
-            if (NetworkModel.PUBLIC_KEYS_FILE.equals(fileName)) {
-                String[] keyArray = content.replace("\\n", "").split(" ");
-                String keyName = "key";
-                if (keyArray.length > 3 && StringUtils.isNotEmpty(keyArray[2])){
-                    keyName = keyArray[2];
-                }
-
-                JsonObject keyLegacy = new JsonObject();
-                keyLegacy.addProperty("type", "ssh");
-                keyLegacy.addProperty("data", content.replace("\\n", ""));
-                keyLegacy.addProperty("name", keyName);
-                metaData.add("keys", arrayOf(keyLegacy));
-
-                JsonObject key = new JsonObject();
-                key.addProperty(keyName, content);
-                metaData.add("public_keys", key);
-            } else if (NetworkModel.openStackFileMapping.get(fileName) != null) {
-                    metaData.addProperty(NetworkModel.openStackFileMapping.get(fileName), content);
-            }
-        }
-        return metaData;
-    }
-
-    private static JsonArray arrayOf(JsonElement... elements) {
-        JsonArray array = new JsonArray();
-        for (JsonElement element : elements) {
-            array.add(element);
-        }
-        return array;
-    }
-
-    private String linkUserData(String tempDirName) {
-        //Hard link the user_data.txt file with the user_data file in the OpenStack directory.
-        String userDataFilePath = tempDirName + cloudStackConfigDriveName + "userdata/user_data.txt";
-        if ((new File(userDataFilePath).exists())) {
-            Script hardLink = new Script(!_inSystemVM, "ln", _timeout, s_logger);
-            hardLink.add(userDataFilePath);
-            hardLink.add(tempDirName + openStackConfigDriveName + "user_data");
-            s_logger.debug("execute command: " + hardLink.toString());
-            return hardLink.execute();
-        }
-        return null;
-    }
-
     protected void copyLocalToNfs(File localFile, File isoFile, DataStoreTO destData) throws ConfigurationException, IOException {
         String scriptsDir = "scripts/storage/secondary";
         String createVolScr = Script.findScript(scriptsDir, "createvolume.sh");