From 1ed0830bc47824a629925b798c4de2efab2e34aa Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Wed, 26 Jan 2022 02:54:40 -0300 Subject: [PATCH] [Vmware][Deploy-as-is] Refactor the OVF parsing (#5825) * Add the list of supported namespaces per document and refactor the disks extraction by using the namespaces * Refactor matching the default OVF schema * Move parser methods to a new class and refactor * Fix import, unit tests * Reduce indentation * Address review comments --- .../cloud/agent/api/storage/OVFHelper.java | 271 ++++++------------ .../cloud/agent/api/storage/OVFParser.java | 112 ++++++++ .../agent/api/storage/OVFHelperTest.java | 29 +- .../cloud/storage/template/OVAProcessor.java | 14 +- .../storage/template/OVAProcessorTest.java | 2 + 5 files changed, 224 insertions(+), 204 deletions(-) create mode 100644 api/src/main/java/com/cloud/agent/api/storage/OVFParser.java diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java b/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java index 35835a523ef..42c198517c5 100644 --- a/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java +++ b/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java @@ -19,7 +19,6 @@ package com.cloud.agent.api.storage; import java.io.File; import java.io.IOException; import java.io.PrintWriter; -import java.io.StringReader; import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; @@ -30,9 +29,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; @@ -55,21 +51,29 @@ import org.apache.commons.lang.math.NumberUtils; import org.apache.log4j.Logger; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.w3c.dom.NamedNodeMap; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.w3c.dom.traversal.DocumentTraversal; import org.w3c.dom.traversal.NodeFilter; import org.w3c.dom.traversal.NodeIterator; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; import com.cloud.agent.api.to.DatadiskTO; import com.cloud.utils.exception.CloudRuntimeException; +import org.xml.sax.SAXException; public class OVFHelper { private static final Logger s_logger = Logger.getLogger(OVFHelper.class); + private final OVFParser ovfParser; + + public OVFHelper() { + ovfParser = new OVFParser(); + } + + public OVFParser getOvfParser() { + return this.ovfParser; + } + /** * Get disk virtual size given its values on fields: 'ovf:capacity' and 'ovf:capacityAllocationUnits' * @param capacity capacity @@ -92,91 +96,27 @@ public class OVFHelper { } } - /** - * Get the text value of a node's child with name or suffix "childNodeName", null if not present - * Example: - * - * Text value - * Text value - * - */ - private String getChildNodeValue(Node node, String childNodeName) { - if (node != null && node.hasChildNodes()) { - NodeList childNodes = node.getChildNodes(); - for (int i = 0; i < childNodes.getLength(); i++) { - Node value = childNodes.item(i); - // Also match if the child's name has a suffix: - // Example: - if (value != null && (value.getNodeName().equals(childNodeName)) || value.getNodeName().endsWith(":" + childNodeName)) { - return value.getTextContent(); - } - } - } - return null; - } - - /** - * Check if there are elements matching the tag name, otherwise check prepending the prefix - */ - public NodeList getElementsByTagNameAndPrefix(Document doc, String name, String prefix) { - if (doc == null) { - return null; - } - NodeList elementsByTagName = doc.getElementsByTagName(name); - if (elementsByTagName.getLength() > 0) { - return elementsByTagName; - } - return doc.getElementsByTagName(String.format("%s:%s", prefix, name)); - } - - /** - * Check if the attribute is present on the element, otherwise check preprending ':' - */ - private String getNodeAttribute(Element element, String prefix, String attr) { - if (element == null) { - return null; - } - if (element.hasAttribute(prefix + ":" + attr)) { - return element.getAttribute(prefix + ":" + attr); - } - else if (element.hasAttribute(attr)) { - return element.getAttribute(attr); - } - - NamedNodeMap attributes = element.getAttributes(); - if (attributes == null || attributes.getLength() == 0) { - return null; - } - for (int i = 0; i < attributes.getLength(); i++) { - Node node = attributes.item(i); - if (node != null && node.getNodeName().endsWith(":" + attr)) { - return node.getTextContent(); - } - } - return null; - } - /** * Create OVFProperty class from the parsed node. Note that some fields may not be present. * The key attribute is required */ protected OVFPropertyTO createOVFPropertyFromNode(Node node, int index, String category) { Element element = (Element) node; - String key = getNodeAttribute(element, "ovf","key"); + String key = ovfParser.getNodeAttribute(element, "key"); if (StringUtils.isBlank(key)) { return null; } - String value = getNodeAttribute(element, "ovf","value"); - String type = getNodeAttribute(element, "ovf","type"); - String qualifiers = getNodeAttribute(element, "ovf","qualifiers"); - String userConfigurableStr = getNodeAttribute(element, "ovf","userConfigurable"); + String value = ovfParser.getNodeAttribute(element, "value"); + String type = ovfParser.getNodeAttribute(element, "type"); + String qualifiers = ovfParser.getNodeAttribute(element, "qualifiers"); + String userConfigurableStr = ovfParser.getNodeAttribute(element, "userConfigurable"); boolean userConfigurable = StringUtils.isNotBlank(userConfigurableStr) && userConfigurableStr.equalsIgnoreCase("true"); - String passStr = getNodeAttribute(element, "ovf","password"); + String passStr = ovfParser.getNodeAttribute(element, "password"); boolean password = StringUtils.isNotBlank(passStr) && passStr.equalsIgnoreCase("true"); - String label = getChildNodeValue(node, "Label"); - String description = getChildNodeValue(node, "Description"); + String label = ovfParser.getChildNodeValue(node, "Label"); + String description = ovfParser.getChildNodeValue(node, "Description"); s_logger.debug("Creating OVF property index " + index + (category == null ? "" : " for category " + category) + " with key = " + key); return new OVFPropertyTO(key, type, value, qualifiers, userConfigurable, @@ -193,7 +133,7 @@ public class OVFHelper { return props; } int propertyIndex = 0; - NodeList productSections = getElementsByTagNameAndPrefix(doc, "ProductSection", "ovf"); + NodeList productSections = ovfParser.getElementsFromOVFDocument(doc, "ProductSection"); if (productSections != null) { String lastCategoryFound = null; for (int i = 0; i < productSections.getLength(); i++) { @@ -208,11 +148,11 @@ public class OVFHelper { continue; } if (child.getNodeName().equalsIgnoreCase("Category") || - child.getNodeName().equalsIgnoreCase("ovf:Category")) { + child.getNodeName().endsWith(":Category")) { lastCategoryFound = child.getTextContent(); s_logger.info("Category found " + lastCategoryFound); } else if (child.getNodeName().equalsIgnoreCase("Property") || - child.getNodeName().equalsIgnoreCase("ovf:Property")) { + child.getNodeName().endsWith(":Property")) { OVFPropertyTO prop = createOVFPropertyFromNode(child, propertyIndex, lastCategoryFound); if (prop != null && prop.isUserConfigurable()) { props.add(prop); @@ -228,39 +168,33 @@ public class OVFHelper { /** * Get properties from OVF XML string */ - protected List getOVFPropertiesFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { - InputSource is = new InputSource(new StringReader(ovfString)); - final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); + protected List getOVFPropertiesFromXmlString(final String ovfString) throws IOException, SAXException { + final Document doc = ovfParser.parseOVF(ovfString); return getConfigurableOVFPropertiesFromDocument(doc); } - protected Pair getOperatingSystemInfoFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { - InputSource is = new InputSource(new StringReader(ovfString)); - final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); + protected Pair getOperatingSystemInfoFromXmlString(final String ovfString) throws IOException, SAXException { + final Document doc = ovfParser.parseOVF(ovfString); return getOperatingSystemInfoFromDocument(doc); } - protected List getOVFDeploymentOptionsFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { - InputSource is = new InputSource(new StringReader(ovfString)); - final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); + protected List getOVFDeploymentOptionsFromXmlString(final String ovfString) throws IOException, SAXException { + final Document doc = ovfParser.parseOVF(ovfString); return getDeploymentOptionsFromDocumentTree(doc); } - protected List getOVFVirtualHardwareSectionFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { - InputSource is = new InputSource(new StringReader(ovfString)); - final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); + protected List getOVFVirtualHardwareSectionFromXmlString(final String ovfString) throws IOException, SAXException { + final Document doc = ovfParser.parseOVF(ovfString); return getVirtualHardwareItemsFromDocumentTree(doc); } - protected OVFVirtualHardwareSectionTO getVirtualHardwareSectionFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { - InputSource is = new InputSource(new StringReader(ovfString)); - final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); + protected OVFVirtualHardwareSectionTO getVirtualHardwareSectionFromXmlString(final String ovfString) throws IOException, SAXException { + final Document doc = ovfParser.parseOVF(ovfString); return getVirtualHardwareSectionFromDocument(doc); } - protected List getOVFEulaSectionFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { - InputSource is = new InputSource(new StringReader(ovfString)); - final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); + protected List getOVFEulaSectionFromXmlString(final String ovfString) throws IOException, SAXException { + final Document doc = ovfParser.parseOVF(ovfString); return getEulaSectionsFromDocument(doc); } @@ -268,7 +202,7 @@ public class OVFHelper { if (StringUtils.isBlank(ovfFilePath)) { return new ArrayList<>(); } - Document doc = getDocumentFromFile(ovfFilePath); + Document doc = ovfParser.parseOVFFile(ovfFilePath); return getOVFVolumeInfoFromFile(ovfFilePath, doc, configurationId); } @@ -287,8 +221,7 @@ public class OVFHelper { .filter(x -> x.getResourceType() == OVFVirtualHardwareItemTO.HardwareResourceType.DiskDrive && hardwareItemContainsConfiguration(x, configurationId)) .collect(Collectors.toList()); - List diskTOs = matchHardwareItemsToDiskAndFilesInformation(diskHardwareItems, files, disks, ovfFile.getParent()); - return diskTOs; + return matchHardwareItemsToDiskAndFilesInformation(diskHardwareItems, files, disks, ovfFile.getParent()); } private String extractDiskIdFromDiskHostResource(String hostResource) { @@ -367,31 +300,23 @@ public class OVFHelper { } protected List extractDisksFromOvfDocumentTree(Document doc) { - NodeList disks = doc.getElementsByTagName("Disk"); - NodeList ovfDisks = doc.getElementsByTagName("ovf:Disk"); - NodeList items = doc.getElementsByTagName("Item"); + NodeList disks = ovfParser.getElementsFromOVFDocument(doc, "Disk"); + NodeList items = ovfParser.getElementsFromOVFDocument(doc, "Item"); - int totalDisksLength = disks.getLength() + ovfDisks.getLength(); ArrayList vd = new ArrayList<>(); - for (int i = 0; i < totalDisksLength; i++) { - Element disk; - if (i >= disks.getLength()) { - int pos = i - disks.getLength(); - disk = (Element) ovfDisks.item(pos); - } else { - disk = (Element) disks.item(i); - } + for (int i = 0; i < disks.getLength(); i++) { + Element disk = (Element) disks.item(i); if (disk == null) { continue; } OVFDisk od = new OVFDisk(); - String virtualSize = getNodeAttribute(disk, "ovf", "capacity"); + String virtualSize = ovfParser.getNodeAttribute(disk, "capacity"); od._capacity = NumberUtils.toLong(virtualSize, 0L); - String allocationUnits = getNodeAttribute(disk,"ovf","capacityAllocationUnits"); - od._diskId = getNodeAttribute(disk,"ovf","diskId"); - od._fileRef = getNodeAttribute(disk,"ovf","fileRef"); - od._populatedSize = NumberUtils.toLong(getNodeAttribute(disk,"ovf","populatedSize")); + String allocationUnits = ovfParser.getNodeAttribute(disk, "capacityAllocationUnits"); + od._diskId = ovfParser.getNodeAttribute(disk, "diskId"); + od._fileRef = ovfParser.getNodeAttribute(disk, "fileRef"); + od._populatedSize = NumberUtils.toLong(ovfParser.getNodeAttribute(disk, "populatedSize")); if ((od._capacity != 0) && (allocationUnits != null)) { long units = 1; @@ -414,16 +339,16 @@ public class OVFHelper { } protected List extractFilesFromOvfDocumentTree(File ovfFile, Document doc) { - NodeList files = getElementsByTagNameAndPrefix(doc, "File", "ovf"); + NodeList files = ovfParser.getElementsFromOVFDocument(doc, "File"); ArrayList vf = new ArrayList<>(); boolean toggle = true; for (int j = 0; j < files.getLength(); j++) { Element file = (Element)files.item(j); OVFFile of = new OVFFile(); - of._href = getNodeAttribute(file,"ovf","href"); + of._href = ovfParser.getNodeAttribute(file, "href"); if (of._href.endsWith("vmdk") || of._href.endsWith("iso")) { - of._id = getNodeAttribute(file,"ovf","id"); - String size = getNodeAttribute(file,"ovf", "size"); + of._id = ovfParser.getNodeAttribute(file, "id"); + String size = ovfParser.getNodeAttribute(file, "size"); if (StringUtils.isNotBlank(size)) { of._size = Long.parseLong(size); } else { @@ -445,20 +370,6 @@ public class OVFHelper { return vf; } - public Document getDocumentFromFile(String ovfFilePath) { - if (org.apache.commons.lang.StringUtils.isBlank(ovfFilePath)) { - return null; - } - DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newDefaultInstance(); - try { - DocumentBuilder builder = documentBuilderFactory.newDocumentBuilder(); - return builder.parse(new File(ovfFilePath)); - } catch (SAXException | IOException | ParserConfigurationException e) { - s_logger.error("Unexpected exception caught while parsing ovf file:" + ovfFilePath, e); - throw new CloudRuntimeException(e); - } - } - private OVFDiskController getControllerType(final NodeList itemList, final String diskId) { for (int k = 0; k < itemList.getLength(); k++) { Element item = (Element)itemList.item(k); @@ -528,18 +439,18 @@ public class OVFHelper { } public void rewriteOVFFileForSingleDisk(final String origOvfFilePath, final String newOvfFilePath, final String diskName) { - final Document doc = getDocumentFromFile(origOvfFilePath); + final Document doc = ovfParser.parseOVFFile(origOvfFilePath); - NodeList disks = getElementsByTagNameAndPrefix(doc, "Disk", "ovf"); - NodeList files = getElementsByTagNameAndPrefix(doc, "File", "ovf"); - NodeList items = getElementsByTagNameAndPrefix(doc, "Item", "ovf"); + NodeList disks = ovfParser.getElementsFromOVFDocument(doc, "Disk"); + NodeList files = ovfParser.getElementsFromOVFDocument(doc, "File"); + NodeList items = ovfParser.getElementsFromOVFDocument(doc, "Item"); String keepfile = null; List toremove = new ArrayList<>(); for (int j = 0; j < files.getLength(); j++) { Element file = (Element)files.item(j); - String href = getNodeAttribute(file,"ovf", "href"); + String href = ovfParser.getNodeAttribute(file, "href"); if (diskName.equals(href)) { - keepfile = getNodeAttribute(file,"ovf","id"); + keepfile = ovfParser.getNodeAttribute(file, "id"); } else { toremove.add(file); } @@ -547,11 +458,11 @@ public class OVFHelper { String keepdisk = null; for (int i = 0; i < disks.getLength(); i++) { Element disk = (Element)disks.item(i); - String fileRef = getNodeAttribute(disk,"ovf","fileRef"); + String fileRef = ovfParser.getNodeAttribute(disk, "fileRef"); if (keepfile == null) { s_logger.info("FATAL: OVA format error"); } else if (keepfile.equals(fileRef)) { - keepdisk = getNodeAttribute(disk,"ovf","diskId"); + keepdisk = ovfParser.getNodeAttribute(disk, "diskId"); } else { toremove.add(disk); } @@ -660,13 +571,13 @@ public class OVFHelper { * @param parentNode the xml container node for nic data */ private void fillNicPrerequisites(OVFNetworkTO nic, Node parentNode) { - String addressOnParentStr = getChildNodeValue(parentNode, "AddressOnParent"); - String automaticAllocationStr = getChildNodeValue(parentNode, "AutomaticAllocation"); - String description = getChildNodeValue(parentNode, "Description"); - String elementName = getChildNodeValue(parentNode, "ElementName"); - String instanceIdStr = getChildNodeValue(parentNode, "InstanceID"); - String resourceSubType = getChildNodeValue(parentNode, "ResourceSubType"); - String resourceType = getChildNodeValue(parentNode, "ResourceType"); + String addressOnParentStr = ovfParser.getChildNodeValue(parentNode, "AddressOnParent"); + String automaticAllocationStr = ovfParser.getChildNodeValue(parentNode, "AutomaticAllocation"); + String description = ovfParser.getChildNodeValue(parentNode, "Description"); + String elementName = ovfParser.getChildNodeValue(parentNode, "ElementName"); + String instanceIdStr = ovfParser.getChildNodeValue(parentNode, "InstanceID"); + String resourceSubType = ovfParser.getChildNodeValue(parentNode, "ResourceSubType"); + String resourceType = ovfParser.getChildNodeValue(parentNode, "ResourceType"); try { int addressOnParent = Integer.parseInt(addressOnParentStr); @@ -693,7 +604,7 @@ public class OVFHelper { private void checkForOnlyOneSystemNode(Document doc) throws InternalErrorException { // get hardware VirtualSystem, for now we support only one of those - NodeList systemElements = getElementsByTagNameAndPrefix(doc, "VirtualSystem", "ovf"); + NodeList systemElements = ovfParser.getElementsFromOVFDocument(doc, "VirtualSystem"); if (systemElements.getLength() != 1) { String msg = "found " + systemElements.getLength() + " system definitions in OVA, can only handle exactly one."; s_logger.warn(msg); @@ -702,14 +613,14 @@ public class OVFHelper { } private Map getNetworksFromDocumentTree(Document doc) { - NodeList networkElements = getElementsByTagNameAndPrefix(doc,"Network", "ovf"); + NodeList networkElements = ovfParser.getElementsFromOVFDocument(doc,"Network"); Map nets = new HashMap<>(); for (int i = 0; i < networkElements.getLength(); i++) { Element networkElement = (Element)networkElements.item(i); - String networkName = getNodeAttribute(networkElement,"ovf","name"); + String networkName = ovfParser.getNodeAttribute(networkElement, "name"); - String description = getChildNodeValue(networkElement, "Description"); + String description = ovfParser.getChildNodeValue(networkElement, "Description"); OVFNetworkTO network = new OVFNetworkTO(); network.setName(networkName); @@ -762,10 +673,10 @@ public class OVFHelper { private String getMinimumHardwareVersionFromDocumentTree(Document doc) { String version = null; if (doc != null) { - NodeList systemNodeList = getElementsByTagNameAndPrefix(doc, "System", "ovf"); + NodeList systemNodeList = ovfParser.getElementsFromOVFDocument(doc, "System"); if (systemNodeList.getLength() != 0) { Node systemItem = systemNodeList.item(0); - String hardwareVersions = getChildNodeValue(systemItem, "VirtualSystemType"); + String hardwareVersions = ovfParser.getChildNodeValue(systemItem, "VirtualSystemType"); if (StringUtils.isNotBlank(hardwareVersions)) { String[] versions = hardwareVersions.split(","); // Order the hardware versions and retrieve the minimum version @@ -782,7 +693,7 @@ public class OVFHelper { if (doc == null) { return options; } - NodeList deploymentOptionSection = getElementsByTagNameAndPrefix(doc,"DeploymentOptionSection", "ovf"); + NodeList deploymentOptionSection = ovfParser.getElementsFromOVFDocument(doc,"DeploymentOptionSection"); if (deploymentOptionSection.getLength() == 0) { return options; } @@ -793,9 +704,9 @@ public class OVFHelper { Node node = childNodes.item(i); if (node != null && (node.getNodeName().equals("Configuration") || node.getNodeName().equals("ovf:Configuration"))) { Element configuration = (Element) node; - String configurationId = getNodeAttribute(configuration,"ovf","id"); - String description = getChildNodeValue(configuration, "Description"); - String label = getChildNodeValue(configuration, "Label"); + String configurationId = ovfParser.getNodeAttribute(configuration, "id"); + String description = ovfParser.getChildNodeValue(configuration, "Description"); + String label = ovfParser.getChildNodeValue(configuration, "Label"); OVFConfigurationTO option = new OVFConfigurationTO(configurationId, label, description, index); options.add(option); index++; @@ -809,7 +720,7 @@ public class OVFHelper { if (doc == null) { return items; } - NodeList hardwareSection = getElementsByTagNameAndPrefix(doc, "VirtualHardwareSection","ovf"); + NodeList hardwareSection = ovfParser.getElementsFromOVFDocument(doc, "VirtualHardwareSection"); if (hardwareSection.getLength() == 0) { return items; } @@ -819,18 +730,18 @@ public class OVFHelper { Node node = childNodes.item(i); if (node != null && (node.getNodeName().equals("Item") || node.getNodeName().equals("ovf:Item"))) { Element configuration = (Element) node; - String configurationIds = getNodeAttribute(configuration, "ovf", "configuration"); - String allocationUnits = getChildNodeValue(configuration, "AllocationUnits"); - String description = getChildNodeValue(configuration, "Description"); - String elementName = getChildNodeValue(configuration, "ElementName"); - String instanceID = getChildNodeValue(configuration, "InstanceID"); - String limit = getChildNodeValue(configuration, "Limit"); - String reservation = getChildNodeValue(configuration, "Reservation"); - String resourceType = getChildNodeValue(configuration, "ResourceType"); - String virtualQuantity = getChildNodeValue(configuration, "VirtualQuantity"); - String hostResource = getChildNodeValue(configuration, "HostResource"); - String addressOnParent = getChildNodeValue(configuration, "AddressOnParent"); - String parent = getChildNodeValue(configuration, "Parent"); + String configurationIds = ovfParser.getNodeAttribute(configuration, "configuration"); + String allocationUnits = ovfParser.getChildNodeValue(configuration, "AllocationUnits"); + String description = ovfParser.getChildNodeValue(configuration, "Description"); + String elementName = ovfParser.getChildNodeValue(configuration, "ElementName"); + String instanceID = ovfParser.getChildNodeValue(configuration, "InstanceID"); + String limit = ovfParser.getChildNodeValue(configuration, "Limit"); + String reservation = ovfParser.getChildNodeValue(configuration, "Reservation"); + String resourceType = ovfParser.getChildNodeValue(configuration, "ResourceType"); + String virtualQuantity = ovfParser.getChildNodeValue(configuration, "VirtualQuantity"); + String hostResource = ovfParser.getChildNodeValue(configuration, "HostResource"); + String addressOnParent = ovfParser.getChildNodeValue(configuration, "AddressOnParent"); + String parent = ovfParser.getChildNodeValue(configuration, "Parent"); OVFVirtualHardwareItemTO item = new OVFVirtualHardwareItemTO(); item.setConfigurationIds(configurationIds); item.setAllocationUnits(allocationUnits); @@ -885,7 +796,7 @@ public class OVFHelper { if (doc == null) { return eulas; } - NodeList eulaSections = getElementsByTagNameAndPrefix(doc, "EulaSection", "ovf"); + NodeList eulaSections = ovfParser.getElementsFromOVFDocument(doc, "EulaSection"); int eulaIndex = 0; if (eulaSections.getLength() > 0) { for (int index = 0; index < eulaSections.getLength(); index++) { @@ -921,14 +832,14 @@ public class OVFHelper { if (doc == null) { return null; } - NodeList guesOsList = getElementsByTagNameAndPrefix(doc, "OperatingSystemSection", "ovf"); + NodeList guesOsList = ovfParser.getElementsFromOVFDocument(doc, "OperatingSystemSection"); if (guesOsList.getLength() == 0) { return null; } Node guestOsNode = guesOsList.item(0); Element guestOsElement = (Element) guestOsNode; - String osType = getNodeAttribute(guestOsElement, "vmw", "osType"); - String description = getChildNodeValue(guestOsNode, "Description"); + String osType = ovfParser.getNodeAttribute(guestOsElement, "osType"); + String description = ovfParser.getChildNodeValue(guestOsNode, "Description"); return new Pair<>(osType, description); } diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFParser.java b/api/src/main/java/com/cloud/agent/api/storage/OVFParser.java new file mode 100644 index 00000000000..4c914464211 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/storage/OVFParser.java @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api.storage; + +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.Map; + +public class OVFParser { + private static final Logger s_logger = Logger.getLogger(OVFParser.class); + + private static final String DEFAULT_OVF_SCHEMA = "http://schemas.dmtf.org/ovf/envelope/1"; + private static final String VMW_SCHEMA = "http://www.vmware.com/schema/ovf"; + + private static final Map ATTRIBUTE_SCHEMA_MAP = Map.of( + "osType", VMW_SCHEMA + ); + + private DocumentBuilder documentBuilder; + + public OVFParser() { + try { + DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); + documentBuilderFactory.setNamespaceAware(true); + documentBuilder = documentBuilderFactory.newDocumentBuilder(); + } catch (ParserConfigurationException e) { + s_logger.error("Cannot start the OVF parser: " + e.getMessage(), e); + } + } + + public Document parseOVF(String ovfString) throws IOException, SAXException { + InputSource is = new InputSource(new StringReader(ovfString)); + return documentBuilder.parse(is); + } + + public Document parseOVFFile(String ovfFilePath) { + if (StringUtils.isBlank(ovfFilePath)) { + return null; + } + try { + return documentBuilder.parse(new File(ovfFilePath)); + } catch (SAXException | IOException e) { + s_logger.error("Error parsing " + ovfFilePath + " " + e.getMessage(), e); + return null; + } + } + + /** + * Retrieve elements with tag name from the document, according to the OVF schema definition + */ + public NodeList getElementsFromOVFDocument(Document doc, String tagName) { + return doc != null ? doc.getElementsByTagNameNS(DEFAULT_OVF_SCHEMA, tagName) : null; + } + + /** + * Retrieve an attribute value from an OVF element + */ + public String getNodeAttribute(Element element, String attr) { + return element != null ? element.getAttributeNS(ATTRIBUTE_SCHEMA_MAP.getOrDefault(attr, DEFAULT_OVF_SCHEMA), attr) : null; + } + + /** + * Get the text value of a node's child with name or suffix "childNodeName", null if not present + * Example: + * + * Text value + * Text value + * + */ + public String getChildNodeValue(Node node, String childNodeName) { + if (node == null || !node.hasChildNodes()) { + return null; + } + NodeList childNodes = node.getChildNodes(); + for (int i = 0; i < childNodes.getLength(); i++) { + Node value = childNodes.item(i); + // Also match if the child's name has a suffix: + // Example: + if (value != null && (value.getNodeName().equals(childNodeName)) || value.getNodeName().endsWith(":" + childNodeName)) { + return value.getTextContent(); + } + } + return null; + } +} diff --git a/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java b/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java index 5e7b7cf9bc9..ec0c10af14a 100644 --- a/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java +++ b/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java @@ -27,14 +27,13 @@ import org.junit.Test; import org.xml.sax.SAXException; import org.xml.sax.SAXParseException; -import javax.xml.parsers.ParserConfigurationException; import java.io.IOException; import java.util.List; public class OVFHelperTest { private String ovfFileProductSection = - "" + + "" + "VM Arguments" + "" + "" + @@ -47,7 +46,7 @@ public class OVFHelperTest { ""; private String ovfFileDeploymentOptionsSection = - "\n" + + "\n" + " Deployment Configuration information\n" + " \n" + " \n" + @@ -61,10 +60,10 @@ public class OVFHelperTest { " \n" + " Use this option to deploy an ASAv with a maximum throughput of 2 Gbps (uses 4 vCPUs and 8 GB of memory).\n" + " \n" + - " "; + " "; private String ovfFileVirtualHardwareSection = - "\n" + + "\n" + "\n" + " The kind of installed guest operating system\n" + " Other 2.6x Linux (64-bit)\n" + @@ -270,7 +269,7 @@ public class OVFHelperTest { ""; private String eulaSections = - "\n" + + "\n" + "\n" + " end-user license agreement\n" + " END USER LICENSE AGREEMENT\n" + @@ -395,7 +394,7 @@ public class OVFHelperTest { ""; private String productSectionWithCategories = - "\n" + + "\n" + "\n" + " Appliance ISV branding information\n" + " VMware vCenter Server Appliance\n" + @@ -704,49 +703,49 @@ public class OVFHelperTest { private OVFHelper ovfHelper = new OVFHelper(); @Test - public void testGetOVFPropertiesValidOVF() throws IOException, SAXException, ParserConfigurationException { + public void testGetOVFPropertiesValidOVF() throws IOException, SAXException { List props = ovfHelper.getOVFPropertiesFromXmlString(ovfFileProductSection); Assert.assertEquals(2, props.size()); } @Test(expected = SAXParseException.class) - public void testGetOVFPropertiesInvalidOVF() throws IOException, SAXException, ParserConfigurationException { + public void testGetOVFPropertiesInvalidOVF() throws IOException, SAXException { ovfHelper.getOVFPropertiesFromXmlString(ovfFileProductSection + "xxxxxxxxxxxxxxxxx"); } @Test - public void testGetOVFDeploymentOptionsValidOVF() throws IOException, SAXException, ParserConfigurationException { + public void testGetOVFDeploymentOptionsValidOVF() throws IOException, SAXException { List options = ovfHelper.getOVFDeploymentOptionsFromXmlString(ovfFileDeploymentOptionsSection); Assert.assertEquals(3, options.size()); } @Test - public void testGetOVFVirtualHardwareSectionValidOVF() throws IOException, SAXException, ParserConfigurationException { + public void testGetOVFVirtualHardwareSectionValidOVF() throws IOException, SAXException { List items = ovfHelper.getOVFVirtualHardwareSectionFromXmlString(ovfFileVirtualHardwareSection); Assert.assertEquals(20, items.size()); } @Test - public void testGetOVFEulaSectionValidOVF() throws IOException, SAXException, ParserConfigurationException { + public void testGetOVFEulaSectionValidOVF() throws IOException, SAXException { List eulas = ovfHelper.getOVFEulaSectionFromXmlString(eulaSections); Assert.assertEquals(2, eulas.size()); } @Test - public void testGetOVFPropertiesWithCategories() throws IOException, SAXException, ParserConfigurationException { + public void testGetOVFPropertiesWithCategories() throws IOException, SAXException { List props = ovfHelper.getOVFPropertiesFromXmlString(productSectionWithCategories); Assert.assertEquals(18, props.size()); } @Test - public void testGetOperatingSystemInfo() throws IOException, SAXException, ParserConfigurationException { + public void testGetOperatingSystemInfo() throws IOException, SAXException { Pair guestOsPair = ovfHelper.getOperatingSystemInfoFromXmlString(ovfFileVirtualHardwareSection); Assert.assertEquals("other26xLinux64Guest", guestOsPair.first()); Assert.assertEquals("Other 2.6x Linux (64-bit)", guestOsPair.second()); } @Test - public void testGetMinimumHardwareVersion() throws IOException, SAXException, ParserConfigurationException { + public void testGetMinimumHardwareVersion() throws IOException, SAXException { OVFVirtualHardwareSectionTO hardwareSection = ovfHelper.getVirtualHardwareSectionFromXmlString(ovfFileVirtualHardwareSection); Assert.assertEquals("vmx-08", hardwareSection.getMinimiumHardwareVersion()); } diff --git a/core/src/main/java/com/cloud/storage/template/OVAProcessor.java b/core/src/main/java/com/cloud/storage/template/OVAProcessor.java index d62897f20dc..1baea0539e3 100644 --- a/core/src/main/java/com/cloud/storage/template/OVAProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/OVAProcessor.java @@ -20,13 +20,10 @@ package com.cloud.storage.template; import java.io.File; -import java.io.IOException; import java.util.List; import java.util.Map; import javax.naming.ConfigurationException; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; import com.cloud.agent.api.to.OVFInformationTO; import com.cloud.agent.api.to.deployasis.OVFConfigurationTO; @@ -50,7 +47,6 @@ import com.cloud.storage.StorageLayer; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.script.Script; -import org.xml.sax.SAXException; /** * processes the content of an OVA for registration of a template @@ -107,7 +103,7 @@ public class OVAProcessor extends AdapterBase implements Processor { private void validateOva(String templateFileFullPath, FormatInfo info) throws InternalErrorException { String ovfFilePath = getOVFFilePath(templateFileFullPath); OVFHelper ovfHelper = new OVFHelper(); - Document doc = ovfHelper.getDocumentFromFile(ovfFilePath); + Document doc = ovfHelper.getOvfParser().parseOVFFile(ovfFilePath); OVFInformationTO ovfInformationTO = createOvfInformationTO(ovfHelper, doc, ovfFilePath); info.ovfInformationTO = ovfInformationTO; @@ -235,15 +231,15 @@ public class OVAProcessor extends AdapterBase implements Processor { String templateFileFullPath = templatePath.endsWith(File.separator) ? templatePath : templatePath + File.separator; templateFileFullPath += templateName.endsWith(ImageFormat.OVA.getFileExtension()) ? templateName : templateName + "." + ImageFormat.OVA.getFileExtension(); String ovfFileName = getOVFFilePath(templateFileFullPath); + OVFHelper ovfHelper = new OVFHelper(); if (ovfFileName == null) { String msg = "Unable to locate OVF file in template package directory: " + templatePath; LOGGER.error(msg); throw new InternalErrorException(msg); } try { - Document ovfDoc = null; - ovfDoc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new File(ovfFileName)); - NodeList diskElements = new OVFHelper().getElementsByTagNameAndPrefix(ovfDoc, "Disk", "ovf"); + Document ovfDoc = ovfHelper.getOvfParser().parseOVFFile(ovfFileName); + NodeList diskElements = ovfHelper.getOvfParser().getElementsFromOVFDocument(ovfDoc, "Disk"); for (int i = 0; i < diskElements.getLength(); i++) { Element disk = (Element)diskElements.item(i); String diskSizeValue = disk.getAttribute("ovf:capacity"); @@ -262,7 +258,7 @@ public class OVAProcessor extends AdapterBase implements Processor { virtualSize += diskSize; } return virtualSize; - } catch (InternalErrorException | IOException | NumberFormatException | ParserConfigurationException | SAXException e) { + } catch (InternalErrorException | NumberFormatException e) { String msg = "getTemplateVirtualSize: Unable to parse OVF XML document " + templatePath + " to get the virtual disk " + templateName + " size due to " + e; LOGGER.error(msg); throw new InternalErrorException(msg); diff --git a/core/src/test/java/com/cloud/storage/template/OVAProcessorTest.java b/core/src/test/java/com/cloud/storage/template/OVAProcessorTest.java index f2f0f4712dc..bb4a5a76ad1 100644 --- a/core/src/test/java/com/cloud/storage/template/OVAProcessorTest.java +++ b/core/src/test/java/com/cloud/storage/template/OVAProcessorTest.java @@ -30,6 +30,7 @@ import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @@ -38,6 +39,7 @@ import java.util.HashMap; import java.util.Map; @RunWith(PowerMockRunner.class) +@PowerMockIgnore({"javax.xml.*", "java.xml.*", "javax.management.*", "org.apache.xerces.*"}) @PrepareForTest(OVAProcessor.class) public class OVAProcessorTest { OVAProcessor processor;