From fe83a854368fb5161a5edce07607e70c0ff91b4b Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Wed, 20 Nov 2013 13:42:18 +0000 Subject: [PATCH] Add support for Primary Storage on Gluster using the libvirt backend The support for Gluster as Primary Storage is mostly based on the implementation for NFS. Like NFS, libvirt can address a Gluster environment through the 'netfs' pool-type. --- api/src/com/cloud/storage/Storage.java | 3 +- .../resource/LibvirtComputingResource.java | 16 ++++++++-- .../kvm/resource/LibvirtStoragePoolDef.java | 29 +++++++++++++++++-- .../resource/LibvirtStoragePoolXMLParser.java | 18 ++++++++++++ .../hypervisor/kvm/resource/LibvirtVMDef.java | 10 +++++-- .../kvm/storage/KVMStorageProcessor.java | 6 ++++ .../kvm/storage/LibvirtStorageAdaptor.java | 27 ++++++++++++----- ...oudStackPrimaryDataStoreLifeCycleImpl.java | 17 ++++++++++- 8 files changed, 111 insertions(+), 15 deletions(-) diff --git a/api/src/com/cloud/storage/Storage.java b/api/src/com/cloud/storage/Storage.java index ff83dfc1166..8ab2463a81b 100755 --- a/api/src/com/cloud/storage/Storage.java +++ b/api/src/com/cloud/storage/Storage.java @@ -98,7 +98,8 @@ public class Storage { PreSetup(true), // for XenServer, Storage Pool is set up by customers. EXT(false), // XenServer local EXT SR OCFS2(true), - SMB(true); + SMB(true), + Gluster(true); boolean shared; diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index f6243797c90..e6c750befac 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -1750,8 +1750,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv if (pool.getType() == StoragePoolType.CLVM && volFormat == PhysicalDiskFormat.RAW) { return "CLVM"; - } else if ((poolType == StoragePoolType.NetworkFilesystem || poolType == StoragePoolType.SharedMountPoint || poolType == StoragePoolType.Filesystem) && - volFormat == PhysicalDiskFormat.QCOW2) { + } else if ((poolType == StoragePoolType.NetworkFilesystem + || poolType == StoragePoolType.SharedMountPoint + || poolType == StoragePoolType.Filesystem + || poolType == StoragePoolType.Gluster) + && volFormat == PhysicalDiskFormat.QCOW2 ) { return "QCOW2"; } return null; @@ -3702,6 +3705,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv */ disk.defNetworkBasedDisk(physicalDisk.getPath().replace("rbd:", ""), pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), pool.getUuid(), devId, diskBusType, diskProtocol.RBD); + } else if (pool.getType() == StoragePoolType.Gluster) { + String mountpoint = pool.getLocalPath(); + String path = physicalDisk.getPath(); + String glusterVolume = pool.getSourceDir().replace("/", ""); + disk.defNetworkBasedDisk(glusterVolume + path.replace(mountpoint, ""), pool.getSourceHost(), pool.getSourcePort(), null, + null, devId, diskBusType, diskProtocol.GLUSTER); } else if (pool.getType() == StoragePoolType.CLVM || physicalDisk.getFormat() == PhysicalDiskFormat.RAW) { disk.defBlockBasedDisk(physicalDisk.getPath(), devId, diskBusType); } else { @@ -3859,6 +3868,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv if (attachingPool.getType() == StoragePoolType.RBD) { diskdef.defNetworkBasedDisk(attachingDisk.getPath(), attachingPool.getSourceHost(), attachingPool.getSourcePort(), attachingPool.getAuthUserName(), attachingPool.getUuid(), devId, DiskDef.diskBus.VIRTIO, diskProtocol.RBD); + } else if (attachingPool.getType() == StoragePoolType.Gluster) { + diskdef.defNetworkBasedDisk(attachingDisk.getPath(), attachingPool.getSourceHost(), attachingPool.getSourcePort(), null, + null, devId, DiskDef.diskBus.VIRTIO, diskProtocol.GLUSTER); } else if (attachingDisk.getFormat() == PhysicalDiskFormat.QCOW2) { diskdef.defFileBasedDisk(attachingDisk.getPath(), devId, DiskDef.diskBus.VIRTIO, DiskDef.diskFmtType.QCOW2); } else if (attachingDisk.getFormat() == PhysicalDiskFormat.RAW) { diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java index dbe5d4b7835..7631169f2b5 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java @@ -18,7 +18,7 @@ package com.cloud.hypervisor.kvm.resource; public class LibvirtStoragePoolDef { public enum poolType { - ISCSI("iscsi"), NETFS("netfs"), LOGICAL("logical"), DIR("dir"), RBD("rbd"); + ISCSI("iscsi"), NETFS("netfs"), LOGICAL("logical"), DIR("dir"), RBD("rbd"), GLUSTERFS("glusterfs"); String _poolType; poolType(String poolType) { @@ -127,7 +127,15 @@ public class LibvirtStoragePoolDef { @Override public String toString() { StringBuilder storagePoolBuilder = new StringBuilder(); - storagePoolBuilder.append("\n"); + if (_poolType == poolType.GLUSTERFS) { + /* libvirt mounts a Gluster volume, similar to NFS */ + storagePoolBuilder.append("\n"); + } else { + storagePoolBuilder.append("\n"); + } + storagePoolBuilder.append("" + _poolName + "\n"); if (_uuid != null) storagePoolBuilder.append("" + _uuid + "\n"); @@ -148,6 +156,23 @@ public class LibvirtStoragePoolDef { } storagePoolBuilder.append("\n"); } + if (_poolType == poolType.GLUSTERFS) { + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + } if (_poolType != poolType.RBD) { storagePoolBuilder.append("\n"); storagePoolBuilder.append("" + _targetPath + "\n"); diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java index a6186f6ba69..3f7909e9344 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java @@ -52,6 +52,7 @@ public class LibvirtStoragePoolXMLParser { Element source = (Element)rootElement.getElementsByTagName("source").item(0); String host = getAttrValue("host", "name", source); + String format = getAttrValue("format", "type", source); if (type.equalsIgnoreCase("rbd")) { int port = Integer.parseInt(getAttrValue("host", "port", source)); @@ -67,6 +68,23 @@ public class LibvirtStoragePoolXMLParser { } else { return new LibvirtStoragePoolDef(LibvirtStoragePoolDef.poolType.valueOf(type.toUpperCase()), poolName, uuid, host, port, pool, ""); } + /* Gluster is a sub-type of LibvirtStoragePoolDef.poolType.NETFS, need to check format */ + } else if (format != null && format.equalsIgnoreCase("glusterfs")) { + /* libvirt does not return the default port, but requires it for a disk-definition */ + int port = 24007; + + String path = getAttrValue("dir", "path", source); + + Element target = (Element) rootElement.getElementsByTagName( + "target").item(0); + String targetPath = getTagValue("path", target); + + String portValue = getAttrValue("host", "port", source); + if (portValue != "") + port = Integer.parseInt(portValue); + + return new LibvirtStoragePoolDef(LibvirtStoragePoolDef.poolType.valueOf(format.toUpperCase()), + poolName, uuid, host, port, path, targetPath); } else { String path = getAttrValue("dir", "path", source); diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index ff75d61e0b5..290c5a93663 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -378,7 +378,7 @@ public class LibvirtVMDef { } public enum diskProtocol { - RBD("rbd"), SHEEPDOG("sheepdog"); + RBD("rbd"), SHEEPDOG("sheepdog"), GLUSTER("gluster"); String _diskProtocol; diskProtocol(String protocol) { @@ -664,7 +664,13 @@ public class LibvirtVMDef { diskBuilder.append(" protocol='" + _diskProtocol + "'"); diskBuilder.append(" name='" + _sourcePath + "'"); diskBuilder.append(">\n"); - diskBuilder.append("\n"); + diskBuilder.append("\n"); diskBuilder.append("\n"); if (_authUserName != null) { diskBuilder.append("\n"); diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 8cdecd8dfa9..1c376070ebc 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -931,6 +931,12 @@ public class KVMStorageProcessor implements StorageProcessor { if (attachingPool.getType() == StoragePoolType.RBD) { diskdef.defNetworkBasedDisk(attachingDisk.getPath(), attachingPool.getSourceHost(), attachingPool.getSourcePort(), attachingPool.getAuthUserName(), attachingPool.getUuid(), devId, DiskDef.diskBus.VIRTIO, diskProtocol.RBD); + } else if (attachingPool.getType() == StoragePoolType.Gluster) { + String mountpoint = attachingPool.getLocalPath(); + String path = attachingDisk.getPath(); + String glusterVolume = attachingPool.getSourceDir().replace("/", ""); + diskdef.defNetworkBasedDisk(glusterVolume + path.replace(mountpoint, ""), attachingPool.getSourceHost(), attachingPool.getSourcePort(), null, + null, devId, DiskDef.diskBus.VIRTIO, diskProtocol.GLUSTER); } else if (attachingDisk.getFormat() == PhysicalDiskFormat.QCOW2) { diskdef.defFileBasedDisk(attachingDisk.getPath(), devId, DiskDef.diskBus.VIRTIO, DiskDef.diskFmtType.QCOW2); } else if (attachingDisk.getFormat() == PhysicalDiskFormat.RAW) { diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index a5f33ebe192..4a8f1f1b8d3 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -129,9 +129,9 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } } - private StoragePool createNfsStoragePool(Connect conn, String uuid, String host, String path) throws LibvirtException { + private StoragePool createNetfsStoragePool(poolType fsType, Connect conn, String uuid, String host, String path) throws LibvirtException { String targetPath = _mountPoint + File.separator + uuid; - LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(poolType.NETFS, uuid, uuid, host, path, targetPath); + LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(fsType, uuid, uuid, host, path, targetPath); _storageLayer.mkdir(targetPath); StoragePool sp = null; try { @@ -170,7 +170,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } sp.free(); } catch (LibvirtException l) { - s_logger.debug("Failed to undefine nfs storage pool with: " + l.toString()); + s_logger.debug("Failed to undefine " + fsType.toString() + " storage pool with: " + l.toString()); } } return null; @@ -345,14 +345,19 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { type = StoragePoolType.RBD; } else if (spd.getPoolType() == LibvirtStoragePoolDef.poolType.LOGICAL) { type = StoragePoolType.CLVM; + } else if (spd.getPoolType() == LibvirtStoragePoolDef.poolType.GLUSTERFS) { + type = StoragePoolType.Gluster; } LibvirtStoragePool pool = new LibvirtStoragePool(uuid, storage.getName(), type, this, storage); - if (pool.getType() != StoragePoolType.RBD) { + if (pool.getType() != StoragePoolType.RBD) pool.setLocalPath(spd.getTargetPath()); - } else { + else pool.setLocalPath(""); + + if (pool.getType() == StoragePoolType.RBD + || pool.getType() == StoragePoolType.Gluster) { pool.setSourceHost(spd.getSourceHost()); pool.setSourcePort(spd.getSourcePort()); pool.setSourceDir(spd.getSourceDir()); @@ -484,9 +489,17 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { if (type == StoragePoolType.NetworkFilesystem) { try { - sp = createNfsStoragePool(conn, name, host, path); + sp = createNetfsStoragePool(poolType.NETFS, conn, name, host, path); } catch (LibvirtException e) { - s_logger.error("Failed to create mount"); + s_logger.error("Failed to create netfs mount: " + host + ":" + path , e); + s_logger.error(e.getStackTrace()); + throw new CloudRuntimeException(e.toString()); + } + } else if (type == StoragePoolType.Gluster) { + try { + sp = createNetfsStoragePool(poolType.GLUSTERFS, conn, name, host, path); + } catch (LibvirtException e) { + s_logger.error("Failed to create glusterfs mount: " + host + ":" + path , e); s_logger.error(e.getStackTrace()); throw new CloudRuntimeException(e.toString()); } diff --git a/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index b90d5fc3780..3c1b76a62d3 100644 --- a/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -166,6 +166,12 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore if (uriPath == null) { throw new InvalidParameterValueException("host or path is null, should be rbd://hostname/pool"); } + } else if (uri.getScheme().equalsIgnoreCase("gluster")) { + String uriHost = uri.getHost(); + String uriPath = uri.getPath(); + if (uriHost == null || uriPath == null || uriHost.trim().isEmpty() || uriPath.trim().isEmpty()) { + throw new InvalidParameterValueException("host or path is null, should be gluster://hostname/volume"); + } } } catch (URISyntaxException e) { throw new InvalidParameterValueException(url + " is not a valid uri"); @@ -288,6 +294,14 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore parameters.setHost("clustered"); parameters.setPort(port); parameters.setPath(hostPath); + } else if (scheme.equalsIgnoreCase("gluster")) { + if (port == -1) { + port = 24007; + } + parameters.setType(StoragePoolType.Gluster); + parameters.setHost(storageHost); + parameters.setPort(port); + parameters.setPath(hostPath); } else { StoragePoolType type = Enum.valueOf(StoragePoolType.class, scheme); @@ -349,7 +363,8 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore if (pool.getPoolType() != StoragePoolType.NetworkFilesystem && pool.getPoolType() != StoragePoolType.Filesystem && pool.getPoolType() != StoragePoolType.IscsiLUN && pool.getPoolType() != StoragePoolType.Iscsi && pool.getPoolType() != StoragePoolType.VMFS && pool.getPoolType() != StoragePoolType.SharedMountPoint && pool.getPoolType() != StoragePoolType.PreSetup && pool.getPoolType() != StoragePoolType.OCFS2 && - pool.getPoolType() != StoragePoolType.RBD && pool.getPoolType() != StoragePoolType.CLVM && pool.getPoolType() != StoragePoolType.SMB) { + pool.getPoolType() != StoragePoolType.RBD && pool.getPoolType() != StoragePoolType.CLVM && pool.getPoolType() != StoragePoolType.SMB && + pool.getPoolType() != StoragePoolType.Gluster) { s_logger.warn(" Doesn't support storage pool type " + pool.getPoolType()); return false; }