mirror of
https://github.com/apache/cloudstack.git
synced 2025-10-26 08:42:29 +01:00
CS-9919: Support for Nexus Swiches (Cisco Vswitches)
Description: Instance configuration to join distributed vNetwork. Conflicts: core/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java core/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java vmware-base/src/com/cloud/hypervisor/vmware/util/VmwareHelper.java
This commit is contained in:
parent
7838b3fed6
commit
9709774b4b
@ -16,6 +16,7 @@ import java.io.File;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
import com.cloud.hypervisor.Hypervisor.HypervisorType;
|
||||||
import com.cloud.hypervisor.vmware.mo.HostMO;
|
import com.cloud.hypervisor.vmware.mo.HostMO;
|
||||||
import com.cloud.hypervisor.vmware.util.VmwareContext;
|
import com.cloud.hypervisor.vmware.util.VmwareContext;
|
||||||
import com.cloud.utils.Pair;
|
import com.cloud.utils.Pair;
|
||||||
@ -71,4 +72,10 @@ public interface VmwareManager {
|
|||||||
String getPublicVSwitchTypeGlobalParameter();
|
String getPublicVSwitchTypeGlobalParameter();
|
||||||
|
|
||||||
String getGuestVSwitchTypeGlobalParameter();
|
String getGuestVSwitchTypeGlobalParameter();
|
||||||
|
|
||||||
|
String getPrivateVSwitchName(long dcId, HypervisorType hypervisorType);
|
||||||
|
|
||||||
|
String getPublicVSwitchName(long dcId, HypervisorType hypervisorType);
|
||||||
|
|
||||||
|
String getGuestVSwitchName(long dcId, HypervisorType hypervisorType);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -156,7 +156,6 @@ import com.cloud.hypervisor.vmware.mo.DatastoreMO;
|
|||||||
import com.cloud.hypervisor.vmware.mo.DiskControllerType;
|
import com.cloud.hypervisor.vmware.mo.DiskControllerType;
|
||||||
import com.cloud.hypervisor.vmware.mo.HostFirewallSystemMO;
|
import com.cloud.hypervisor.vmware.mo.HostFirewallSystemMO;
|
||||||
import com.cloud.hypervisor.vmware.mo.HostMO;
|
import com.cloud.hypervisor.vmware.mo.HostMO;
|
||||||
import com.cloud.hypervisor.vmware.mo.HostVirtualNicType;
|
|
||||||
import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper;
|
import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper;
|
||||||
import com.cloud.hypervisor.vmware.mo.NetworkDetails;
|
import com.cloud.hypervisor.vmware.mo.NetworkDetails;
|
||||||
import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType;
|
import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType;
|
||||||
@ -1445,35 +1444,39 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
|
|||||||
if (vol.getType() != Volume.Type.ISO) {
|
if (vol.getType() != Volume.Type.ISO) {
|
||||||
Pair<ManagedObjectReference, DatastoreMO> volumeDsDetails = dataStoresDetails.get(vol.getPoolUuid());
|
Pair<ManagedObjectReference, DatastoreMO> volumeDsDetails = dataStoresDetails.get(vol.getPoolUuid());
|
||||||
assert (volumeDsDetails != null);
|
assert (volumeDsDetails != null);
|
||||||
VirtualDevice device;
|
VirtualDevice device;
|
||||||
datastoreDiskPath = String.format("[%s] %s.vmdk", volumeDsDetails.second().getName(), vol.getPath());
|
datastoreDiskPath = String.format("[%s] %s.vmdk", volumeDsDetails.second().getName(), vol.getPath());
|
||||||
String chainInfo = vol.getChainInfo();
|
String chainInfo = vol.getChainInfo();
|
||||||
|
|
||||||
|
if (chainInfo != null && !chainInfo.isEmpty()) {
|
||||||
|
String[] diskChain = _gson.fromJson(chainInfo, String[].class);
|
||||||
|
if (diskChain == null || diskChain.length < 1) {
|
||||||
|
s_logger.warn("Empty previously-saved chain info, fall back to the original");
|
||||||
|
device = VmwareHelper.prepareDiskDevice(vmMo, controllerKey, new String[] { datastoreDiskPath }, volumeDsDetails.first(), i, i + 1);
|
||||||
|
} else {
|
||||||
|
s_logger.info("Attach the disk with stored chain info: " + chainInfo);
|
||||||
|
for (int j = 0; j < diskChain.length; j++) {
|
||||||
|
diskChain[j] = String.format("[%s] %s", volumeDsDetails.second().getName(), diskChain[j]);
|
||||||
|
}
|
||||||
|
|
||||||
|
device = VmwareHelper.prepareDiskDevice(vmMo, controllerKey, diskChain, volumeDsDetails.first(), i, i + 1);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
device = VmwareHelper.prepareDiskDevice(vmMo, controllerKey, new String[] { datastoreDiskPath }, volumeDsDetails.first(), i, i + 1);
|
||||||
|
}
|
||||||
|
deviceConfigSpecArray[i].setDevice(device);
|
||||||
|
deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.add);
|
||||||
|
|
||||||
|
if(s_logger.isDebugEnabled())
|
||||||
|
s_logger.debug("Prepare volume at new device " + _gson.toJson(device));
|
||||||
|
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (chainInfo != null && !chainInfo.isEmpty()) {
|
String switchUuid;
|
||||||
String[] diskChain = _gson.fromJson(chainInfo, String[].class);
|
ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter();
|
||||||
if (diskChain == null || diskChain.length < 1) {
|
DatacenterMO dataCenterMo = new DatacenterMO(context, dcMor);
|
||||||
s_logger.warn("Empty previously-saved chain info, fall back to the original");
|
|
||||||
device = VmwareHelper.prepareDiskDevice(vmMo, controllerKey, new String[] { datastoreDiskPath }, volumeDsDetails.first(), i, i + 1);
|
|
||||||
} else {
|
|
||||||
s_logger.info("Attach the disk with stored chain info: " + chainInfo);
|
|
||||||
for (int j = 0; j < diskChain.length; j++) {
|
|
||||||
diskChain[j] = String.format("[%s] %s", volumeDsDetails.second().getName(), diskChain[j]);
|
|
||||||
}
|
|
||||||
|
|
||||||
device = VmwareHelper.prepareDiskDevice(vmMo, controllerKey, diskChain, volumeDsDetails.first(), i, i + 1);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
device = VmwareHelper.prepareDiskDevice(vmMo, controllerKey, new String[] { datastoreDiskPath }, volumeDsDetails.first(), i, i + 1);
|
|
||||||
}
|
|
||||||
deviceConfigSpecArray[i].setDevice(device);
|
|
||||||
deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.add);
|
|
||||||
|
|
||||||
if(s_logger.isDebugEnabled())
|
|
||||||
s_logger.debug("Prepare volume at new device " + _gson.toJson(device));
|
|
||||||
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
VirtualDevice nic;
|
VirtualDevice nic;
|
||||||
int nicMask = 0;
|
int nicMask = 0;
|
||||||
@ -1482,13 +1485,17 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
|
|||||||
s_logger.info("Prepare NIC device based on NicTO: " + _gson.toJson(nicTo));
|
s_logger.info("Prepare NIC device based on NicTO: " + _gson.toJson(nicTo));
|
||||||
|
|
||||||
Pair<ManagedObjectReference, String> networkInfo = prepareNetworkFromNicInfo(vmMo.getRunningHost(), nicTo);
|
Pair<ManagedObjectReference, String> networkInfo = prepareNetworkFromNicInfo(vmMo.getRunningHost(), nicTo);
|
||||||
|
|
||||||
nic = VmwareHelper.prepareNicDevice(vmMo, networkInfo.first(), nicDeviceType, networkInfo.second(), nicTo.getMac(), i, i + 1, true, true);
|
ManagedObjectReference dvsMor = dataCenterMo.getDvSwitchMor(networkInfo.first());
|
||||||
deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec();
|
switchUuid = dataCenterMo.getDvSwitchUuid(dvsMor);
|
||||||
deviceConfigSpecArray[i].setDevice(nic);
|
s_logger.info("Preparing NIC device on dvSwitch : " + switchUuid);
|
||||||
deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.add);
|
|
||||||
|
nic = VmwareHelper.prepareNicDevice(vmMo, networkInfo.first(), nicDeviceType, networkInfo.second(), nicTo.getMac(), i, i + 1, true, true);
|
||||||
if(s_logger.isDebugEnabled())
|
deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec();
|
||||||
|
deviceConfigSpecArray[i].setDevice(nic);
|
||||||
|
deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.add);
|
||||||
|
|
||||||
|
if(s_logger.isDebugEnabled())
|
||||||
s_logger.debug("Prepare NIC at new device " + _gson.toJson(deviceConfigSpecArray[i]));
|
s_logger.debug("Prepare NIC at new device " + _gson.toJson(deviceConfigSpecArray[i]));
|
||||||
|
|
||||||
// this is really a hacking for DomR, upon DomR startup, we will reset all the NIC allocation after eth3
|
// this is really a hacking for DomR, upon DomR startup, we will reset all the NIC allocation after eth3
|
||||||
@ -4022,16 +4029,25 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
|
|||||||
cfmMo.ensureCustomFieldDef("Network", CustomFieldConstants.CLOUD_GC);
|
cfmMo.ensureCustomFieldDef("Network", CustomFieldConstants.CLOUD_GC);
|
||||||
cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_UUID);
|
cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_UUID);
|
||||||
cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_NIC_MASK);
|
cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_NIC_MASK);
|
||||||
|
|
||||||
VmwareHypervisorHost hostMo = this.getHyperHost(context);
|
VmwareHypervisorHost hostMo = this.getHyperHost(context);
|
||||||
_hostName = hostMo.getHyperHostName();
|
_hostName = hostMo.getHyperHostName();
|
||||||
} catch (Exception e) {
|
_privateNetworkVSwitchName = mgr.getPrivateVSwitchName(Long.parseLong(_dcId), HypervisorType.VMware);
|
||||||
s_logger.error("Unexpected Exception ", e);
|
_publicNetworkVSwitchName = mgr.getPublicVSwitchName(Long.parseLong(_dcId), HypervisorType.VMware);
|
||||||
|
_guestNetworkVSwitchName = mgr.getGuestVSwitchName(Long.parseLong(_dcId), HypervisorType.VMware);
|
||||||
|
} catch (Exception e) {
|
||||||
|
s_logger.error("Unexpected Exception ", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
if(_privateNetworkVSwitchName == null) {
|
||||||
|
_privateNetworkVSwitchName = (String) params.get("private.network.vswitch.name");
|
||||||
|
}
|
||||||
|
if(_publicNetworkVSwitchName == null) {
|
||||||
|
_publicNetworkVSwitchName = (String) params.get("public.network.vswitch.name");
|
||||||
|
}
|
||||||
|
if(_guestNetworkVSwitchName == null) {
|
||||||
|
_guestNetworkVSwitchName = (String) params.get("guest.network.vswitch.name");
|
||||||
}
|
}
|
||||||
|
|
||||||
_privateNetworkVSwitchName = (String) params.get("private.network.vswitch.name");
|
|
||||||
_publicNetworkVSwitchName = (String) params.get("public.network.vswitch.name");
|
|
||||||
_guestNetworkVSwitchName = (String) params.get("guest.network.vswitch.name");
|
|
||||||
|
|
||||||
String value = (String) params.get("cpu.overprovisioning.factor");
|
String value = (String) params.get("cpu.overprovisioning.factor");
|
||||||
if(value != null)
|
if(value != null)
|
||||||
|
|||||||
@ -379,7 +379,22 @@ public class VmwareManagerImpl implements VmwareManager, VmwareStorageMount, Lis
|
|||||||
public String composeWorkerName() {
|
public String composeWorkerName() {
|
||||||
return UUID.randomUUID().toString().replace("-", "");
|
return UUID.randomUUID().toString().replace("-", "");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getPrivateVSwitchName(long dcId, HypervisorType hypervisorType) {
|
||||||
|
return _netMgr.getDefaultManagementTrafficLabel(dcId, hypervisorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getPublicVSwitchName(long dcId, HypervisorType hypervisorType) {
|
||||||
|
return _netMgr.getDefaultPublicTrafficLabel(dcId, hypervisorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getGuestVSwitchName(long dcId, HypervisorType hypervisorType) {
|
||||||
|
return _netMgr.getDefaultGuestTrafficLabel(dcId, hypervisorType);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<ManagedObjectReference> addHostToPodCluster(VmwareContext serviceContext, long dcId, Long podId, Long clusterId,
|
public List<ManagedObjectReference> addHostToPodCluster(VmwareContext serviceContext, long dcId, Long podId, Long clusterId,
|
||||||
String hostInventoryPath) throws Exception {
|
String hostInventoryPath) throws Exception {
|
||||||
|
|||||||
@ -395,7 +395,7 @@ public class DatacenterMO extends BaseMO {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public ManagedObjectReference getDvSwitchMor(ManagedObjectReference morDatacenter, ManagedObjectReference dvPortGroupMor) throws Exception {
|
public ManagedObjectReference getDvSwitchMor(ManagedObjectReference dvPortGroupMor) throws Exception {
|
||||||
String dvPortGroupKey = null;
|
String dvPortGroupKey = null;
|
||||||
ManagedObjectReference dvSwitchMor = null;
|
ManagedObjectReference dvSwitchMor = null;
|
||||||
PropertySpec pSpec = new PropertySpec();
|
PropertySpec pSpec = new PropertySpec();
|
||||||
@ -408,7 +408,7 @@ public class DatacenterMO extends BaseMO {
|
|||||||
datacenter2DvPortGroupTraversal.setName("datacenter2DvPortgroupTraversal");
|
datacenter2DvPortGroupTraversal.setName("datacenter2DvPortgroupTraversal");
|
||||||
|
|
||||||
ObjectSpec oSpec = new ObjectSpec();
|
ObjectSpec oSpec = new ObjectSpec();
|
||||||
oSpec.setObj(morDatacenter);
|
oSpec.setObj(_mor);
|
||||||
oSpec.setSkip(Boolean.TRUE);
|
oSpec.setSkip(Boolean.TRUE);
|
||||||
oSpec.setSelectSet(new SelectionSpec[] { datacenter2DvPortGroupTraversal });
|
oSpec.setSelectSet(new SelectionSpec[] { datacenter2DvPortGroupTraversal });
|
||||||
|
|
||||||
|
|||||||
@ -10,7 +10,8 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
//
|
//
|
||||||
// Automatically generated by addcopyright.py at 04/03/2012
|
// Automatically generated by addcopyright.py at 04/03/2012
|
||||||
package com.cloud.hypervisor.vmware.util;
|
|
||||||
|
package com.cloud.hypervisor.vmware.util;
|
||||||
|
|
||||||
import java.io.BufferedWriter;
|
import java.io.BufferedWriter;
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
@ -32,6 +33,7 @@ import com.cloud.hypervisor.vmware.mo.VirtualMachineMO;
|
|||||||
import com.cloud.utils.Pair;
|
import com.cloud.utils.Pair;
|
||||||
import com.cloud.utils.Ternary;
|
import com.cloud.utils.Ternary;
|
||||||
import com.cloud.utils.exception.ExceptionUtil;
|
import com.cloud.utils.exception.ExceptionUtil;
|
||||||
|
import com.vmware.vim25.DistributedVirtualSwitchPortConnection;
|
||||||
import com.vmware.vim25.DynamicProperty;
|
import com.vmware.vim25.DynamicProperty;
|
||||||
import com.vmware.vim25.ManagedObjectReference;
|
import com.vmware.vim25.ManagedObjectReference;
|
||||||
import com.vmware.vim25.MethodFault;
|
import com.vmware.vim25.MethodFault;
|
||||||
@ -53,6 +55,7 @@ import com.vmware.vim25.VirtualDiskSparseVer1BackingInfo;
|
|||||||
import com.vmware.vim25.VirtualDiskSparseVer2BackingInfo;
|
import com.vmware.vim25.VirtualDiskSparseVer2BackingInfo;
|
||||||
import com.vmware.vim25.VirtualE1000;
|
import com.vmware.vim25.VirtualE1000;
|
||||||
import com.vmware.vim25.VirtualEthernetCard;
|
import com.vmware.vim25.VirtualEthernetCard;
|
||||||
|
import com.vmware.vim25.VirtualEthernetCardDistributedVirtualPortBackingInfo;
|
||||||
import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo;
|
import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo;
|
||||||
import com.vmware.vim25.VirtualMachineConfigSpec;
|
import com.vmware.vim25.VirtualMachineConfigSpec;
|
||||||
import com.vmware.vim25.VirtualMachineSnapshotTree;
|
import com.vmware.vim25.VirtualMachineSnapshotTree;
|
||||||
@ -101,25 +104,72 @@ public class VmwareHelper {
|
|||||||
nic.setAddressType("Manual");
|
nic.setAddressType("Manual");
|
||||||
nic.setConnectable(connectInfo);
|
nic.setConnectable(connectInfo);
|
||||||
nic.setMacAddress(macAddress);
|
nic.setMacAddress(macAddress);
|
||||||
|
nic.setUnitNumber(deviceNumber);
|
||||||
nic.setUnitNumber(deviceNumber);
|
nic.setKey(-contextNumber);
|
||||||
nic.setKey(-contextNumber);
|
return nic;
|
||||||
return nic;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// vmdkDatastorePath: [datastore name] vmdkFilePath
|
public static VirtualDevice prepareDvNicDevice(VirtualMachineMO vmMo, ManagedObjectReference morNetwork, VirtualEthernetCardType deviceType,
|
||||||
public static VirtualDevice prepareDiskDevice(VirtualMachineMO vmMo, int controllerKey, String vmdkDatastorePath,
|
String dvPortGroupName, String dvSwitchUuid, String macAddress, int deviceNumber, int contextNumber, boolean conntected, boolean connectOnStart) throws Exception {
|
||||||
int sizeInMb, ManagedObjectReference morDs, int deviceNumber, int contextNumber) throws Exception {
|
|
||||||
|
|
||||||
VirtualDisk disk = new VirtualDisk();
|
|
||||||
|
|
||||||
VirtualDiskFlatVer2BackingInfo backingInfo = new VirtualDiskFlatVer2BackingInfo();
|
VirtualEthernetCard nic;
|
||||||
backingInfo.setDiskMode(VirtualDiskMode.persistent.toString());
|
switch (deviceType) {
|
||||||
backingInfo.setThinProvisioned(true);
|
case E1000:
|
||||||
backingInfo.setEagerlyScrub(false);
|
nic = new VirtualE1000();
|
||||||
backingInfo.setDatastore(morDs);
|
break;
|
||||||
backingInfo.setFileName(vmdkDatastorePath);
|
|
||||||
disk.setBacking(backingInfo);
|
case PCNet32:
|
||||||
|
nic = new VirtualPCNet32();
|
||||||
|
break;
|
||||||
|
|
||||||
|
case Vmxnet2:
|
||||||
|
nic = new VirtualVmxnet2();
|
||||||
|
break;
|
||||||
|
|
||||||
|
case Vmxnet3:
|
||||||
|
nic = new VirtualVmxnet3();
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
assert (false);
|
||||||
|
nic = new VirtualE1000();
|
||||||
|
}
|
||||||
|
|
||||||
|
final VirtualEthernetCardDistributedVirtualPortBackingInfo dvPortBacking = new VirtualEthernetCardDistributedVirtualPortBackingInfo();
|
||||||
|
final DistributedVirtualSwitchPortConnection dvPortConnection = new DistributedVirtualSwitchPortConnection();
|
||||||
|
final VirtualDeviceConnectInfo connectInfo = new VirtualDeviceConnectInfo();
|
||||||
|
|
||||||
|
dvPortConnection.setSwitchUuid(dvSwitchUuid);
|
||||||
|
dvPortConnection.setPortgroupKey(morNetwork.get_value());
|
||||||
|
dvPortBacking.setPort(dvPortConnection);
|
||||||
|
nic.setBacking(dvPortBacking);
|
||||||
|
nic.setKey(30);
|
||||||
|
|
||||||
|
connectInfo.setAllowGuestControl(true);
|
||||||
|
connectInfo.setConnected(conntected);
|
||||||
|
connectInfo.setStartConnected(connectOnStart);
|
||||||
|
nic.setAddressType("Manual");
|
||||||
|
nic.setConnectable(connectInfo);
|
||||||
|
nic.setMacAddress(macAddress);
|
||||||
|
|
||||||
|
nic.setUnitNumber(deviceNumber);
|
||||||
|
nic.setKey(-contextNumber);
|
||||||
|
return nic;
|
||||||
|
}
|
||||||
|
|
||||||
|
// vmdkDatastorePath: [datastore name] vmdkFilePath
|
||||||
|
public static VirtualDevice prepareDiskDevice(VirtualMachineMO vmMo, int controllerKey, String vmdkDatastorePath,
|
||||||
|
int sizeInMb, ManagedObjectReference morDs, int deviceNumber, int contextNumber) throws Exception {
|
||||||
|
|
||||||
|
VirtualDisk disk = new VirtualDisk();
|
||||||
|
|
||||||
|
VirtualDiskFlatVer2BackingInfo backingInfo = new VirtualDiskFlatVer2BackingInfo();
|
||||||
|
backingInfo.setDiskMode(VirtualDiskMode.persistent.toString());
|
||||||
|
backingInfo.setThinProvisioned(true);
|
||||||
|
backingInfo.setEagerlyScrub(false);
|
||||||
|
backingInfo.setDatastore(morDs);
|
||||||
|
backingInfo.setFileName(vmdkDatastorePath);
|
||||||
|
disk.setBacking(backingInfo);
|
||||||
|
|
||||||
if(controllerKey < 0)
|
if(controllerKey < 0)
|
||||||
controllerKey = vmMo.getIDEDeviceControllerKey();
|
controllerKey = vmMo.getIDEDeviceControllerKey();
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user