kvm: Enable PVLAN support on L2 networks (#4040)

This is an extention of #3732 for kvm.
This is restricted to ovs > 2.9.2
Since Xen uses ovs 2.6, pvlan is unsupported.
This also fixes issues of vms on the same pvlan unable to communicate if they're on the same host
This commit is contained in:
davidjumani 2020-08-20 15:46:34 +05:30 committed by GitHub
parent 400641b1cf
commit 3872bf1ff9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 636 additions and 135 deletions

View File

@ -811,7 +811,10 @@ public class Agent implements HandlerFactory, IAgentControl {
public void processReadyCommand(final Command cmd) {
final ReadyCommand ready = (ReadyCommand)cmd;
// Set human readable sizes;
NumbersUtil.enableHumanReadableSizes = ready.getEnableHumanReadableSizes();
Boolean humanReadable = ready.getEnableHumanReadableSizes();
if (humanReadable != null){
NumbersUtil.enableHumanReadableSizes = humanReadable;
}
s_logger.info("Processing agent ready command, agent id = " + ready.getHostId());
if (ready.getHostId() != null) {

View File

@ -34,6 +34,7 @@ public class PvlanSetupCommand extends Command {
private String dhcpIp;
private Type type;
private String networkTag;
private String pvlanType;
protected PvlanSetupCommand() {
}
@ -43,6 +44,7 @@ public class PvlanSetupCommand extends Command {
this.op = op;
this.primary = NetUtils.getPrimaryPvlanFromUri(uri);
this.isolated = NetUtils.getIsolatedPvlanFromUri(uri);
this.pvlanType = NetUtils.getPvlanTypeFromUri(uri);
this.networkTag = networkTag;
}
@ -116,4 +118,8 @@ public class PvlanSetupCommand extends Command {
public String getNetworkTag() {
return networkTag;
}
public String getPvlanType() {
return pvlanType;
}
}

View File

@ -3756,6 +3756,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
try {
result = plugNic(network, nicTO, vmTO, context, dest);
if (result) {
_userVmMgr.setupVmForPvlan(true, vm.getHostId(), nic);
s_logger.debug("Nic is plugged successfully for vm " + vm + " in network " + network + ". Vm is a part of network now");
final long isDefault = nic.isDefaultNic() ? 1 : 0;
// insert nic's Id into DB as resource_name
@ -3863,6 +3864,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
s_logger.debug("Un-plugging nic " + nic + " for vm " + vm + " from network " + network);
final boolean result = unplugNic(network, nicTO, vmTO, context, dest);
if (result) {
_userVmMgr.setupVmForPvlan(false, vm.getHostId(), nicProfile);
s_logger.debug("Nic is unplugged successfully for vm " + vm + " in network " + network);
final long isDefault = nic.isDefaultNic() ? 1 : 0;
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(),

View File

@ -16,7 +16,6 @@
// under the License.
package org.apache.cloudstack.engine.orchestration;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
@ -215,6 +214,7 @@ import com.cloud.vm.NicProfile;
import com.cloud.vm.NicVO;
import com.cloud.vm.ReservationContext;
import com.cloud.vm.ReservationContextImpl;
import com.cloud.vm.UserVmManager;
import com.cloud.vm.UserVmVO;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
@ -299,6 +299,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
RemoteAccessVpnDao _remoteAccessVpnDao;
@Inject
VpcVirtualNetworkApplianceService _routerService;
@Inject
UserVmManager _userVmMgr;
List<NetworkGuru> networkGurus;
@ -1792,6 +1794,11 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
s_logger.error("NetworkGuru " + guru + " prepareForMigration failed."); // XXX: Transaction error
}
}
if (network.getGuestType() == Network.GuestType.L2 && vm.getType() == VirtualMachine.Type.User) {
_userVmMgr.setupVmForPvlan(false, vm.getVirtualMachine().getHostId(), profile);
}
final List<Provider> providersToImplement = getNetworkProviders(network.getId());
for (final NetworkElement element : networkElements) {
if (providersToImplement.contains(element.getProvider())) {
@ -1912,6 +1919,11 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
if (guru instanceof NetworkMigrationResponder) {
((NetworkMigrationResponder)guru).commitMigration(nicSrc, network, src, src_context, dst_context);
}
if (network.getGuestType() == Network.GuestType.L2 && src.getType() == VirtualMachine.Type.User) {
_userVmMgr.setupVmForPvlan(true, src.getVirtualMachine().getHostId(), nicSrc);
}
final List<Provider> providersToImplement = getNetworkProviders(network.getId());
for (final NetworkElement element : networkElements) {
if (providersToImplement.contains(element.getProvider())) {
@ -1943,6 +1955,11 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
if (guru instanceof NetworkMigrationResponder) {
((NetworkMigrationResponder)guru).rollbackMigration(nicDst, network, dst, src_context, dst_context);
}
if (network.getGuestType() == Network.GuestType.L2 && src.getType() == VirtualMachine.Type.User) {
_userVmMgr.setupVmForPvlan(true, dst.getVirtualMachine().getHostId(), nicDst);
}
final List<Provider> providersToImplement = getNetworkProviders(network.getId());
for (final NetworkElement element : networkElements) {
if (providersToImplement.contains(element.getProvider())) {
@ -2498,6 +2515,12 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
} else {
uri = BroadcastDomainType.fromString(vlanIdFinal);
}
if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) {
throw new InvalidParameterValueException("Network with vlan " + vlanIdFinal +
" already exists or overlaps with other network pvlans in zone " + zoneId);
}
userNetwork.setBroadcastUri(uri);
if (!vlanIdFinal.equalsIgnoreCase(Vlan.UNTAGGED)) {
userNetwork.setBroadcastDomainType(BroadcastDomainType.Vlan);
@ -2508,7 +2531,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
if (vlanIdFinal.equalsIgnoreCase(Vlan.UNTAGGED)) {
throw new InvalidParameterValueException("Cannot support pvlan with untagged primary vlan!");
}
URI uri = NetUtils.generateUriForPvlan(vlanIdFinal, isolatedPvlan);
URI uri = NetUtils.generateUriForPvlan(vlanIdFinal, isolatedPvlan, isolatedPvlanType.toString());
if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString(), isolatedPvlanType).size() > 0) {
throw new InvalidParameterValueException("Network with primary vlan " + vlanIdFinal +
" and secondary vlan " + isolatedPvlan + " type " + isolatedPvlanType +

View File

@ -126,4 +126,6 @@ public interface NetworkDao extends GenericDao<NetworkVO, Long>, StateDao<State,
List<NetworkVO> listByAccountIdNetworkName(long accountId, String name);
List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri, Network.PVlanType pVlanType);
List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri);
}

View File

@ -740,6 +740,7 @@ public class NetworkDaoImpl extends GenericDaoBase<NetworkVO, Long>implements Ne
* - The requested exact PVLAN pair exists
* - The requested secondary VLAN ID is secondary VLAN ID of an existing PVLAN pair
* - The requested secondary VLAN ID is primary VLAN ID of an existing PVLAN pair
* - The requested primary VLAN ID is secondary VLAN ID of an existing PVLAN pair
*/
protected boolean isNetworkOverlappingRequestedPvlan(Integer existingPrimaryVlan, Integer existingSecondaryVlan, Network.PVlanType existingPvlanType,
Integer requestedPrimaryVlan, Integer requestedSecondaryVlan, Network.PVlanType requestedPvlanType) {
@ -749,6 +750,7 @@ public class NetworkDaoImpl extends GenericDaoBase<NetworkVO, Long>implements Ne
}
boolean exactMatch = existingPrimaryVlan.equals(requestedPrimaryVlan) && existingSecondaryVlan.equals(requestedSecondaryVlan);
boolean secondaryVlanUsed = requestedPvlanType != Network.PVlanType.Promiscuous && requestedSecondaryVlan.equals(existingPrimaryVlan) || requestedSecondaryVlan.equals(existingSecondaryVlan);
boolean primaryVlanUsed = existingPvlanType != Network.PVlanType.Promiscuous && requestedPrimaryVlan.equals(existingSecondaryVlan);
boolean isolatedMax = false;
boolean promiscuousMax = false;
if (requestedPvlanType == Network.PVlanType.Isolated && existingPrimaryVlan.equals(requestedPrimaryVlan) && existingPvlanType.equals(Network.PVlanType.Isolated)) {
@ -756,7 +758,12 @@ public class NetworkDaoImpl extends GenericDaoBase<NetworkVO, Long>implements Ne
} else if (requestedPvlanType == Network.PVlanType.Promiscuous && existingPrimaryVlan.equals(requestedPrimaryVlan) && existingPvlanType == Network.PVlanType.Promiscuous) {
promiscuousMax = true;
}
return exactMatch || secondaryVlanUsed || isolatedMax || promiscuousMax;
return exactMatch || secondaryVlanUsed || primaryVlanUsed || isolatedMax || promiscuousMax;
}
// True when a VLAN ID overlaps with an existing PVLAN primary or secondary ID
protected boolean isNetworkOverlappingRequestedPvlan(Integer existingPrimaryVlan, Integer existingSecondaryVlan, Integer requestedVlan) {
return requestedVlan.equals(existingPrimaryVlan) || requestedVlan.equals(existingSecondaryVlan);
}
protected Network.PVlanType getNetworkPvlanType(long networkId, List<Integer> existingPvlan) {
@ -770,6 +777,38 @@ public class NetworkDaoImpl extends GenericDaoBase<NetworkVO, Long>implements Ne
return existingPvlanType;
}
@Override
public List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri) {
final URI searchUri = BroadcastDomainType.fromString(broadcastUri);
if (!searchUri.getScheme().equalsIgnoreCase("vlan")) {
throw new CloudRuntimeException("VLAN requested but URI is not in the expected format: " + searchUri.toString());
}
final String searchRange = BroadcastDomainType.getValue(searchUri);
final List<Integer> searchVlans = UriUtils.expandVlanUri(searchRange);
final List<NetworkVO> overlappingNetworks = new ArrayList<>();
final SearchCriteria<NetworkVO> sc = PhysicalNetworkSearch.create();
sc.setParameters("physicalNetworkId", physicalNetworkId);
for (final NetworkVO network : listBy(sc)) {
if (network.getBroadcastUri() == null || !network.getBroadcastUri().getScheme().equalsIgnoreCase("pvlan")) {
continue;
}
// Ensure existing and proposed VLAN don't overlap
final String networkVlanRange = BroadcastDomainType.getValue(network.getBroadcastUri());
if (networkVlanRange == null || networkVlanRange.isEmpty()) {
continue;
}
List<Integer> existingPvlan = UriUtils.expandPvlanUri(networkVlanRange);
if (isNetworkOverlappingRequestedPvlan(existingPvlan.get(0), existingPvlan.get(1), searchVlans.get(0))) {
overlappingNetworks.add(network);
break;
}
}
return overlappingNetworks;
}
@Override
public List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri, Network.PVlanType pVlanType) {
final URI searchUri = BroadcastDomainType.fromString(broadcastUri);

View File

@ -798,14 +798,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
throw new ConfigurationException("Unable to find the router_proxy.sh");
}
_ovsPvlanDhcpHostPath = Script.findScript(networkScriptsDir, "ovs-pvlan-dhcp-host.sh");
_ovsPvlanDhcpHostPath = Script.findScript(networkScriptsDir, "ovs-pvlan-kvm-dhcp-host.sh");
if (_ovsPvlanDhcpHostPath == null) {
throw new ConfigurationException("Unable to find the ovs-pvlan-dhcp-host.sh");
throw new ConfigurationException("Unable to find the ovs-pvlan-kvm-dhcp-host.sh");
}
_ovsPvlanVmPath = Script.findScript(networkScriptsDir, "ovs-pvlan-vm.sh");
_ovsPvlanVmPath = Script.findScript(networkScriptsDir, "ovs-pvlan-kvm-vm.sh");
if (_ovsPvlanVmPath == null) {
throw new ConfigurationException("Unable to find the ovs-pvlan-vm.sh");
throw new ConfigurationException("Unable to find the ovs-pvlan-kvm-vm.sh");
}
String value = (String)params.get("developer");

View File

@ -19,22 +19,17 @@
package com.cloud.hypervisor.kvm.resource.wrapper;
import java.util.List;
import org.apache.log4j.Logger;
import org.joda.time.Duration;
import org.libvirt.Connect;
import org.libvirt.LibvirtException;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.PvlanSetupCommand;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef;
import com.cloud.resource.CommandWrapper;
import com.cloud.resource.ResourceWrapper;
import com.cloud.utils.script.Script;
@ResourceWrapper(handles = PvlanSetupCommand.class)
@ResourceWrapper(handles = PvlanSetupCommand.class)
public final class LibvirtPvlanSetupCommandWrapper extends CommandWrapper<PvlanSetupCommand, Answer, LibvirtComputingResource> {
private static final Logger s_logger = Logger.getLogger(LibvirtPvlanSetupCommandWrapper.class);
@ -43,66 +38,50 @@ public final class LibvirtPvlanSetupCommandWrapper extends CommandWrapper<PvlanS
public Answer execute(final PvlanSetupCommand command, final LibvirtComputingResource libvirtComputingResource) {
final String primaryPvlan = command.getPrimary();
final String isolatedPvlan = command.getIsolated();
final String pvlanType = "-" + command.getPvlanType();
final String op = command.getOp();
final String dhcpName = command.getDhcpName();
final String dhcpMac = command.getDhcpMac();
final String vmMac = command.getVmMac();
final String vmMac = command.getVmMac() == null ? dhcpMac : command.getVmMac();
final String dhcpIp = command.getDhcpIp();
boolean add = true;
String opr = "-A";
if (op.equals("delete")) {
opr = "-D";
add = false;
}
String result = null;
try {
final String guestBridgeName = libvirtComputingResource.getGuestBridgeName();
final Duration timeout = libvirtComputingResource.getTimeout();
if (command.getType() == PvlanSetupCommand.Type.DHCP) {
final String ovsPvlanDhcpHostPath = libvirtComputingResource.getOvsPvlanDhcpHostPath();
final Script script = new Script(ovsPvlanDhcpHostPath, timeout, s_logger);
final String guestBridgeName = libvirtComputingResource.getGuestBridgeName();
final Duration timeout = libvirtComputingResource.getTimeout();
if (add) {
final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper();
final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(dhcpName);
if (command.getType() == PvlanSetupCommand.Type.DHCP) {
final String ovsPvlanDhcpHostPath = libvirtComputingResource.getOvsPvlanDhcpHostPath();
final Script script = new Script(ovsPvlanDhcpHostPath, timeout, s_logger);
final List<InterfaceDef> ifaces = libvirtComputingResource.getInterfaces(conn, dhcpName);
final InterfaceDef guestNic = ifaces.get(0);
script.add(opr, "-b", guestBridgeName, "-p", primaryPvlan, "-i", isolatedPvlan, "-n", dhcpName, "-d", dhcpIp, "-m", dhcpMac, "-I",
guestNic.getDevName());
} else {
script.add(opr, "-b", guestBridgeName, "-p", primaryPvlan, "-i", isolatedPvlan, "-n", dhcpName, "-d", dhcpIp, "-m", dhcpMac);
}
script.add(opr, pvlanType, "-b", guestBridgeName, "-p", primaryPvlan, "-s", isolatedPvlan, "-m", dhcpMac,
"-d", dhcpIp);
result = script.execute();
result = script.execute();
if (result != null) {
s_logger.warn("Failed to program pvlan for dhcp server with mac " + dhcpMac);
return new Answer(command, false, result);
} else {
s_logger.info("Programmed pvlan for dhcp server with mac " + dhcpMac);
}
} else if (command.getType() == PvlanSetupCommand.Type.VM) {
final String ovsPvlanVmPath = libvirtComputingResource.getOvsPvlanVmPath();
final Script script = new Script(ovsPvlanVmPath, timeout, s_logger);
script.add(opr, "-b", guestBridgeName, "-p", primaryPvlan, "-i", isolatedPvlan, "-v", vmMac);
result = script.execute();
if (result != null) {
s_logger.warn("Failed to program pvlan for vm with mac " + vmMac);
return new Answer(command, false, result);
} else {
s_logger.info("Programmed pvlan for vm with mac " + vmMac);
}
if (result != null) {
s_logger.warn("Failed to program pvlan for dhcp server with mac " + dhcpMac);
} else {
s_logger.info("Programmed pvlan for dhcp server with mac " + dhcpMac);
}
} catch (final LibvirtException e) {
s_logger.error("Error whislt executing OVS Setup command! ==> " + e.getMessage());
return new Answer(command, false, e.getMessage());
}
// We run this even for DHCP servers since they're all vms after all
final String ovsPvlanVmPath = libvirtComputingResource.getOvsPvlanVmPath();
final Script script = new Script(ovsPvlanVmPath, timeout, s_logger);
script.add(opr, pvlanType, "-b", guestBridgeName, "-p", primaryPvlan, "-s", isolatedPvlan, "-m", vmMac);
result = script.execute();
if (result != null) {
s_logger.warn("Failed to program pvlan for vm with mac " + vmMac);
return new Answer(command, false, result);
} else {
s_logger.info("Programmed pvlan for vm with mac " + vmMac);
}
return new Answer(command, true, result);
}
}

View File

@ -4337,7 +4337,7 @@ public class LibvirtComputingResourceTest {
@Test
public void testPvlanSetupCommandDhcpAdd() {
final String op = "add";
final URI uri = URI.create("http://localhost");
final URI uri = URI.create("pvlan://200-p200");
final String networkTag = "/105";
final String dhcpName = "dhcp";
final String dhcpMac = "00:00:00:00";
@ -4345,46 +4345,24 @@ public class LibvirtComputingResourceTest {
final PvlanSetupCommand command = PvlanSetupCommand.createDhcpSetup(op, uri, networkTag, dhcpName, dhcpMac, dhcpIp);
final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class);
final Connect conn = Mockito.mock(Connect.class);
final String guestBridgeName = "br0";
when(libvirtComputingResource.getGuestBridgeName()).thenReturn(guestBridgeName);
when(libvirtComputingResource.getTimeout()).thenReturn(Duration.ZERO);
final String ovsPvlanDhcpHostPath = "/pvlan";
when(libvirtComputingResource.getOvsPvlanDhcpHostPath()).thenReturn(ovsPvlanDhcpHostPath);
when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
final List<InterfaceDef> ifaces = new ArrayList<InterfaceDef>();
final InterfaceDef nic = Mockito.mock(InterfaceDef.class);
ifaces.add(nic);
try {
when(libvirtUtilitiesHelper.getConnectionByVmName(dhcpName)).thenReturn(conn);
when(libvirtComputingResource.getInterfaces(conn, dhcpName)).thenReturn(ifaces);
} catch (final LibvirtException e) {
fail(e.getMessage());
}
final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance();
assertNotNull(wrapper);
final Answer answer = wrapper.execute(command, libvirtComputingResource);
assertFalse(answer.getResult());
verify(libvirtComputingResource, times(1)).getLibvirtUtilitiesHelper();
try {
verify(libvirtUtilitiesHelper, times(1)).getConnectionByVmName(dhcpName);
} catch (final LibvirtException e) {
fail(e.getMessage());
}
}
@Test
public void testPvlanSetupCommandVm() {
final String op = "add";
final URI uri = URI.create("http://localhost");
final URI uri = URI.create("pvlan://200-p200");
final String networkTag = "/105";
final String vmMac = "00:00:00:00";
@ -4404,52 +4382,10 @@ public class LibvirtComputingResourceTest {
assertFalse(answer.getResult());
}
@SuppressWarnings("unchecked")
@Test
public void testPvlanSetupCommandDhcpException() {
final String op = "add";
final URI uri = URI.create("http://localhost");
final String networkTag = "/105";
final String dhcpName = "dhcp";
final String dhcpMac = "00:00:00:00";
final String dhcpIp = "127.0.0.1";
final PvlanSetupCommand command = PvlanSetupCommand.createDhcpSetup(op, uri, networkTag, dhcpName, dhcpMac, dhcpIp);
final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class);
final String guestBridgeName = "br0";
when(libvirtComputingResource.getGuestBridgeName()).thenReturn(guestBridgeName);
when(libvirtComputingResource.getTimeout()).thenReturn(Duration.ZERO);
final String ovsPvlanDhcpHostPath = "/pvlan";
when(libvirtComputingResource.getOvsPvlanDhcpHostPath()).thenReturn(ovsPvlanDhcpHostPath);
when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
try {
when(libvirtUtilitiesHelper.getConnectionByVmName(dhcpName)).thenThrow(LibvirtException.class);
} catch (final LibvirtException e) {
fail(e.getMessage());
}
final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance();
assertNotNull(wrapper);
final Answer answer = wrapper.execute(command, libvirtComputingResource);
assertFalse(answer.getResult());
verify(libvirtComputingResource, times(1)).getLibvirtUtilitiesHelper();
try {
verify(libvirtUtilitiesHelper, times(1)).getConnectionByVmName(dhcpName);
} catch (final LibvirtException e) {
fail(e.getMessage());
}
}
@Test
public void testPvlanSetupCommandDhcpDelete() {
final String op = "delete";
final URI uri = URI.create("http://localhost");
final URI uri = URI.create("pvlan://200-p200");
final String networkTag = "/105";
final String dhcpName = "dhcp";
final String dhcpMac = "00:00:00:00";
@ -4457,15 +4393,12 @@ public class LibvirtComputingResourceTest {
final PvlanSetupCommand command = PvlanSetupCommand.createDhcpSetup(op, uri, networkTag, dhcpName, dhcpMac, dhcpIp);
final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class);
final String guestBridgeName = "br0";
when(libvirtComputingResource.getGuestBridgeName()).thenReturn(guestBridgeName);
when(libvirtComputingResource.getTimeout()).thenReturn(Duration.ZERO);
final String ovsPvlanDhcpHostPath = "/pvlan";
when(libvirtComputingResource.getOvsPvlanDhcpHostPath()).thenReturn(ovsPvlanDhcpHostPath);
when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance();
assertNotNull(wrapper);

View File

@ -0,0 +1,137 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#!/bin/bash
# We're trying to do the impossible here by allowing pvlan on kvm / xen. As only God can do the impossible, and we've got too much ego to
# admit that we can't, we're just hacking our way around it.
# We're pretty much crafting two vlan headers, one with the primary vlan and the other with the secondary and with a few fancy rules
# it managed to work. But take note that the'res no checking over here for secondary vlan overlap. That has to be handled while
# creating the pvlan!!
exec 2>&1
usage() {
printf "Usage: %s: (-A|-D) (-P/I/C) -b <bridge/switch> -p <primary vlan> -s <secondary vlan> -m <VM MAC> -d <DHCP IP> -h \n" $(basename $0) >&2
exit 2
}
br=
pri_vlan=
sec_vlan=
vm_mac=
dhcp_ip=
op=
type=
while getopts 'ADPICb:p:s:m:d:h' OPTION
do
case $OPTION in
A) op="add"
;;
D) op="del"
;;
P) type="P"
;;
I) type="I"
;;
C) type="C"
;;
b) br="$OPTARG"
;;
p) pri_vlan="$OPTARG"
;;
s) sec_vlan="$OPTARG"
;;
m) vm_mac="$OPTARG"
;;
d) dhcp_ip="$OPTARG"
;;
h) usage
exit 1
;;
esac
done
if [ -z "$op" ]
then
echo Missing operation pararmeter!
exit 1
fi
if [ -z "$type" ]
then
echo Missing pvlan type pararmeter!
exit 1
fi
if [ -z "$br" ]
then
echo Missing parameter bridge!
exit 1
fi
if [ -z "$vm_mac" ]
then
echo Missing parameter VM MAC!
exit 1
fi
if [ -z "$pri_vlan" ]
then
echo Missing parameter primary vlan!
exit 1
fi
if [ -z "$sec_vlan" ]
then
echo Missing parameter secondary vlan!
exit 1
fi
if [ -z "$dhcp_ip" ]
then
echo Missing parameter DHCP IP!
exit 1
fi
find_port() {
mac=`echo "$1" | sed -e 's/:/\\\:/g'`
port=`ovs-vsctl --column ofport find interface external_ids:attached-mac="$mac" | tr -d ' ' | cut -d ':' -f 2`
echo $port
}
ovs-vsctl set bridge $br protocols=OpenFlow10,OpenFlow11,OpenFlow12,OpenFlow13
ovs-vsctl set Open_vSwitch . other_config:vlan-limit=2
if [ "$op" == "add" ]
then
dhcp_port=$(find_port $vm_mac)
ovs-ofctl add-flow $br table=0,priority=200,arp,dl_vlan=$pri_vlan,nw_dst=$dhcp_ip,actions=strip_vlan,resubmit\(,1\)
ovs-ofctl add-flow $br table=1,priority=200,arp,dl_vlan=$sec_vlan,nw_dst=$dhcp_ip,actions=strip_vlan,output:$dhcp_port
ovs-ofctl add-flow $br table=0,priority=100,udp,dl_vlan=$pri_vlan,nw_dst=255.255.255.255,tp_dst=67,actions=strip_vlan,resubmit\(,1\)
ovs-ofctl add-flow $br table=1,priority=100,udp,dl_vlan=$sec_vlan,nw_dst=255.255.255.255,tp_dst=67,actions=strip_vlan,output:$dhcp_port
else
ovs-ofctl del-flows --strict $br table=0,priority=200,arp,dl_vlan=$pri_vlan,nw_dst=$dhcp_ip
ovs-ofctl del-flows --strict $br table=1,priority=200,arp,dl_vlan=$sec_vlan,nw_dst=$dhcp_ip
ovs-ofctl del-flows --strict $br table=0,priority=100,udp,dl_vlan=$pri_vlan,nw_dst=255.255.255.255,tp_dst=67
ovs-ofctl del-flows --strict $br table=1,priority=100,udp,dl_vlan=$sec_vlan,nw_dst=255.255.255.255,tp_dst=67
fi

View File

@ -0,0 +1,290 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#!/bin/bash
# We're trying to do the impossible here by allowing pvlan on kvm / xen. As only God can do the impossible, and we've got too much ego to
# admit that we can't, we're just hacking our way around it.
# We're pretty much crafting two vlan headers, one with the primary vlan and the other with the secondary and with a few fancy rules
# it managed to work. But take note that the'res no checking over here for secondary vlan overlap. That has to be handled while
# creating the pvlan!!
exec 2>&1
usage() {
printf "Usage: %s: (-A|-D) (-P/I/C) -b <bridge/switch> -p <primary vlan> -s <secondary vlan> -m <VM MAC> -h \n" $(basename $0) >&2
exit 2
}
br=
pri_vlan=
sec_vlan=
vm_mac=
op=
type=
while getopts 'ADPICb:p:s:m:h' OPTION
do
case $OPTION in
A) op="add"
;;
D) op="del"
;;
P) type="P"
;;
I) type="I"
;;
C) type="C"
;;
b) br="$OPTARG"
;;
p) pri_vlan="$OPTARG"
;;
s) sec_vlan="$OPTARG"
;;
m) vm_mac="$OPTARG"
;;
h) usage
exit 1
;;
esac
done
if [ -z "$op" ]
then
echo Missing operation pararmeter!
exit 1
fi
if [ -z "$type" ]
then
echo Missing pvlan type pararmeter!
exit 1
fi
if [ -z "$br" ]
then
echo Missing parameter bridge!
exit 1
fi
if [ -z "$vm_mac" ]
then
echo Missing parameter VM MAC!
exit 1
fi
if [ -z "$pri_vlan" ]
then
echo Missing parameter primary vlan!
exit 1
fi
if [ -z "$sec_vlan" ]
then
echo Missing parameter secondary vlan!
exit 1
fi
find_port() {
mac=`echo "$1" | sed -e 's/:/\\\:/g'`
port=`ovs-vsctl --column ofport find interface external_ids:attached-mac="$mac" | tr -d ' ' | cut -d ':' -f 2`
echo $port
}
find_port_group() {
ovs-ofctl -O OpenFlow13 dump-groups $br | grep group_id=$1, | sed -e 's/.*type=all,//g' -e 's/bucket=actions=//g' -e 's/resubmit(,1)//g' -e 's/strip_vlan,//g' -e 's/pop_vlan,//g' -e 's/output://g' -e 's/^,//g' -e 's/,$//g' -e 's/,,/,/g' -e 's/ //g'
}
# try to find the physical link to outside, only supports eth and em prefix now
trunk_port=`ovs-ofctl show $br | egrep "\((eth|em)[0-9]" | cut -d '(' -f 1|tr -d ' '`
vm_port=$(find_port $vm_mac)
# craft the vlan headers. Adding 4096 as in hex, it must be of the form 0x1XXX
pri_vlan_header=$((4096 + $pri_vlan))
sec_vlan_header=$((4096 + $sec_vlan))
# Get the groups for broadcast. Ensure we end the group id with ',' so that we wont accidentally match groupid 111 with 1110.
# We're using the header value for the pri vlan port group, as anything from a promiscuous device has to go to every device in the vlan.
# Since we're creating a separate group for just the promiscuous devices, adding 4096 so that it'll be unique. Hence we're restricted to 4096 vlans!
# Not a big deal because if you have vxlan, why do you even need pvlan!!
pri_vlan_ports=$(find_port_group $pri_vlan_header)
sec_vlan_ports=$(find_port_group $sec_vlan)
add_to_ports() {
if [ -z "$1" ]
then
# To ensure that we don't get trailing commas
echo "$2"
else
# Dont add it if it already exists
echo "$1" | grep -w -q "$2" && echo "$1" && return
echo "$2,$1"
fi
}
del_from_ports() {
# Delete when only, begining, middle and end of string
echo "$1" | sed -e "s/^$2$//g" -e "s/^$2,//g" -e "s/,$2$//g" -e "s/,$2,/,/g"
}
mod_group() {
# Ensure that we don't delete the prom port group, because if we do, the rules that have it go away!
actions=`echo "$2" | sed -e 's/,/,bucket=actions=/g'`
if [ "$1" == "$pri_vlan" ]
then
actions=`echo "$2" | sed -e 's/,/,bucket=actions=strip_vlan,output:/g'`
if [ -z "$2" ]
then
ovs-ofctl -O OpenFlow13 mod-group --may-create $br group_id=$1,type=all,bucket=resubmit\(,1\)
else
ovs-ofctl -O OpenFlow13 mod-group --may-create $br group_id=$1,type=all,bucket=resubmit\(,1\),bucket=actions=strip_vlan,output:$actions
fi
return
fi
if [ -z "$2" ]
then
ovs-ofctl -O OpenFlow13 del-groups $br group_id=$1
else
ovs-ofctl -O OpenFlow13 mod-group --may-create $br group_id=$1,type=all,bucket=actions=$actions
fi
}
cleanup_flows() {
ovs-ofctl del-flows $br --strict table=0,priority=70,dl_vlan=$pri_vlan,dl_dst=ff:ff:ff:ff:ff:ff
ovs-ofctl del-flows $br --strict table=1,priority=70,dl_vlan=$pri_vlan,dl_dst=ff:ff:ff:ff:ff:ff
ovs-ofctl -O OpenFlow13 del-groups $br group_id=$pri_vlan
}
# Allow the neccessary protocols and QinQ
ovs-vsctl set bridge $br protocols=OpenFlow10,OpenFlow11,OpenFlow12,OpenFlow13
ovs-vsctl set Open_vSwitch . other_config:vlan-limit=2
# So that we're friendly to non pvlan devices
ovs-ofctl add-flow $br priority=0,actions=NORMAL
if [ "$op" == "add" ]
then
# From our pri vlan
if [ "$type" == "P" ]
then
ovs-ofctl add-flow $br table=0,priority=70,dl_vlan=$pri_vlan,dl_dst=$vm_mac,actions=strip_vlan,strip_vlan,output:$vm_port
else
ovs-ofctl add-flow $br table=0,priority=70,dl_vlan=$pri_vlan,dl_dst=$vm_mac,actions=strip_vlan,resubmit\(,1\)
fi
# Accept from promiscuous
ovs-ofctl add-flow $br table=1,priority=70,dl_vlan=$pri_vlan,dl_dst=$vm_mac,actions=strip_vlan,output:$vm_port
# From others in our own community
if [ "$type" == "C" ]
then
ovs-ofctl add-flow $br table=1,priority=70,dl_vlan=$sec_vlan,dl_dst=$vm_mac,actions=strip_vlan,output:$vm_port
fi
# Allow only dhcp to isolated vm
if [ "$type" == "I" ]
then
ovs-ofctl add-flow $br table=1,priority=70,udp,dl_vlan=$sec_vlan,dl_dst=$vm_mac,tp_src=67,actions=strip_vlan,output:$vm_port
fi
# Security101
ovs-ofctl add-flow $br table=1,priority=0,actions=drop
# If the dest isn't on our switch send it out
ovs-ofctl add-flow $br table=0,priority=60,dl_vlan=$pri_vlan,dl_src=$vm_mac,actions=output:$trunk_port
# QinQ the packet. Outter header is the primary vlan and inner is the secondary
ovs-ofctl add-flow -O OpenFlow13 $br table=0,priority=50,vlan_tci=0x0000,dl_src=$vm_mac,actions=push_vlan:0x8100,set_field:$sec_vlan_header-\>vlan_vid,push_vlan:0x8100,set_field:$pri_vlan_header-\>vlan_vid,resubmit:$trunk_port
# BROADCASTS
# Create the respective groups
# pri_vlan_ports are the list of ports of all iso & comm dev for a give pvlan
if [ "$type" != "P" ]
then
pri_vlan_ports=$(add_to_ports "$pri_vlan_ports" "$vm_port")
mod_group $pri_vlan_header $pri_vlan_ports
fi
# sec_vlan_ports are the list of ports for a given secondary pvlan
sec_vlan_ports=$(add_to_ports "$sec_vlan_ports" "$vm_port")
mod_group $sec_vlan $sec_vlan_ports
# Ensure we have the promiscuous port group because if we don't, it'll fail to create the following rule
prom_ports=$(find_port_group $pri_vlan)
mod_group $pri_vlan $prom_ports
# From a device on this switch. Pass it to the trunk port and process it ourselves for other devices on the switch.
ovs-ofctl add-flow $br table=0,priority=300,dl_vlan=$pri_vlan,dl_src=$vm_mac,dl_dst=ff:ff:ff:ff:ff:ff,actions=output:$trunk_port,strip_vlan,group:$pri_vlan
# Got a packet from the trunk port from out pri vlan, pass it to pri_vlan_group which sends the packet out to the promiscuous devices as well as passes it onto table 1
ovs-ofctl add-flow $br table=0,priority=70,dl_vlan=$pri_vlan,dl_dst=ff:ff:ff:ff:ff:ff,actions=strip_vlan,group:$pri_vlan
# From a promiscuous device, so send it to all community and isolated devices on this switch. Passed to all promiscuous devices in the prior step ^^
ovs-ofctl add-flow $br table=1,priority=70,dl_vlan=$pri_vlan,dl_dst=ff:ff:ff:ff:ff:ff,actions=strip_vlan,group:$pri_vlan_header
# Since it's from a community, gotta braodcast it to all community devices
if [ "$type" == "C" ]
then
ovs-ofctl add-flow $br table=1,priority=70,dl_vlan=$sec_vlan,dl_dst=ff:ff:ff:ff:ff:ff,actions=strip_vlan,group:$sec_vlan
fi
# Allow only dhcp form isolated router to isolated vm
if [ "$type" == "I" ]
then
ovs-ofctl add-flow $br table=1,priority=70,udp,dl_vlan=$sec_vlan,tp_src=67,dl_dst=ff:ff:ff:ff:ff:ff,actions=strip_vlan,group:$sec_vlan
fi
else
# Delete whatever we've added that's vm specific
ovs-ofctl del-flows $br --strict table=0,priority=70,dl_vlan=$pri_vlan,dl_dst=$vm_mac
# Need to ge the vmport from the rules as it's already been removed from the switch
vm_port=`ovs-ofctl dump-flows $br | grep "priority=70,dl_vlan=$pri_vlan,dl_dst=$vm_mac" | tr ':' '\n' | tail -n 1`
ovs-ofctl del-flows $br --strict table=1,priority=70,dl_vlan=$pri_vlan,dl_dst=$vm_mac
if [ "$type" == "C" ]
then
ovs-ofctl del-flows $br --strict table=1,priority=70,dl_vlan=$sec_vlan,dl_dst=$vm_mac
fi
if [ "$type" == "I" ]
then
ovs-ofctl del-flows $br --strict table=1,priority=70,udp,dl_vlan=$sec_vlan,dl_dst=$vm_mac,tp_src=67
fi
ovs-ofctl del-flows $br --strict table=0,priority=60,dl_vlan=$pri_vlan,dl_src=$vm_mac
ovs-ofctl del-flows $br --strict table=0,priority=50,vlan_tci=0x0000,dl_src=$vm_mac
# For some ovs versions
ovs-ofctl del-flows $br --strict table=0,priority=50,vlan_tci=0x0000/0x1fff,dl_src=$vm_mac
# Remove the port from the groups
pri_vlan_ports=$(del_from_ports "$pri_vlan_ports" "$vm_port")
mod_group $pri_vlan_header $pri_vlan_ports
sec_vlan_ports=$(del_from_ports "$sec_vlan_ports" "$vm_port")
mod_group $sec_vlan $sec_vlan_ports
# Remove vm specific rules
ovs-ofctl del-flows $br --strict table=0,priority=300,dl_vlan=$pri_vlan,dl_src=$vm_mac,dl_dst=ff:ff:ff:ff:ff:ff
# If the vm is going to be migrated but not yet removed. Remove the rules if it's the only vm in the vlan
res1=`ovs-vsctl --column _uuid find port tag=$pri_vlan | wc -l`
res2=`find_port $vm_mac | wc -l`
if [ "$res1" -eq 1 ] && [ "$res2" -eq 1 ]
then
cleanup_flows
fi
# If no more vms exist on this host, clear up all the rules
result=`ovs-vsctl find port tag=$pri_vlan`
if [ -z "$result" ]
then
cleanup_flows
fi
fi

View File

@ -52,4 +52,14 @@ public class NetworkDaoTest extends TestCase {
Assert.assertFalse(dao.isNetworkOverlappingRequestedPvlan(existingPrimaryVlan, existingSecondaryVlan, Network.PVlanType.Community,
existingPrimaryVlan, requestedVlan, Network.PVlanType.Community));
}
public void testNetworkOverlappingVlanPvlanTrue() {
Assert.assertTrue(dao.isNetworkOverlappingRequestedPvlan(existingPrimaryVlan, existingSecondaryVlan, existingPrimaryVlan));
Assert.assertTrue(dao.isNetworkOverlappingRequestedPvlan(existingPrimaryVlan, existingSecondaryVlan, existingSecondaryVlan));
}
public void testNetworkOverlappingVlanPvlanFalse() {
Assert.assertFalse(dao.isNetworkOverlappingRequestedPvlan(existingPrimaryVlan, existingSecondaryVlan, requestedVlan));
}
}

View File

@ -250,4 +250,9 @@ public class MockNetworkDaoImpl extends GenericDaoBase<NetworkVO, Long> implemen
public List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri, Network.PVlanType pVlanType) {
return null;
}
@Override
public List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri) {
return null;
}
}

View File

@ -1571,8 +1571,22 @@ class TestPrivateVlansL2Networks(cloudstackTestCase):
isDvSwitch = True
break
supported = isVmware and isDvSwitch
cls.vmwareHypervisorDvSwitchesForGuestTrafficNotPresent = not supported
# Supported hypervisor = KVM using OVS
isKVM = cls.hypervisor.lower() in ["kvm"]
isOVSEnabled = False
hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][0].__dict__
if isKVM :
# Test only if all the hosts use OVS
grepCmd = 'grep "network.bridge.type=openvswitch" /etc/cloudstack/agent/agent.properties'
hosts = list_hosts(cls.apiclient, type='Routing', hypervisor='kvm')
if len(hosts) > 0 :
isOVSEnabled = True
for host in hosts :
isOVSEnabled = isOVSEnabled and len(SshClient(host.ipaddress, port=22, user=hostConfig["username"],
passwd=hostConfig["password"]).execute(grepCmd)) != 0
supported = isVmware and isDvSwitch or isKVM and isOVSEnabled
cls.unsupportedHardware = not supported
cls._cleanup = []
@ -1730,7 +1744,7 @@ class TestPrivateVlansL2Networks(cloudstackTestCase):
return vm_ip, eth_device
@attr(tags=["advanced", "advancedns", "smoke", "pvlan"], required_hardware="true")
@skipTestIf("vmwareHypervisorDvSwitchesForGuestTrafficNotPresent")
@skipTestIf("unsupportedHardware")
def test_l2_network_pvlan_connectivity(self):
try:
vm_community1_one = self.deploy_vm_multiple_nics("vmcommunity1one", self.l2_pvlan_community1)
@ -1788,6 +1802,7 @@ class TestPrivateVlansL2Networks(cloudstackTestCase):
# Isolated PVLAN checks
same_isolated = self.is_vm_l2_isolated_from_dest(vm_isolated1, vm_isolated1_eth, vm_isolated2_ip)
isolated_to_community_isolated = self.is_vm_l2_isolated_from_dest(vm_isolated1, vm_isolated1_eth, vm_community1_one_ip)
isolated_to_promiscuous_isolated = self.is_vm_l2_isolated_from_dest(vm_isolated1, vm_isolated1_eth, vm_promiscuous1_ip)
self.assertTrue(
same_isolated,
@ -1797,6 +1812,10 @@ class TestPrivateVlansL2Networks(cloudstackTestCase):
isolated_to_community_isolated,
"VMs on isolated PVLANs must be isolated on layer 2 to Vms on community PVLAN"
)
self.assertFalse(
isolated_to_promiscuous_isolated,
"VMs on isolated PVLANs must not be isolated on layer 2 to Vms on promiscuous PVLAN",
)
# Promiscuous PVLAN checks
same_promiscuous = self.is_vm_l2_isolated_from_dest(vm_promiscuous1, vm_promiscuous1_eth, vm_promiscuous2_ip)

View File

@ -630,7 +630,7 @@ public class UriUtils {
if (Strings.isNullOrEmpty(pvlanRange)) {
return expandedVlans;
}
String[] parts = pvlanRange.split("-i");
String[] parts = pvlanRange.split("-\\w");
expandedVlans.add(Integer.parseInt(parts[0]));
expandedVlans.add(Integer.parseInt(parts[1]));
return expandedVlans;

View File

@ -1471,6 +1471,24 @@ public class NetUtils {
return URI.create("pvlan://" + primaryVlan + "-i" + isolatedPvlan);
}
public static URI generateUriForPvlan(final String primaryVlan, final String isolatedPvlan, final String isolatedPvlanType) {
// Defaulting to isolated for backward compatibility
if (isolatedPvlan.length() < 1) {
return generateUriForPvlan(primaryVlan, isolatedPvlan);
}
char type = isolatedPvlanType.charAt(0);
switch(type) {
case 'c':
case 'C':
return URI.create("pvlan://" + primaryVlan + "-c" + isolatedPvlan);
case 'p':
case 'P':
return URI.create("pvlan://" + primaryVlan + "-p" + primaryVlan);
default :
return generateUriForPvlan(primaryVlan, isolatedPvlan);
}
}
public static String getPrimaryPvlanFromUri(final URI uri) {
final String[] vlans = uri.getHost().split("-");
if (vlans.length < 1) {
@ -1488,6 +1506,31 @@ public class NetUtils {
if (vlan.startsWith("i")) {
return vlan.replace("i", " ").trim();
}
if (vlan.startsWith("p")) {
return vlan.replace("p", " ").trim();
}
if (vlan.startsWith("c")) {
return vlan.replace("c", " ").trim();
}
}
return null;
}
public static String getPvlanTypeFromUri(final URI uri) {
final String[] vlans = uri.getHost().split("-");
if (vlans.length < 2) {
return null;
}
for (final String vlan : vlans) {
if (vlan.startsWith("i")) {
return "I";
}
if (vlan.startsWith("p")) {
return "P";
}
if (vlan.startsWith("c")) {
return "C";
}
}
return null;
}

View File

@ -249,10 +249,20 @@ public class NetUtilsTest {
}
@Test
public void testGenerateUriForPvlan() {
public void testGenerateUriForIsolatedPvlan() {
assertEquals("pvlan://123-i456", NetUtils.generateUriForPvlan("123", "456").toString());
}
@Test
public void testGenerateUriForCommunityPvlan() {
assertEquals("pvlan://123-c456", NetUtils.generateUriForPvlan("123", "456", "Community").toString());
}
@Test
public void testGenerateUriForPromiscuousPvlan() {
assertEquals("pvlan://123-p123", NetUtils.generateUriForPvlan("123", "123", "promiscuous").toString());
}
@Test
public void testGetPrimaryPvlanFromUri() {
assertEquals("123", NetUtils.getPrimaryPvlanFromUri(NetUtils.generateUriForPvlan("123", "456")));