apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

CLOUDSTACK-768: ACL on private gateway

Browse Source
This commit is contained in:
Jayapal Reddy 2013-05-13 19:24:23 +05:30 committed by Kishan Kavala
parent bcc320f8b0
commit 2d950e2858
34 changed files with 1116 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/src/com/cloud/network/NetworkService.java
View File

@ -163,7 +163,7 @@ public interface NetworkService {
* @throws ResourceAllocationException
*/
Network createPrivateNetwork(String networkName, String displayText, long physicalNetworkId, String vlan,
String startIp, String endIP, String gateway, String netmask, long networkOwnerId, Long vpcId, Boolean sourceNat)
String startIp, String endIP, String gateway, String netmask, long networkOwnerId, Long vpcId, Boolean sourceNat)
throws ResourceAllocationException, ConcurrentOperationException, InsufficientCapacityException;
/* Requests an IP address for the guest nic */

2
api/src/com/cloud/network/element/VpcProvider.java
View File

@ -52,4 +52,6 @@ public interface VpcProvider extends NetworkElement{
boolean deletePrivateGateway(PrivateGateway privateGateway) throws ConcurrentOperationException, ResourceUnavailableException;
boolean applyStaticRoutes(Vpc vpc, List<StaticRouteProfile> routes) throws ResourceUnavailableException;
boolean applyACLItemsToPrivateGw(PrivateGateway gateway) throws ResourceUnavailableException;
}

10
api/src/com/cloud/network/vpc/NetworkACLService.java
View File

@ -122,4 +122,14 @@ public interface NetworkACLService {
NetworkACLItem updateNetworkACLItem(Long id, String protocol, List<String> sourceCidrList, NetworkACLItem.TrafficType trafficType,
String action, Integer number, Integer sourcePortStart, Integer sourcePortEnd,
Integer icmpCode, Integer icmpType) throws ResourceUnavailableException;
/**
* Associates ACL with specified Network
* @param aclId
* @param privateGatewayId
* @return
* @throws ResourceUnavailableException
*/
boolean replaceNetworkACLonPrivateGw(long aclId, long privateGatewayId) throws ResourceUnavailableException;
}

5
api/src/com/cloud/network/vpc/VpcGateway.java
View File

@ -81,4 +81,9 @@ public interface VpcGateway extends Identity, ControlledEntity, InternalIdentity
* @return
*/
boolean getSourceNat();
/**
* @return
*/
long getNetworkACLId();
}

3
api/src/com/cloud/network/vpc/VpcService.java
View File

@ -172,13 +172,14 @@ public interface VpcService {
* @param netmask
* @param gatewayOwnerId
* @param isSourceNat
* @param aclId
* @return
* @throws InsufficientCapacityException
* @throws ConcurrentOperationException
* @throws ResourceAllocationException
*/
public PrivateGateway createVpcPrivateGateway(long vpcId, Long physicalNetworkId, String vlan, String ipAddress,
String gateway, String netmask, long gatewayOwnerId, Boolean isSourceNat) throws ResourceAllocationException,
String gateway, String netmask, long gatewayOwnerId, Boolean isSoruceNat, Long aclId) throws ResourceAllocationException,
ConcurrentOperationException, InsufficientCapacityException;
/**

13
api/src/org/apache/cloudstack/api/command/admin/vpc/CreatePrivateGatewayCmd.java
View File

@ -23,6 +23,7 @@ import org.apache.cloudstack.api.BaseAsyncCmd;
import org.apache.cloudstack.api.BaseAsyncCreateCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.NetworkACLResponse;
import org.apache.cloudstack.api.response.PhysicalNetworkResponse;
import org.apache.cloudstack.api.response.PrivateGatewayResponse;
import org.apache.cloudstack.api.response.VpcResponse;
@ -74,6 +75,11 @@ public class CreatePrivateGatewayCmd extends BaseAsyncCreateCmd {
" 'false': sourcenat is not supported")
private Boolean isSourceNat;
@Parameter(name=ApiConstants.ACL_ID, type=CommandType.UUID, entityType = NetworkACLResponse.class,
required=false, description="the ID of the network ACL")
private Long aclId;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
@ -109,6 +115,11 @@ public class CreatePrivateGatewayCmd extends BaseAsyncCreateCmd {
return true;
}
public Long getAclId() {
return aclId;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@ -123,7 +134,7 @@ public class CreatePrivateGatewayCmd extends BaseAsyncCreateCmd {
PrivateGateway result = null;
try {
result = _vpcService.createVpcPrivateGateway(getVpcId(), getPhysicalNetworkId(),
getVlan(), getStartIp(), getGateway(), getNetmask(), getEntityOwnerId(), getIsSourceNat());
getVlan(), getStartIp(), getGateway(), getNetmask(), getEntityOwnerId(), getIsSourceNat(), getAclId());
} catch (InsufficientCapacityException ex){
s_logger.info(ex);
s_logger.trace(ex);

32
api/src/org/apache/cloudstack/api/command/user/network/ReplaceNetworkACLListCmd.java
View File

@ -26,10 +26,11 @@ import com.cloud.user.UserContext;
import org.apache.cloudstack.api.*;
import org.apache.cloudstack.api.response.NetworkACLResponse;
import org.apache.cloudstack.api.response.NetworkResponse;
import org.apache.cloudstack.api.response.PrivateGatewayResponse;
import org.apache.cloudstack.api.response.SuccessResponse;
import org.apache.log4j.Logger;
@APICommand(name = "replaceNetworkACLList", description="Replaces ACL associated with a Network", responseObject=SuccessResponse.class)
@APICommand(name = "replaceNetworkACLList", description="Replaces ACL associated with a Network or private gateway", responseObject=SuccessResponse.class)
public class ReplaceNetworkACLListCmd extends BaseAsyncCmd {
public static final Logger s_logger = Logger.getLogger(ReplaceNetworkACLListCmd.class.getName());
private static final String s_name = "replacenetworkacllistresponse";
@ -43,8 +44,12 @@ public class ReplaceNetworkACLListCmd extends BaseAsyncCmd {
private long aclId;
@Parameter(name=ApiConstants.NETWORK_ID, type=CommandType.UUID, entityType = NetworkResponse.class,
required=true, description="the ID of the network")
private long networkId;
description="the ID of the network")
private Long networkId;
@Parameter(name=ApiConstants.GATEWAY_ID, type=CommandType.UUID, entityType = PrivateGatewayResponse.class,
description="the ID of the private gateway")
private Long privateGatewayId;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
@ -54,10 +59,14 @@ public class ReplaceNetworkACLListCmd extends BaseAsyncCmd {
return aclId;
}
public long getNetworkId(){
public Long getNetworkId(){
return networkId;
}
public Long getPrivateGatewayId() {
return privateGatewayId;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@ -84,8 +93,21 @@ public class ReplaceNetworkACLListCmd extends BaseAsyncCmd {
@Override
public void execute() throws ResourceUnavailableException {
if (getNetworkId() == null && getPrivateGatewayId() == null) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Network id and private gateway can't be null at the same time");
}
if (getNetworkId() != null && getPrivateGatewayId() != null) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Network id and private gateway can't be passed at the same time");
}
UserContext.current().setEventDetails("Network ACL Id: " + aclId);
boolean result = _networkACLService.replaceNetworkACL(aclId, networkId);
boolean result = false;
if (getPrivateGatewayId() != null) {
result = _networkACLService.replaceNetworkACLonPrivateGw(aclId, privateGatewayId);
} else {
result = _networkACLService.replaceNetworkACL(aclId, networkId);
}
if (result) {
SuccessResponse response = new SuccessResponse(getCommandName());

9
api/src/org/apache/cloudstack/api/response/PrivateGatewayResponse.java
View File

@ -80,6 +80,10 @@ public class PrivateGatewayResponse extends BaseResponse implements ControlledEn
private Boolean sourceNat;
@SerializedName(ApiConstants.ACL_ID) @Param(description = "ACL Id set for private gateway")
private String aclId;
@Override
public String getObjectId() {
return this.id;
@ -154,6 +158,11 @@ public class PrivateGatewayResponse extends BaseResponse implements ControlledEn
this.sourceNat = sourceNat;
}
public void setAclId(String aclId) {
this.aclId = aclId;
}
}

2
core/src/com/cloud/agent/api/routing/NetworkElementCommand.java
View File

@ -32,6 +32,8 @@ public abstract class NetworkElementCommand extends Command {
public static final String ROUTER_GUEST_IP = "router.guest.ip";
public static final String ZONE_NETWORK_TYPE = "zone.network.type";
public static final String GUEST_BRIDGE = "guest.bridge";
public static final String VPC_PRIVATE_GATEWAY = "vpc.gateway.private";
protected NetworkElementCommand() {
super();

15
core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java
View File

@ -892,12 +892,17 @@ public class VirtualRoutingResource implements Manager {
}
public String assignNetworkACL(final String routerIP, final String dev,
final String routerGIP, final String netmask, final String rule){
final String routerGIP, final String netmask, final String rule, String privateGw){
String args = " -d " + dev;
args += " -i " + routerGIP;
args += " -m " + netmask;
args += " -a " + rule;
return routerProxy("vpc_acl.sh", routerIP, args);
if (privateGw != null) {
args += " -a " + rule;
return routerProxy("vpc_privategw_acl.sh", routerIP, args);
} else {
args += " -i " + routerGIP;
args += " -m " + netmask;
args += " -a " + rule;
return routerProxy("vpc_acl.sh", routerIP, args);
}
}
public String assignSourceNat(final String routerIP, final String pubIP, final String dev) {

17
engine/schema/src/com/cloud/network/vpc/VpcGatewayVO.java
View File

@ -87,6 +87,11 @@ public class VpcGatewayVO implements VpcGateway {
@Column(name="source_nat")
boolean sourceNat;
@Column(name="network_acl_id")
long networkACLId;
protected VpcGatewayVO(){
this.uuid = UUID.randomUUID().toString();
}
@ -106,7 +111,7 @@ public class VpcGatewayVO implements VpcGateway {
* @param sourceNat
*/
public VpcGatewayVO(String ip4Address, Type type, Long vpcId, long zoneId, Long networkId, String vlanTag,
String gateway, String netmask, long accountId, long domainId, boolean sourceNat) {
String gateway, String netmask, long accountId, long domainId, boolean sourceNat, long networkACLId) {
this.ip4Address = ip4Address;
this.type = type;
this.vpcId = vpcId;
@ -120,6 +125,8 @@ public class VpcGatewayVO implements VpcGateway {
this.domainId = domainId;
this.state = State.Creating;
this.sourceNat = sourceNat;
this.networkACLId = networkACLId;
}
@Override
@ -203,4 +210,12 @@ public class VpcGatewayVO implements VpcGateway {
return this.sourceNat;
}
public void setNetworkACLId(long networkACLId) {
this.networkACLId = networkACLId;
}
@Override
public long getNetworkACLId() {
return networkACLId;
}
}

2
engine/schema/src/com/cloud/network/vpc/dao/VpcGatewayDao.java
View File

@ -23,4 +23,6 @@ import com.cloud.utils.db.GenericDao;
public interface VpcGatewayDao extends GenericDao<VpcGatewayVO, Long>{
VpcGatewayVO getPrivateGatewayForVpc(long vpcId);
VpcGatewayVO getVpnGatewayForVpc(long vpcId);
Long getNetworkAclIdForPrivateIp(long vpcId, long networkId, String ipaddr);
}

18
engine/schema/src/com/cloud/network/vpc/dao/VpcGatewayDaoImpl.java
View File

@ -37,6 +37,8 @@ public class VpcGatewayDaoImpl extends GenericDaoBase<VpcGatewayVO, Long> implem
AllFieldsSearch = createSearchBuilder();
AllFieldsSearch.and("vpcId", AllFieldsSearch.entity().getVpcId(), SearchCriteria.Op.EQ);
AllFieldsSearch.and("type", AllFieldsSearch.entity().getType(), SearchCriteria.Op.EQ);
AllFieldsSearch.and("networkid", AllFieldsSearch.entity().getNetworkId(), SearchCriteria.Op.EQ);
AllFieldsSearch.and("ipaddress", AllFieldsSearch.entity().getIp4Address(), SearchCriteria.Op.EQ);
AllFieldsSearch.done();
}
@ -59,4 +61,20 @@ public class VpcGatewayDaoImpl extends GenericDaoBase<VpcGatewayVO, Long> implem
return findOneBy(sc);
}
@Override
public Long getNetworkAclIdForPrivateIp (long vpcId, long networkId, String ipaddr) {
SearchCriteria<VpcGatewayVO> sc = AllFieldsSearch.create();
sc.setParameters("vpcId", vpcId);
sc.setParameters("networkid", networkId);
sc.setParameters("ipaddress", ipaddr);
VpcGateway vpcGateway = findOneBy(sc);
if (vpcGateway != null) {
return vpcGateway.getNetworkACLId();
} else {
return null;
}
}
}

224
patches/systemvm/debian/config/opt/cloud/bin/vpc_privategw_acl.sh Executable file
View File

@ -0,0 +1,224 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# vpc_privategw_acl.sh_rule.sh -- allow/block some ports / protocols to vm instances
# @VERSION@
source /root/func.sh
lock="biglock"
locked=$(getLockFile $lock)
if [ "$locked" != "1" ]
then
exit 1
fi
usage() {
printf "Usage: %s: -a <public ip address:protocol:startport:endport:sourcecidrs> \n" $(basename $0) >&2
printf "sourcecidrs format: cidr1-cidr2-cidr3-...\n"
}
#set -x
#FIXME: eating up the error code during execution of iptables
acl_switch_to_new() {
sudo iptables -D FORWARD -o $dev -j _ACL_INBOUND_$dev 2>/dev/null
sudo iptables-save | grep "\-j _ACL_INBOUND_$dev" | grep "\-A" | while read rule;
do
rule1=$(echo $rule | sed 's/\_ACL_INBOUND/ACL_INBOUND/')
sudo iptables $rule1
rule2=$(echo $rule | sed 's/\-A/\-D/')
sudo iptables $rule2
done
sudo iptables -F _ACL_INBOUND_$dev 2>/dev/null
sudo iptables -X _ACL_INBOUND_$dev 2>/dev/null
sudo iptables -t mangle -F _ACL_OUTBOUND_$dev 2>/dev/null
sudo iptables -t mangle -D PREROUTING -m state --state NEW -i $dev -j _ACL_OUTBOUND_$dev 2>/dev/null
sudo iptables -t mangle -X _ACL_OUTBOUND_$dev 2>/dev/null
}
acl_remove_backup() {
sudo iptables -F _ACL_INBOUND_$dev 2>/dev/null
sudo iptables -D FORWARD -o $dev -j _ACL_INBOUND_$dev 2>/dev/null
sudo iptables -X _ACL_INBOUND_$dev 2>/dev/null
sudo iptables -t mangle -F _ACL_OUTBOUND_$dev 2>/dev/null
sudo iptables -t mangle -D PREROUTING -m state --state NEW -i $dev -j _ACL_OUTBOUND_$dev 2>/dev/null
sudo iptables -t mangle -X _ACL_OUTBOUND_$dev 2>/dev/null
}
acl_remove() {
sudo iptables -F ACL_INBOUND_$dev 2>/dev/null
sudo iptables -D FORWARD -o $dev -j ACL_INBOUND_$dev 2>/dev/null
sudo iptables -X ACL_INBOUND_$dev 2>/dev/null
sudo iptables -t mangle -F ACL_OUTBOUND_$dev 2>/dev/null
sudo iptables -t mangle -D PREROUTING -m state --state NEW -i $dev -j ACL_OUTBOUND_$dev 2>/dev/null
sudo iptables -t mangle -X ACL_OUTBOUND_$dev 2>/dev/null
}
acl_restore() {
acl_remove
sudo iptables -E _ACL_INBOUND_$dev ACL_INBOUND_$dev 2>/dev/null
sudo iptables -t mangle -E _ACL_OUTBOUND_$dev ACL_OUTBOUND_$dev 2>/dev/null
}
acl_save() {
acl_remove_backup
sudo iptables -E ACL_INBOUND_$dev _ACL_INBOUND_$dev 2>/dev/null
sudo iptables -t mangle -E ACL_OUTBOUND_$dev _ACL_OUTBOUND_$dev 2>/dev/null
}
acl_chain_for_guest_network () {
acl_save
# inbound
sudo iptables -N ACL_INBOUND_$dev 2>/dev/null
# drop if no rules match (this will be the last rule in the chain)
sudo iptables -A ACL_INBOUND_$dev -j DROP 2>/dev/null
sudo iptables -A FORWARD -o $dev -j ACL_INBOUND_$dev 2>/dev/null
# outbound
sudo iptables -t mangle -N ACL_OUTBOUND_$dev 2>/dev/null
sudo iptables -t mangle -A PREROUTING -m state --state NEW -i $dev -j ACL_OUTBOUND_$dev 2>/dev/null
}
acl_entry_for_guest_network() {
local rule=$1
local ttype=$(echo $rule | cut -d: -f1)
local prot=$(echo $rule | cut -d: -f2)
local sport=$(echo $rule | cut -d: -f3)
local eport=$(echo $rule | cut -d: -f4)
local cidrs=$(echo $rule | cut -d: -f5 | sed 's/-/ /g')
local action=$(echo $rule | cut -d: -f6)
if [ "$sport" == "0" -a "$eport" == "0" ]
then
DPORT=""
else
DPORT="--dport $sport:$eport"
fi
logger -t cloud "$(basename $0): enter apply acl rules on private gateway interface : $dev, inbound:$inbound:$prot:$sport:$eport:$cidrs"
# note that rules are inserted after the RELATED,ESTABLISHED rule
# but before the DROP rule
for lcidr in $cidrs
do
[ "$prot" == "reverted" ] && continue;
if [ "$prot" == "icmp" ]
then
typecode="$sport/$eport"
[ "$eport" == "-1" ] && typecode="$sport"
[ "$sport" == "-1" ] && typecode="any"
if [ "$ttype" == "Ingress" ]
then
sudo iptables -I ACL_INBOUND_$dev -p $prot -s $lcidr \
--icmp-type $typecode -j $action
else
let egress++
sudo iptables -t mangle -I ACL_OUTBOUND_$dev -p $prot -d $lcidr \
--icmp-type $typecode -j $action
fi
else
if [ "$ttype" == "Ingress" ]
then
sudo iptables -I ACL_INBOUND_$dev -p $prot -s $lcidr \
$DPORT -j $action
else
let egress++
sudo iptables -t mangle -I ACL_OUTBOUND_$dev -p $prot -d $lcidr \
$DPORT -j $action
fi
fi
result=$?
[ $result -gt 0 ] &&
logger -t cloud "Error adding iptables entry for private gateway interface : $dev,inbound:$inbound:$prot:$sport:$eport:$cidrs" &&
break
done
logger -t cloud "$(basename $0): exit apply acl rules for private gw interface : $dev"
return $result
}
dflag=0
gflag=0
aflag=0
rules=""
rules_list=""
dev=""
while getopts 'd:a:' OPTION
do
case $OPTION in
d) dflag=1
dev="$OPTARG"
;;
a) aflag=1
rules="$OPTARG"
;;
?) usage
unlock_exit 2 $lock $locked
;;
esac
done
if [ "$dflag$aflag" != "11" ]
then
usage
unlock_exit 2 $lock $locked
fi
if [ -n "$rules" ]
then
rules_list=$(echo $rules | cut -d, -f1- --output-delimiter=" ")
fi
# rule format
# protocal:sport:eport:cidr
#-a tcp:80:80:0.0.0.0/0::tcp:220:220:0.0.0.0/0:,172.16.92.44:tcp:222:222:192.168.10.0/24-75.57.23.0/22-88.100.33.1/32
# if any entry is reverted , entry will be in the format <ip>:reverted:0:0:0
# example : 172.16.92.44:tcp:80:80:0.0.0.0/0:ACCEPT:,172.16.92.44:tcp:220:220:0.0.0.0/0:DROP,200.1.1.2:reverted:0:0:0
success=0
acl_chain_for_guest_network
egress=0
for r in $rules_list
do
acl_entry_for_guest_network $r
success=$?
if [ $success -gt 0 ]
then
logger -t cloud "$(basename $0): failure to apply acl rules on private gateway interface : $dev"
break
else
logger -t cloud "$(basename $0): successful in applying acl rules on private gateway interface : $dev"
fi
done
if [ $success -gt 0 ]
then
logger -t cloud "$(basename $0): restoring from backup on private gateway interface : $dev"
acl_restore
else
logger -t cloud "$(basename $0): deleting backup on private gateway interface : $dev"
if [ $egress -eq 0 ]
then
sudo iptables -t mangle -A ACL_OUTBOUND_$dev -j ACCEPT 2>/dev/null
else
sudo iptables -t mangle -A ACL_OUTBOUND_$dev -j DROP 2>/dev/null
fi
acl_switch_to_new
fi
unlock_exit $success $lock $locked

3
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View File

@ -1643,6 +1643,7 @@ ServerResource {
Connect conn;
String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME);
String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
String privateGw = cmd.getAccessDetail(NetworkElementCommand.VPC_PRIVATE_GATEWAY);
try {
conn = LibvirtConnection.getConnectionByVmName(routerName);
@ -1660,7 +1661,7 @@ ServerResource {
String rule = sb.toString();
String result = _virtRouterResource.assignNetworkACL(routerIp,
dev, nic.getIp(), netmask, rule);
dev, nic.getIp(), netmask, rule, privateGw);
if (result != null) {
for (int i=0; i < results.length; i++) {

37
plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java
View File

@ -1159,6 +1159,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME);
String privateGw = cmd.getAccessDetail(NetworkElementCommand.VPC_PRIVATE_GATEWAY);
String routerIp = getRouterSshControlIp(cmd);
String[] results = new String[cmd.getRules().length];
@ -1177,19 +1178,37 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
NicTO nic = cmd.getNic();
int ethDeviceNum = findRouterEthDeviceIndex(routerName, routerIp, nic.getMac());
String args = "";
args += " -d " + "eth" + ethDeviceNum;
args += " -i " + nic.getIp();
args += " -m " + Long.toString(NetUtils.getCidrSize(nic.getNetmask()));
args += " -a " + sb.toString();
Pair<Boolean, String> result;
Pair<Boolean, String> result = SshHelper.sshExecute(routerIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null,
"/opt/cloud/bin/vpc_acl.sh " + args);
if (privateGw != null) {
s_logger.debug("Private gateway configuration is set");
args += " -d " + "eth" + ethDeviceNum;
args += " -a " + sb.toString();
result = SshHelper.sshExecute(routerIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null,
"/opt/cloud/bin/vpc_privategw_acl.sh " + args);
if (!result.first()) {
String msg = "SetNetworkACLAnswer on domain router " + routerIp + " failed. message: " + result.second();
s_logger.error(msg);
if (!result.first()) {
String msg = "SetNetworkACLAnswer on domain router " + routerIp + " failed. message: " + result.second();
s_logger.error(msg);
}
return new SetNetworkACLAnswer(cmd, false, results);
} else {
args="";
args += " -d " + "eth" + ethDeviceNum;
args += " -i " + nic.getIp();
args += " -m " + Long.toString(NetUtils.getCidrSize(nic.getNetmask()));
args += " -a " + sb.toString();
result = SshHelper.sshExecute(routerIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null,
"/opt/cloud/bin/vpc_acl.sh " + args);
if (!result.first()) {
String msg = "SetNetworkACLAnswer on domain router " + routerIp + " failed. message: " + result.second();
s_logger.error(msg);
return new SetNetworkACLAnswer(cmd, false, results);
}
}
return new SetNetworkACLAnswer(cmd, true, results);

47
plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java
View File

@ -2326,7 +2326,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
result = callHostPlugin(conn, "vmops", "routerProxy", "args", snatArgs);
if (result == null || result.isEmpty()) {
throw new InternalErrorException("Xen plugin \"vcp_privateGateway\" failed.");
throw new InternalErrorException("Xen plugin \"vpc_privateGateway\" failed.");
}
}
@ -8153,6 +8153,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
Connection conn = getConnection();
String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME);
String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
String privateGw = cmd.getAccessDetail(NetworkElementCommand.VPC_PRIVATE_GATEWAY);
try {
VM router = getVM(conn, routerName);
String [][] rules = cmd.generateFwRules();
@ -8163,20 +8165,41 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
sb.append(aclRules[i]).append(',');
}
if (privateGw != null) {
s_logger.debug("Private gateway configuration is set");
}
NicTO nic = cmd.getNic();
VIF vif = getVifByMac(conn, router, nic.getMac());
String args = "vpc_acl.sh " + routerIp;
args += " -d " + "eth" + vif.getDevice(conn);
args += " -i " + nic.getIp();
args += " -m " + Long.toString(NetUtils.getCidrSize(nic.getNetmask()));
args += " -a " + sb.toString();
callResult = callHostPlugin(conn, "vmops", "routerProxy", "args", args);
if (callResult == null || callResult.isEmpty()) {
//FIXME - in the future we have to process each rule separately; now we temporarily set every rule to be false if single rule fails
for (int i=0; i < results.length; i++) {
results[i] = "Failed";
if (privateGw != null) {
s_logger.debug("Private gateway configuration is set");
String args = "vpc_privategw_acl.sh " + routerIp;
args += " -d " + "eth" + vif.getDevice(conn);
args += " -a " + sb.toString();
callResult = callHostPlugin(conn, "vmops", "routerProxy", "args", args);
if (callResult == null || callResult.isEmpty()) {
//FIXME - in the future we have to process each rule separately; now we temporarily set every rule to be false if single rule fails
for (int i=0; i < results.length; i++) {
results[i] = "Failed";
}
return new SetNetworkACLAnswer(cmd, false, results);
}
} else {
String args = "vpc_acl.sh " + routerIp;
args += " -d " + "eth" + vif.getDevice(conn);
args += " -i " + nic.getIp();
args += " -m " + Long.toString(NetUtils.getCidrSize(nic.getNetmask()));
args += " -a " + sb.toString();
callResult = callHostPlugin(conn, "vmops", "routerProxy", "args", args);
if (callResult == null || callResult.isEmpty()) {
//FIXME - in the future we have to process each rule separately; now we temporarily set every rule to be false if single rule fails
for (int i=0; i < results.length; i++) {
results[i] = "Failed";
}
return new SetNetworkACLAnswer(cmd, false, results);
}
return new SetNetworkACLAnswer(cmd, false, results);
}
return new SetNetworkACLAnswer(cmd, true, results);
} catch (Exception e) {

5
server/src/com/cloud/api/ApiResponseHelper.java
View File

@ -3163,6 +3163,11 @@ public class ApiResponseHelper implements ResponseGenerator {
response.setState(result.getState().toString());
response.setSourceNat(result.getSourceNat());
NetworkACL acl = ApiDBUtils.findByNetworkACLId(result.getNetworkACLId());
if (acl != null) {
response.setAclId(acl.getUuid());
}
response.setObjectName("privategateway");

50
server/src/com/cloud/network/element/VpcVirtualRouterElement.java
View File

@ -25,7 +25,9 @@ import java.util.Set;
import javax.ejb.Local;
import javax.inject.Inject;
import com.cloud.network.dao.NetworkDao;
import com.cloud.network.vpc.*;
import com.cloud.network.vpc.dao.VpcGatewayDao;
import org.apache.log4j.Logger;
import com.cloud.dc.DataCenter;
@ -48,7 +50,6 @@ import com.cloud.network.dao.Site2SiteVpnGatewayDao;
import com.cloud.network.router.VirtualRouter;
import com.cloud.network.router.VirtualRouter.Role;
import com.cloud.network.router.VpcVirtualNetworkApplianceManager;
import com.cloud.network.rules.FirewallRule;
import com.cloud.offering.NetworkOffering;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.DomainRouterVO;
@ -75,6 +76,12 @@ public class VpcVirtualRouterElement extends VirtualRouterElement implements Vpc
IPAddressDao _ipAddressDao;
@Inject
NetworkModel _ntwkModel;
@Inject
NetworkDao _networkDao;
@Inject
VpcGatewayDao _vpcGatewayDao;
@Inject
NetworkACLItemDao _networkACLItemDao;
private static final Map<Service, Map<Capability, String>> capabilities = setCapabilities();
@ -334,7 +341,18 @@ public class VpcVirtualRouterElement extends VirtualRouterElement implements Vpc
VirtualRouter router = routers.get(0);
return _vpcRouterMgr.setupPrivateGateway(gateway, router);
if ( _vpcRouterMgr.setupPrivateGateway(gateway, router) ) {
try {
if (!applyACLItemsToPrivateGw(gateway)) {
s_logger.debug ("Failed to apply network acl id "+ gateway.getNetworkACLId() + " on gateway ");
return false;
}
} catch (Exception ex) {
s_logger.debug ("Failed to apply network acl id "+ gateway.getNetworkACLId() + " on gateway ");
return false;
}
}
return true;
}
@Override
@ -395,8 +413,8 @@ public class VpcVirtualRouterElement extends VirtualRouterElement implements Vpc
return true;
}
if (!_vpcRouterMgr.applyNetworkACLs(config, rules, routers)) {
throw new CloudRuntimeException("Failed to apply firewall rules in network " + config.getId());
if (!_vpcRouterMgr.applyNetworkACLs(config, rules, routers, false)) {
throw new CloudRuntimeException("Failed to apply network acl rules in network " + config.getId());
} else {
return true;
}
@ -427,6 +445,30 @@ public class VpcVirtualRouterElement extends VirtualRouterElement implements Vpc
}
}
@Override
public boolean applyACLItemsToPrivateGw(PrivateGateway gateway) throws ResourceUnavailableException {
VpcGatewayVO vpcGatewayVo = _vpcGatewayDao.findById(gateway.getId());
List<NetworkACLItemVO> rules = _networkACLItemDao.listByACL(vpcGatewayVo.getNetworkACLId());
Network config = _networkDao.findById(gateway.getNetworkId());
boolean isPrivateGateway = true;
List<DomainRouterVO> routers = _vpcRouterMgr.getVpcRouters(gateway.getVpcId());
if (routers == null || routers.isEmpty()) {
s_logger.debug("Virtual router element doesn't need to apply network acl rules on the backend; virtual " +
"router doesn't exist in the network " + config.getId());
return true;
}
if (!_vpcRouterMgr.applyNetworkACLs(config, rules, routers, isPrivateGateway)) {
throw new CloudRuntimeException("Failed to apply network acl in network " + config.getId());
} else {
return true;
}
}
@Override
public boolean startSite2SiteVpn(Site2SiteVpnConnection conn) throws ResourceUnavailableException {
Site2SiteVpnGateway vpnGw = _vpnGatewayDao.findById(conn.getVpnGatewayId());

4
server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManager.java
View File

@ -48,13 +48,15 @@ public interface VpcVirtualNetworkApplianceManager extends VirtualNetworkApplian
ResourceUnavailableException;
/**
*
* @param network
* @param rules
* @param routers
* @param privateGateway
* @return
* @throws ResourceUnavailableException
*/
boolean applyNetworkACLs(Network network, List<? extends NetworkACLItem> rules, List<? extends VirtualRouter> routers)
boolean applyNetworkACLs(Network network, List<? extends NetworkACLItem> rules, List<? extends VirtualRouter> routers, boolean privateGateway)
throws ResourceUnavailableException;
/**

68
server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java
View File

@ -27,7 +27,24 @@ import java.util.TreeSet;
import javax.ejb.Local;
import javax.inject.Inject;
import com.cloud.network.vpc.*;
import com.cloud.network.vpc.NetworkACLItem;
import com.cloud.network.vpc.NetworkACLItemDao;
import com.cloud.network.vpc.NetworkACLItemVO;
import com.cloud.network.vpc.NetworkACLManager;
import com.cloud.network.vpc.PrivateGateway;
import com.cloud.network.vpc.PrivateIpAddress;
import com.cloud.network.vpc.PrivateIpVO;
import com.cloud.network.vpc.StaticRoute;
import com.cloud.network.vpc.StaticRouteProfile;
import com.cloud.network.vpc.Vpc;
import com.cloud.network.vpc.VpcGateway;
import com.cloud.network.vpc.VpcManager;
import com.cloud.network.vpc.VpcVO;
import com.cloud.network.vpc.dao.PrivateIpDao;
import com.cloud.network.vpc.dao.StaticRouteDao;
import com.cloud.network.vpc.dao.VpcDao;
import com.cloud.network.vpc.dao.VpcGatewayDao;
import com.cloud.network.vpc.dao.VpcOfferingDao;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@ -91,13 +108,6 @@ import com.cloud.network.dao.Site2SiteCustomerGatewayVO;
import com.cloud.network.dao.Site2SiteVpnConnectionDao;
import com.cloud.network.dao.Site2SiteVpnGatewayDao;
import com.cloud.network.dao.Site2SiteVpnGatewayVO;
import com.cloud.network.rules.FirewallRule;
import com.cloud.network.rules.FirewallRule.Purpose;
import com.cloud.network.rules.FirewallRuleVO;
import com.cloud.network.vpc.dao.PrivateIpDao;
import com.cloud.network.vpc.dao.StaticRouteDao;
import com.cloud.network.vpc.dao.VpcDao;
import com.cloud.network.vpc.dao.VpcOfferingDao;
import com.cloud.network.vpn.Site2SiteVpnManager;
import com.cloud.offering.NetworkOffering;
import com.cloud.user.Account;
@ -152,6 +162,10 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian
FirewallRulesDao _firewallDao;
@Inject
Site2SiteVpnManager _s2sVpnMgr;
@Inject
VpcGatewayDao _vpcGatewayDao;
@Inject
NetworkACLItemDao _networkACLItemDao;
@Override
public List<DomainRouterVO> deployVirtualRouterInVpc(Vpc vpc, DeployDestination dest, Account owner,
@ -695,7 +709,7 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian
}
@Override
public boolean applyNetworkACLs(Network network, final List<? extends NetworkACLItem> rules, List<? extends VirtualRouter> routers)
public boolean applyNetworkACLs(Network network, final List<? extends NetworkACLItem> rules, List<? extends VirtualRouter> routers, final boolean isPrivateGateway)
throws ResourceUnavailableException {
if (rules == null || rules.isEmpty()) {
s_logger.debug("No network ACLs to be applied for network " + network.getId());
@ -704,21 +718,21 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian
return applyRules(network, routers, "network acls", false, null, false, new RuleApplier() {
@Override
public boolean execute(Network network, VirtualRouter router) throws ResourceUnavailableException {
return sendNetworkACLs(router, rules, network.getId());
return sendNetworkACLs(router, rules, network.getId(), isPrivateGateway);
}
});
}
protected boolean sendNetworkACLs(VirtualRouter router, List<? extends NetworkACLItem> rules, long guestNetworkId)
protected boolean sendNetworkACLs(VirtualRouter router, List<? extends NetworkACLItem> rules, long guestNetworkId, boolean isPrivateGateway)
throws ResourceUnavailableException {
Commands cmds = new Commands(OnError.Continue);
createNetworkACLsCommands(rules, router, cmds, guestNetworkId);
createNetworkACLsCommands(rules, router, cmds, guestNetworkId, isPrivateGateway);
return sendCommandsToRouter(router, cmds);
}
private void createNetworkACLsCommands(List<? extends NetworkACLItem> rules, VirtualRouter router, Commands cmds,
long guestNetworkId) {
long guestNetworkId, boolean privateGateway) {
List<NetworkACLTO> rulesTO = null;
String guestVlan = null;
Network guestNtwk = _networkDao.findById(guestNetworkId);
@ -746,6 +760,10 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian
cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId());
cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString());
if (privateGateway) {
cmd.setAccessDetail(NetworkElementCommand.VPC_PRIVATE_GATEWAY, String.valueOf(VpcGateway.Type.Private));
}
cmds.addCommand(cmd);
}
@ -854,7 +872,18 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian
List<PrivateIpAddress> privateIps = new ArrayList<PrivateIpAddress>(1);
privateIps.add(ip);
createVpcAssociatePrivateIPCommands(router, privateIps, cmds, true);
}
Long privateGwAclId = _vpcGatewayDao.getNetworkAclIdForPrivateIp(ipVO.getVpcId(), ipVO.getNetworkId(), ipVO.getIpAddress());
if (privateGwAclId != null) {
//set network acl on private gateway
List<NetworkACLItemVO> networkACLs = _networkACLItemDao.listByACL(privateGwAclId);
s_logger.debug("Found " + networkACLs.size() + " network ACLs to apply as a part of VPC VR " + router
+ " start for private gateway ip = " + ipVO.getIpAddress());
createNetworkACLsCommands(networkACLs, router, cmds, ipVO.getNetworkId(), true);
}
}
}
} catch (Exception ex) {
s_logger.warn("Failed to add router " + router + " to network due to exception ", ex);
@ -924,7 +953,7 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian
s_logger.debug("Found " + networkACLs.size() + " network ACLs to apply as a part of VPC VR " + router
+ " start for guest network id=" + guestNetworkId);
if (!networkACLs.isEmpty()) {
createNetworkACLsCommands(networkACLs, router, cmds, guestNetworkId);
createNetworkACLsCommands(networkACLs, router, cmds, guestNetworkId, false);
}
}
}
@ -1020,11 +1049,16 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian
s_logger.warn("Failed to release private ip for gateway " + gateway + " on router " + router);
return false;
}
//revoke network acl on the private gateway.
if (!_networkACLMgr.revokeACLItemsForPrivateGw(gateway)) {
s_logger.debug("Failed to delete network acl items on " + gateway +" from router " + router);
return false;
}
s_logger.debug("Removing router " + router + " from private network " + privateNetwork + " as a part of delete private gateway");
result = result && _itMgr.removeVmFromNetwork(router, privateNetwork, null);
s_logger.debug("Private gateawy " + gateway + " is removed from router " + router);
return result;
}

13
server/src/com/cloud/network/vpc/NetworkACLManager.java
View File

@ -139,4 +139,17 @@ public interface NetworkACLManager{
NetworkACLItem updateNetworkACLItem(Long id, String protocol, List<String> sourceCidrList, NetworkACLItem.TrafficType trafficType,
String action, Integer number, Integer sourcePortStart, Integer sourcePortEnd,
Integer icmpCode, Integer icmpType) throws ResourceUnavailableException;
/**
* Associates acl with a network and applies the ACLItems
* @param acl
* @param gateway
* @return
*/
boolean replaceNetworkACLForPrivateGw(NetworkACL acl, PrivateGateway gateway) throws ResourceUnavailableException;
boolean revokeACLItemsForPrivateGw(PrivateGateway gateway) throws ResourceUnavailableException;
boolean applyACLToPrivateGw(PrivateGateway gateway) throws ResourceUnavailableException;
}

70
server/src/com/cloud/network/vpc/NetworkACLManagerImpl.java
View File

@ -25,8 +25,10 @@ import com.cloud.network.NetworkModel;
import com.cloud.network.dao.NetworkDao;
import com.cloud.network.dao.NetworkVO;
import com.cloud.network.element.NetworkACLServiceProvider;
import com.cloud.network.element.VpcProvider;
import com.cloud.network.vpc.NetworkACLItem.State;
import com.cloud.network.vpc.dao.NetworkACLDao;
import com.cloud.network.vpc.dao.VpcGatewayDao;
import com.cloud.tags.dao.ResourceTagDao;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
@ -40,6 +42,7 @@ import org.springframework.stereotype.Component;
import javax.ejb.Local;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.List;
@ -66,6 +69,10 @@ public class NetworkACLManagerImpl extends ManagerBase implements NetworkACLMana
NetworkModel _networkModel;
@Inject
NetworkDao _networkDao;
@Inject
VpcGatewayDao _vpcGatewayDao;
@Inject
NetworkModel _ntwkModel;
@Override
public NetworkACL createNetworkACL(String name, String description, long vpcId) {
@ -113,6 +120,17 @@ public class NetworkACLManagerImpl extends ManagerBase implements NetworkACLMana
return _networkACLDao.remove(acl.getId());
}
@Override
public boolean replaceNetworkACLForPrivateGw(NetworkACL acl, PrivateGateway gateway) throws ResourceUnavailableException {
VpcGatewayVO vpcGatewayVo = _vpcGatewayDao.findById(gateway.getId());
vpcGatewayVo.setNetworkACLId(acl.getId());
if (_vpcGatewayDao.update(vpcGatewayVo.getId(),vpcGatewayVo)) {
return applyACLToPrivateGw(gateway);
}
return false;
}
@Override
public boolean replaceNetworkACL(NetworkACL acl, NetworkVO network) throws ResourceUnavailableException {
network.setNetworkACLId(acl.getId());
@ -225,6 +243,36 @@ public class NetworkACLManagerImpl extends ManagerBase implements NetworkACLMana
return success;
}
@Override
public boolean revokeACLItemsForPrivateGw(PrivateGateway gateway) throws ResourceUnavailableException {
List<NetworkACLItemVO> aclItems = _networkACLItemDao.listByACL(gateway.getNetworkACLId());
if (aclItems.isEmpty()) {
s_logger.debug("Found no network ACL Items for private gateway id=" + gateway.getId());
return true;
}
if (s_logger.isDebugEnabled()) {
s_logger.debug("Releasing " + aclItems.size() + " Network ACL Items for private gateway id=" + gateway.getId());
}
for (NetworkACLItemVO aclItem : aclItems) {
// Mark all Network ACLs rules as Revoke, but don't update in DB
if (aclItem.getState() == State.Add || aclItem.getState() == State.Active) {
aclItem.setState(State.Revoke);
}
}
boolean success = applyACLItemsToPrivateGw(gateway, aclItems);
if (s_logger.isDebugEnabled() && success) {
s_logger.debug("Successfully released Network ACLs for private gateway id=" + gateway.getId() + " and # of rules now = "
+ aclItems.size());
}
return success;
}
@Override
public List<NetworkACLItemVO> listNetworkACLItems(long guestNtwkId) {
Network network = _networkMgr.getNetwork(guestNtwkId);
@ -236,6 +284,28 @@ public class NetworkACLManagerImpl extends ManagerBase implements NetworkACLMana
_networkACLItemDao.remove(rule.getId());
}
@Override
public boolean applyACLToPrivateGw(PrivateGateway gateway) throws ResourceUnavailableException {
VpcGatewayVO vpcGatewayVO = _vpcGatewayDao.findById(gateway.getId());
List<NetworkACLItemVO> rules = _networkACLItemDao.listByACL(vpcGatewayVO.getNetworkACLId());
return applyACLItemsToPrivateGw(gateway, rules);
}
private boolean applyACLItemsToPrivateGw(PrivateGateway gateway, List<NetworkACLItemVO> rules) throws ResourceUnavailableException {
List<VpcProvider> vpcElements = null;
vpcElements = new ArrayList<VpcProvider>();
vpcElements.add((VpcProvider)_ntwkModel.getElementImplementingProvider(Network.Provider.VPCVirtualRouter.getName()));
if (vpcElements == null) {
throw new CloudRuntimeException("Failed to initialize vpc elements");
}
for (VpcProvider provider: vpcElements){
return provider.applyACLItemsToPrivateGw(gateway);
}
return false;
}
@Override
public boolean applyACLToNetwork(long networkId) throws ResourceUnavailableException {
Network network = _networkDao.findById(networkId);

47
server/src/com/cloud/network/vpc/NetworkACLServiceImpl.java
View File

@ -24,6 +24,7 @@ import com.cloud.network.Networks;
import com.cloud.network.dao.NetworkDao;
import com.cloud.network.dao.NetworkVO;
import com.cloud.network.vpc.dao.NetworkACLDao;
import com.cloud.network.vpc.dao.VpcGatewayDao;
import com.cloud.projects.Project.ListProjectResourcesCriteria;
import com.cloud.server.ResourceTag.TaggedResourceType;
import com.cloud.tags.ResourceTagVO;
@ -65,8 +66,6 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
@Inject
NetworkModel _networkMgr;
@Inject
VpcManager _vpcMgr;
@Inject
ResourceTagDao _resourceTagDao;
@Inject
NetworkACLDao _networkACLDao;
@ -78,6 +77,10 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
NetworkDao _networkDao;
@Inject
NetworkACLManager _networkAclMgr;
@Inject
VpcGatewayDao _vpcGatewayDao;
@Inject
VpcManager _vpcMgr;
@Override
public NetworkACL createNetworkACL(String name, String description, long vpcId) {
@ -150,6 +153,46 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
_accountMgr.checkAccess(caller, null, true, vpc);
return _networkAclMgr.deleteNetworkACL(acl);
}
@Override
public boolean replaceNetworkACLonPrivateGw(long aclId, long privateGatewayId) throws ResourceUnavailableException {
Account caller = UserContext.current().getCaller();
VpcGateway gateway = _vpcGatewayDao.findById(privateGatewayId);
if (gateway == null) {
throw new InvalidParameterValueException("Unable to find specified private gateway");
}
VpcGatewayVO vo = _vpcGatewayDao.findById(privateGatewayId);
if (vo.getState() != VpcGateway.State.Ready) {
throw new InvalidParameterValueException("Gateway is not in Ready state");
}
NetworkACL acl = _networkACLDao.findById(aclId);
if(acl == null){
throw new InvalidParameterValueException("Unable to find specified NetworkACL");
}
if (gateway.getVpcId() == null) {
throw new InvalidParameterValueException("Unable to find specified vpc id");
}
if(aclId != NetworkACL.DEFAULT_DENY || aclId != NetworkACL.DEFAULT_ALLOW) {
Vpc vpc = _vpcMgr.getVpc(acl.getVpcId());
if(vpc == null){
throw new InvalidParameterValueException("Unable to find Vpc associated with the NetworkACL");
}
_accountMgr.checkAccess(caller, null, true, vpc);
if(gateway.getVpcId() != acl.getVpcId()){
throw new InvalidParameterValueException("private gateway: "+privateGatewayId+" and ACL: "+aclId+" do not belong to the same VPC");
}
}
PrivateGateway privateGateway = _vpcMgr.getVpcPrivateGateway(privateGatewayId);
_accountMgr.checkAccess(caller, null, true, privateGateway);
return _networkAclMgr.replaceNetworkACLForPrivateGw(acl, privateGateway);
}
@Override
public boolean replaceNetworkACL(long aclId, long networkId) throws ResourceUnavailableException {

6
server/src/com/cloud/network/vpc/PrivateGatewayProfile.java
View File

@ -105,4 +105,10 @@ public class PrivateGatewayProfile implements PrivateGateway {
public boolean getSourceNat() {
return vpcGateway.getSourceNat();
}
@Override
public long getNetworkACLId() {
return vpcGateway.getNetworkACLId();
}
}

17
server/src/com/cloud/network/vpc/VpcManagerImpl.java
View File

@ -87,6 +87,7 @@ import com.cloud.network.vpc.dao.VpcGatewayDao;
import com.cloud.network.vpc.dao.VpcOfferingDao;
import com.cloud.network.vpc.dao.VpcOfferingServiceMapDao;
import com.cloud.network.vpc.dao.VpcServiceMapDao;
import com.cloud.network.vpc.dao.NetworkACLDao;
import com.cloud.network.vpn.Site2SiteVpnManager;
import com.cloud.offering.NetworkOffering;
import com.cloud.offerings.NetworkOfferingServiceMapVO;
@ -180,6 +181,8 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis
DataCenterDao _dcDao;
@Inject
ConfigurationServer _configServer;
@Inject
NetworkACLDao _networkAclDao;
private final ScheduledExecutorService _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("VpcChecker"));
private List<VpcProvider> vpcElements = null;
@ -1315,7 +1318,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis
@DB
@ActionEvent(eventType = EventTypes.EVENT_PRIVATE_GATEWAY_CREATE, eventDescription = "creating vpc private gateway", create=true)
public PrivateGateway createVpcPrivateGateway(long vpcId, Long physicalNetworkId, String vlan, String ipAddress,
String gateway, String netmask, long gatewayOwnerId, Boolean isSourceNat) throws ResourceAllocationException,
String gateway, String netmask, long gatewayOwnerId, Boolean isSourceNat, Long aclId) throws ResourceAllocationException,
ConcurrentOperationException, InsufficientCapacityException {
//Validate parameters
@ -1342,10 +1345,18 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis
String networkName = "vpc-" + vpc.getName() + "-privateNetwork";
Network privateNtwk = _ntwkSvc.createPrivateNetwork(networkName, networkName, physicalNetworkId,
vlan, ipAddress, null, gateway, netmask, gatewayOwnerId, vpcId, isSourceNat);
long networkAclId = NetworkACL.DEFAULT_DENY;
if (aclId != null) {
if ( _networkAclDao.findById(aclId) == null) {
throw new InvalidParameterValueException("Invalid network acl id passed ");
}
networkAclId = aclId;
}
//2) create gateway entry
VpcGatewayVO gatewayVO = new VpcGatewayVO(ipAddress, VpcGateway.Type.Private, vpcId, privateNtwk.getDataCenterId(),
privateNtwk.getId(), vlan, gateway, netmask, vpc.getAccountId(), vpc.getDomainId(), isSourceNat);
privateNtwk.getId(), vlan, gateway, netmask, vpc.getAccountId(), vpc.getDomainId(), isSourceNat, networkAclId);
_vpcGatewayDao.persist(gatewayVO);
s_logger.debug("Created vpc gateway entry " + gatewayVO);

11
server/test/com/cloud/vpc/MockVpcManagerImpl.java
View File

@ -161,15 +161,14 @@ public class MockVpcManagerImpl extends ManagerBase implements VpcManager {
return null;
}
@Override
public PrivateGateway createVpcPrivateGateway(long vpcId, Long physicalNetworkId, String vlan, String ipAddress, String gateway, String netmask, long gatewayOwnerId, Boolean isSoruceNat, Long aclId) throws ResourceAllocationException, ConcurrentOperationException, InsufficientCapacityException {
return null;
}
/* (non-Javadoc)
* @see com.cloud.network.vpc.VpcService#createVpcPrivateGateway(long, java.lang.Long, java.lang.String, java.lang.String, java.lang.String, java.lang.String, long)
*/
@Override
public PrivateGateway createVpcPrivateGateway(long vpcId, Long physicalNetworkId, String vlan, String ipAddress, String gateway, String netmask, long gatewayOwnerId, Boolean isSourceNat) throws ResourceAllocationException,
ConcurrentOperationException, InsufficientCapacityException {
// TODO Auto-generated method stub
return null;
}
/* (non-Javadoc)
* @see com.cloud.network.vpc.VpcService#applyVpcPrivateGateway(long, boolean)

2
server/test/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java
View File

@ -348,7 +348,7 @@ VpcVirtualNetworkApplianceService {
}
@Override
public boolean applyNetworkACLs(Network network, List<? extends NetworkACLItem> rules, List<? extends VirtualRouter> routers) throws ResourceUnavailableException {
public boolean applyNetworkACLs(Network network, List<? extends NetworkACLItem> rules, List<? extends VirtualRouter> routers, boolean privateGateway) throws ResourceUnavailableException {
return false; //To change body of implemented methods use File | Settings | File Templates.
}

6
server/test/com/cloud/vpc/NetworkACLManagerTest.java
View File

@ -29,6 +29,7 @@ import com.cloud.network.vpc.NetworkACLManagerImpl;
import com.cloud.network.vpc.NetworkACLVO;
import com.cloud.network.vpc.VpcManager;
import com.cloud.network.vpc.dao.NetworkACLDao;
import com.cloud.network.vpc.dao.VpcGatewayDao;
import com.cloud.tags.dao.ResourceTagDao;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
@ -182,6 +183,11 @@ public class NetworkACLManagerTest extends TestCase{
return Mockito.mock(NetworkACLServiceProvider.class);
}
@Bean
public VpcGatewayDao vpcGatewayDao () {
return Mockito.mock(VpcGatewayDao.class);
}
public static class Library implements TypeFilter {
@Override
public boolean match(MetadataReader mdr, MetadataReaderFactory arg1) throws IOException {

7
server/test/com/cloud/vpc/NetworkACLServiceTest.java
View File

@ -21,6 +21,7 @@ import com.cloud.network.NetworkModel;
import com.cloud.network.dao.NetworkDao;
import com.cloud.network.vpc.*;
import com.cloud.network.vpc.dao.NetworkACLDao;
import com.cloud.network.vpc.dao.VpcGatewayDao;
import com.cloud.tags.dao.ResourceTagDao;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
@ -206,6 +207,12 @@ public class NetworkACLServiceTest extends TestCase{
return Mockito.mock(NetworkACLManager.class);
}
@Bean
public VpcGatewayDao vpcGatewayDao () {
return Mockito.mock(VpcGatewayDao.class);
}
public static class Library implements TypeFilter {
@Override
public boolean match(MetadataReader mdr, MetadataReaderFactory arg1) throws IOException {

286
server/test/org/apache/cloudstack/privategw/AclOnPrivateGwTest.java Normal file
View File

@ -0,0 +1,286 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.privategw;
import com.cloud.configuration.ConfigurationManager;
import com.cloud.configuration.dao.ConfigurationDao;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.dc.dao.VlanDao;
import com.cloud.exception.*;
import com.cloud.network.NetworkManager;
import com.cloud.network.NetworkModel;
import com.cloud.network.NetworkService;
import com.cloud.network.dao.*;
import com.cloud.network.vpc.VpcManagerImpl;
import com.cloud.network.vpc.VpcService;
import com.cloud.network.vpc.dao.*;
import com.cloud.network.vpn.Site2SiteVpnManager;
import com.cloud.offerings.dao.NetworkOfferingServiceMapDao;
import com.cloud.server.ConfigurationServer;
import com.cloud.tags.dao.ResourceTagDao;
import com.cloud.user.AccountManager;
import com.cloud.user.ResourceLimitService;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.dao.DomainRouterDao;
import junit.framework.Assert;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.command.admin.vpc.CreatePrivateGatewayCmd;
import org.apache.cloudstack.test.utils.SpringUtils;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.core.type.classreading.MetadataReader;
import org.springframework.core.type.classreading.MetadataReaderFactory;
import org.springframework.core.type.filter.TypeFilter;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.support.AnnotationConfigContextLoader;
import javax.naming.ConfigurationException;
import java.io.IOException;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(loader = AnnotationConfigContextLoader.class)
public class AclOnPrivateGwTest {
private CreatePrivateGatewayCmd createPrivateGwCmd;
@Before
public void setUp() throws ConfigurationException {
createPrivateGwCmd = new CreatePrivateGatewayCmd() {
@Override
public Long getEntityId () {
return 2L;
}
};
}
@Test
public void testExecuteSuccess() {
VpcService _vpcService = Mockito.mock(VpcService.class);
try {
_vpcService.applyVpcPrivateGateway(Mockito.anyLong(), Mockito.anyBoolean());
} catch (ResourceUnavailableException e) {
e.printStackTrace();
} catch (ConcurrentOperationException e) {
e.printStackTrace();
}
}
@Test
public void testExecuteFail() {
VpcService vpcService= Mockito.mock(VpcService.class);
createPrivateGwCmd._vpcService = vpcService;
try {
Mockito.when(vpcService.applyVpcPrivateGateway(Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(null);
} catch (ResourceUnavailableException e) {
e.printStackTrace();
} catch (ConcurrentOperationException e) {
e.printStackTrace();
}
try {
createPrivateGwCmd.execute();
} catch (ServerApiException exception) {
Assert.assertEquals("Failed to create private gateway",
exception.getDescription());
} catch (ResourceAllocationException e) {
e.printStackTrace();
} catch (InsufficientCapacityException e) {
e.printStackTrace();
} catch (ConcurrentOperationException e) {
e.printStackTrace();
} catch (ResourceUnavailableException e) {
e.printStackTrace();
}
}
@Configuration
@ComponentScan(basePackageClasses = {VpcManagerImpl.class}, includeFilters = {@ComponentScan.Filter(value = TestConfiguration.Library.class, type = FilterType.CUSTOM)}, useDefaultFilters = false)
public static class TestConfiguration extends SpringUtils.CloudStackTestConfiguration {
@Bean
public VpcOfferingDao vpcOfferingDao() {
return Mockito.mock(VpcOfferingDao.class);
}
@Bean
public VpcOfferingServiceMapDao vpcOfferingServiceMapDao () {
return Mockito.mock(VpcOfferingServiceMapDao.class);
}
@Bean
public VpcDao vpcDao () {
return Mockito.mock(VpcDao.class);
}
@Bean
public ConfigurationDao configurationDao () {
return Mockito.mock(ConfigurationDao.class);
}
@Bean
public ConfigurationManager configurationManager () {
return Mockito.mock(ConfigurationManager.class);
}
@Bean
public AccountManager accountManager () {
return Mockito.mock(AccountManager.class);
}
@Bean
public NetworkDao networkDao () {
return Mockito.mock(NetworkDao.class);
}
@Bean
public NetworkManager networkManager () {
return Mockito.mock(NetworkManager.class);
}
@Bean
public NetworkModel networkModel () {
return Mockito.mock(NetworkModel.class);
}
@Bean
public NetworkService networkService () {
return Mockito.mock(NetworkService.class);
}
@Bean
public IPAddressDao iPAddressDao () {
return Mockito.mock(IPAddressDao.class);
}
@Bean
public DomainRouterDao domainRouterDao () {
return Mockito.mock(DomainRouterDao.class);
}
@Bean
public VpcGatewayDao vpcGatewayDao () {
return Mockito.mock(VpcGatewayDao.class);
}
@Bean
public PrivateIpDao privateIpDao () {
return Mockito.mock(PrivateIpDao.class);
}
@Bean
public StaticRouteDao staticRouteDao () {
return Mockito.mock(StaticRouteDao.class);
}
@Bean
public NetworkOfferingServiceMapDao networkOfferingServiceMapDao () {
return Mockito.mock(NetworkOfferingServiceMapDao.class);
}
@Bean
public PhysicalNetworkDao physicalNetworkDao () {
return Mockito.mock(PhysicalNetworkDao.class);
}
@Bean
public ResourceTagDao resourceTagDao () {
return Mockito.mock(ResourceTagDao.class);
}
@Bean
public FirewallRulesDao firewallRulesDao () {
return Mockito.mock(FirewallRulesDao.class);
}
@Bean
public Site2SiteVpnGatewayDao site2SiteVpnGatewayDao () {
return Mockito.mock(Site2SiteVpnGatewayDao.class);
}
@Bean
public Site2SiteVpnManager site2SiteVpnManager () {
return Mockito.mock(Site2SiteVpnManager.class);
}
@Bean
public VlanDao vlanDao () {
return Mockito.mock(VlanDao.class);
}
@Bean
public ResourceLimitService resourceLimitService () {
return Mockito.mock(ResourceLimitService.class);
}
@Bean
public VpcServiceMapDao vpcServiceMapDao () {
return Mockito.mock(VpcServiceMapDao.class);
}
@Bean
public DataCenterDao dataCenterDao () {
return Mockito.mock(DataCenterDao.class);
}
@Bean
public ConfigurationServer configurationServer () {
return Mockito.mock(ConfigurationServer.class);
}
@Bean
public NetworkACLDao networkACLDao () {
return Mockito.mock(NetworkACLDao.class);
}
public static class Library implements TypeFilter {
@Override
public boolean match(MetadataReader mdr, MetadataReaderFactory arg1) throws IOException {
ComponentScan cs = TestConfiguration.class.getAnnotation(ComponentScan.class);
return SpringUtils.includedInBasePackageClasses(mdr.getClassMetadata().getClassName(), cs);
}
}
}
}

7
setup/db/db/schema-410to420.sql
View File

@ -1521,8 +1521,8 @@ ALTER TABLE `cloud`.`account_details` MODIFY value varchar(255);
INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'midonet.apiserver.address', 'http://localhost:8081', 'Specify the address at which the Midonet API server can be contacted (if using Midonet)');
INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'midonet.providerrouter.id', 'd7c5e6a3-e2f4-426b-b728-b7ce6a0448e5', 'Specifies the UUID of the Midonet provider router (if using Midonet)');
alter table cloud.vpc_gateways add column source_nat boolean default false;
alter table cloud.private_ip_address add column source_nat boolean default false;
alter table `cloud`.`vpc_gateways` add column `source_nat` boolean default false;
alter table `cloud`.`private_ip_address` add column `source_nat` boolean default false;
CREATE TABLE `cloud`.`account_vnet_map` (
`id` bigint unsigned NOT NULL UNIQUE AUTO_INCREMENT,
@ -1600,3 +1600,6 @@ CREATE TABLE `cloud`.`nic_ip_alias` (
`state` char(32) NOT NULL,
PRIMARY KEY (`id`) ,
UNIQUE INDEX `id_UNIQUE` (`id` ASC) );
alter table `cloud`.`vpc_gateways` add column network_acl_id bigint unsigned default 1 NOT NULL;
update table cloud.vpc_gateways set network_acl_id = 2;

1
test/integration/smoke/test_network_acl.py
View File

@ -53,7 +53,6 @@ class TestNetworkACL(cloudstackTestCase):
# 4) Create network with ACL
self.createNetwork()
# 5) Deploy a vm
self.deployVm()

148
test/integration/smoke/test_privategw_acl.py Normal file
View File

@ -0,0 +1,148 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
""" Tests for Network ACLs in VPC
"""
#Import Local Modules
from marvin.cloudstackTestCase import *
from marvin.cloudstackAPI import *
from marvin.integration.lib.utils import *
from marvin.integration.lib.base import *
from marvin.integration.lib.common import *
class TestPrivateGwACL(cloudstackTestCase):
def setUp(self):
self.apiClient = self.testClient.getApiClient()
self.networkOfferingId = 11
self.networkId = None
self.vmId = None
self.vpcId = None
self.aclId = None
self.zoneId = 1
self.serviceOfferingId = 1
self.templateId = 5
self.privateGwId = None
def test_privategw_acl(self):
# 1) Create VPC
self.createVPC()
# 2) Create ACl
self.createACL()
# 3) Create ACl Item
self.createACLItem()
# 4) Create network with ACL
self.createNetwork()
# 5) create private gw
self.createPvtGw()
# 6) update acl id
self.replaceacl()
def createACL(self):
createAclCmd = createNetworkACLList.createNetworkACLListCmd()
createAclCmd.name = "acl1"
createAclCmd.description = "new acl"
createAclCmd.vpcid = self.vpcId
createAclResponse = self.apiClient.createNetworkACLList(createAclCmd)
self.aclId = createAclResponse.id
def createACLItem(self):
createAclItemCmd = createNetworkACL.createNetworkACLCmd()
createAclItemCmd.cidr = "0.0.0.0/0"
createAclItemCmd.protocol = "TCP"
createAclItemCmd.number = "10"
createAclItemCmd.action = "Deny"
createAclItemCmd.aclid = self.aclId
createAclItemResponse = self.apiClient.createNetworkACL(createAclItemCmd)
self.assertIsNotNone(createAclItemResponse.id, "Network failed to aclItem")
def createVPC(self):
createVPCCmd = createVPC.createVPCCmd()
createVPCCmd.name = "new vpc"
createVPCCmd.cidr = "10.1.1.0/24"
createVPCCmd.displaytext = "new vpc"
createVPCCmd.vpcofferingid = 1
createVPCCmd.zoneid = self.zoneId
createVPCResponse = self.apiClient.createVPC(createVPCCmd)
self.vpcId = createVPCResponse.id
def createNetwork(self):
createNetworkCmd = createNetwork.createNetworkCmd()
createNetworkCmd.name = "vpc network"
createNetworkCmd.displaytext = "vpc network"
createNetworkCmd.netmask = "255.255.255.0"
createNetworkCmd.gateway = "10.1.1.1"
createNetworkCmd.zoneid = self.zoneId
createNetworkCmd.vpcid = self.vpcId
createNetworkCmd.networkofferingid = self.networkOfferingId
createNetworkCmd.aclid = self.aclId
createNetworkResponse = self.apiClient.createNetwork(createNetworkCmd)
self.assertIsNotNone(createNetworkResponse.id, "Network failed to create")
self.networkId = createNetworkResponse.id
def deployVm(self):
deployVirtualMachineCmd = deployVirtualMachine.deployVirtualMachineCmd()
deployVirtualMachineCmd.networkids = self.networkId
deployVirtualMachineCmd.serviceofferingid = self.serviceOfferingId
deployVirtualMachineCmd.zoneid = self.zoneId
deployVirtualMachineCmd.templateid = self.templateId
deployVirtualMachineCmd.hypervisor = "XenServer"
deployVMResponse = self.apiClient.deployVirtualMachine(deployVirtualMachineCmd)
def deployVm(self):
deployVirtualMachineCmd = deployVirtualMachine.deployVirtualMachineCmd()
deployVirtualMachineCmd.networkids = TestNetworkACL.networkId
deployVirtualMachineCmd.serviceofferingid = TestNetworkACL.serviceOfferingId
deployVirtualMachineCmd.zoneid = TestNetworkACL.zoneId
deployVirtualMachineCmd.templateid = TestNetworkACL.templateId
deployVirtualMachineCmd.hypervisor = "XenServer"
deployVMResponse = self.apiClient.deployVirtualMachine(deployVirtualMachineCmd)
TestNetworkACL.vmId = deployVMResponse.id
self.vmId = deployVMResponse.id
def createPvtGw(self):
createPrivateGatewayCmd = createPrivateGateway.createPrivateGatewayCmd()
createPrivateGatewayCmd.physicalnetworkid = 200
createPrivateGatewayCmd.gateway = "10.147.30.1"
createPrivateGatewayCmd.netmask = "255.255.255.0"
createPrivateGatewayCmd.ipaddress = "10.147.30.200"
createPrivateGatewayCmd.vlan = "30"
createPrivateGatewayCmd.vpcid = self.vpcId
createPrivateGatewayCmd.sourcenatsupported = "true"
createPrivateGatewayCmd.aclid = self.aclId
privateGatewayResponse = self.apiClient.createPrivateGateway(createPrivateGatewayCmd)
self.privateGwId = privateGatewayResponse.id
def replaceacl(self):
replaceNetworkACLListCmd = replaceNetworkACLList.replaceNetworkACLListCmd()
replaceNetworkACLListCmd.aclid = self.aclId
replaceNetworkACLListCmd.gatewayid = self.privateGwId
successResponse = self.apiClient.replaceNetworkACLList(replaceNetworkACLListCmd);
def tearDown(self):
#destroy the vm
if self.vmId is not None:
destroyVirtualMachineCmd = destroyVirtualMachine.destroyVirtualMachineCmd()
destroyVirtualMachineCmd.id = self.vmId
destroyVirtualMachineResponse = self.apiClient.destroyVirtualMachine(destroyVirtualMachineCmd)