Implement PVLAN on Xen

Start/stop vm/dhcp server are done. Not done with VM migration.

A new command(PvlanSetupCommand) is sent for setting up PVLAN for vms. Currently
it's focus on OVS implementation. Need to be more abstruct and add vSwitch part.

api/src/com/cloud/agent/api/PvlanSetupCommand.java

@@ -0,0 +1,130 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api;
+
+import java.net.URI;
+
+import com.cloud.utils.net.NetUtils;
+
+public class PvlanSetupCommand extends Command {
+	public enum Type {
+		DHCP,
+		VM,
+		VM_IN_DHCP_HOST
+	}
+	private String op;
+	private String bridge;
+	private String primary;
+	private String isolated;
+	private String vmMac;
+	private String dhcpMac;
+	private String dhcpIp;
+	private boolean strict;
+	private Type type;
+
+	protected PvlanSetupCommand() {}
+	
+	protected PvlanSetupCommand(Type type, String op, String bridge, URI uri)
+	{
+		this.type = type;
+		this.op = op;
+		this.bridge = bridge;
+		this.primary = NetUtils.getPrimaryPvlanFromUri(uri);
+		this.isolated = NetUtils.getIsolatedPvlanFromUri(uri);
+		this.strict = true;
+	}
+	
+	static public PvlanSetupCommand createDhcpSetup(String op, String bridge, URI uri, String dhcpMac, String dhcpIp)
+	{
+		PvlanSetupCommand cmd = new PvlanSetupCommand(Type.DHCP, op, bridge, uri);
+		cmd.setDhcpMac(dhcpMac);
+		cmd.setDhcpIp(dhcpIp);
+		return cmd;
+	}
+	
+	static public PvlanSetupCommand createVmSetup(String op, String bridge, URI uri, String vmMac)
+	{
+		PvlanSetupCommand cmd = new PvlanSetupCommand(Type.VM, op, bridge, uri);
+		cmd.setVmMac(vmMac);
+		return cmd;
+	}
+	
+	static public PvlanSetupCommand createVmInDhcpHostSetup(String op, String bridge, URI uri, String dhcpMac, String vmMac)
+	{
+		PvlanSetupCommand cmd = new PvlanSetupCommand(Type.VM_IN_DHCP_HOST, op, bridge, uri);
+		cmd.setDhcpMac(dhcpMac);
+		cmd.setVmMac(vmMac);
+		return cmd;
+	}
+	
+	@Override
+	public boolean executeInSequence() {
+		return true;
+	}
+
+	public String getOp() {
+		return op;
+	}
+
+	public String getBridge() {
+		return bridge;
+	}
+
+	public String getPrimary() {
+		return primary;
+	}
+
+	public String getIsolated() {
+		return isolated;
+	}
+
+	public String getVmMac() {
+		return vmMac;
+	}
+
+	protected void setVmMac(String vmMac) {
+		this.vmMac = vmMac;
+	}
+
+	public String getDhcpMac() {
+		return dhcpMac;
+	}
+
+	protected void setDhcpMac(String dhcpMac) {
+		this.dhcpMac = dhcpMac;
+	}
+
+	public String getDhcpIp() {
+		return dhcpIp;
+	}
+
+	protected void setDhcpIp(String dhcpIp) {
+		this.dhcpIp = dhcpIp;
+	}
+
+	public Type getType() {
+		return type;
+	}
+
+	public boolean isStrict() {
+		return strict;
+	}
+
+	public void setStrict(boolean strict) {
+		this.strict = strict;
+	}
+}

plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java

@@ -604,6 +604,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
             return execute((NetworkRulesVmSecondaryIpCommand)cmd);
         } else if (clazz == ScaleVmCommand.class) {
             return execute((ScaleVmCommand) cmd);
+        } else if (clazz == PvlanSetupCommand.class) {
+            return execute((PvlanSetupCommand) cmd);
         } else {
             return Answer.createUnsupportedCommandAnswer(cmd);
         }
@@ -1466,6 +1468,48 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
         }
     }
     
+    private Answer execute(PvlanSetupCommand cmd) {
+    	Connection conn = getConnection();
+    	
+    	String primaryPvlan = cmd.getPrimary();
+    	String isolatedPvlan = cmd.getIsolated();
+    	String op = cmd.getOp();
+    	String bridge = cmd.getBridge();
+    	String result = null;
+    	String dhcpMac = cmd.getDhcpMac();
+    	String dhcpIp = cmd.getDhcpIp();
+    	String vmMac = cmd.getVmMac();
+    	if (cmd.getType() == PvlanSetupCommand.Type.DHCP) {
+    		result = callHostPlugin(conn, "ovs-pvlan", "setup-pvlan-dhcp", "op", op, "bridge", bridge,
+    				"primary-pvlan", primaryPvlan, "isolated-pvlan", isolatedPvlan, "dhcp-ip", dhcpIp, "dhcp-mac", dhcpMac);
+    		if (result == null || result.isEmpty() || !Boolean.parseBoolean(result)) {
+    			s_logger.warn("Failed to program pvlan for dhcp server with mac " + dhcpMac);
+    			return new Answer(cmd, false, result);
+    		} else {
+    			s_logger.info("Programmed pvlan for dhcp server with mac " + dhcpMac);
+    		}
+    	} else if (cmd.getType() == PvlanSetupCommand.Type.VM) {
+    		result = callHostPlugin(conn, "ovs-pvlan", "setup-pvlan-vm-alone", "op", op, "bridge", bridge,
+    				"primary-pvlan", primaryPvlan, "isolated-pvlan", isolatedPvlan, "vm-mac", vmMac);
+    		if (result == null || result.isEmpty() || !Boolean.parseBoolean(result)) {
+    			s_logger.warn("Failed to program pvlan for vm with mac " + vmMac);
+    			return new Answer(cmd, false, result);
+    		} else {
+    			s_logger.info("Programmed pvlan for vm with mac " + vmMac);
+    		}
+    	} else if (cmd.getType() == PvlanSetupCommand.Type.VM_IN_DHCP_HOST) {
+    		result = callHostPlugin(conn, "ovs-pvlan", "setup-pvlan-vm-dhcp", "op", op, "bridge", bridge,
+    				"primary-pvlan", primaryPvlan, "isolated-pvlan", isolatedPvlan, "vm-mac", vmMac, "dhcp-mac", dhcpMac);
+    		if (result == null || result.isEmpty() || !Boolean.parseBoolean(result)) {
+    			s_logger.warn("Failed to program pvlan for vm in dhcp host with mac " + vmMac);
+    			return new Answer(cmd, false, result);
+    		} else {
+    			s_logger.info("Programmed pvlan for vm in dhcp host with mac " + vmMac);
+    		}
+    	}
+    	return new Answer(cmd, true, result);
+    }
+
     @Override
     public StartAnswer execute(StartCommand cmd) {
         Connection conn = getConnection();

scripts/vm/hypervisor/xenserver/ovs-pvlan

@@ -0,0 +1,168 @@
+#!/usr/bin/python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+import cloudstack_pluginlib as lib
+import logging
+import os
+import sys
+import subprocess
+import time
+import XenAPIPlugin
+
+sys.path.append("/opt/xensource/sm/")
+import util
+
+from time import localtime as _localtime, asctime as _asctime
+
+xePath = "/opt/xensource/bin/xe"
+lib.setup_logging("/var/log/ovs-pvlan.log")
+dhcpSetupPath = "/opt/xensource/bin/ovs-pvlan-dhcp-host.sh"
+vmAloneSetupPath = "/opt/xensource/bin/ovs-pvlan-vm.sh"
+vmDhcpSetupPath = "/opt/xensource/bin/ovs-pvlan-vm-in-dhcp-host.sh"
+pvlanCleanupPath = "/opt/xensource/bin/ovs-pvlan-cleanup.sh"
+
+def echo(fn):
+    def wrapped(*v, **k):
+        name = fn.__name__
+        util.SMlog("#### VMOPS enter  %s ####" % name)
+        res = fn(*v, **k)
+        util.SMlog("#### VMOPS exit  %s ####" % name)
+        return res
+    return wrapped
+
+
+@echo
+def setup_pvlan_dhcp(session, args):
+    op = args.pop("op")
+    bridge = args.pop("bridge")
+    primary = args.pop("primary-pvlan")
+    isolated = args.pop("isolated-pvlan")
+    dhcp_ip = args.pop("dhcp-ip");
+    dhcp_mac = args.pop("dhcp-mac");
+
+    res = lib.check_switch()
+    if res != "SUCCESS":
+        return "FAILURE:%s" % res
+
+    if op == "add":
+        logging.debug("About to setup dhcp vm on the switch:%s" % bridge)
+        res = lib.do_cmd([dhcpSetupPath, "-A", "-b", bridge, "-p", primary,
+            "-i", isolated, "-d", dhcp_ip, "-m", dhcp_mac])
+	if res:
+	    result = "FAILURE:%s" % res
+	    return result;
+	logging.debug("Setup dhcp vm on switch program done")
+    elif op == "delete":
+        logging.debug("About to remove dhcp the switch:%s" % bridge)
+        res = lib.do_cmd([dhcpSetupPath, "-D", "-b", bridge, "-p", primary,
+            "-i", isolated, "-d", dhcp_ip, "-m", dhcp_mac])
+	if res:
+	    result = "FAILURE:%s" % res
+	    return result;
+	logging.debug("Remove DHCP on switch program done")
+    
+    result = "true"
+    logging.debug("Setup_pvlan_dhcp completed with result:%s" % result)
+    return result
+
+@echo
+def setup_pvlan_vm_alone(session, args):
+    op = args.pop("op")
+    bridge = args.pop("bridge")
+    isolated = args.pop("isolated-pvlan")
+    vm_mac = args.pop("vm-mac")
+    trunk_port = 1
+
+    res = lib.check_switch()
+    if res != "SUCCESS":
+        return "FAILURE:%s" % res
+
+    if op == "add":
+        logging.debug("About to setup vm alone on the switch:%s" % bridge)
+        res = lib.do_cmd([vmAloneSetupPath, "-A", "-b", bridge, "-i", isolated, "-v", vm_mac])
+	if res:
+	    result = "FAILURE:%s" % res
+	    return result;
+	logging.debug("Setup vm alone on switch program done")
+    elif op == "delete":
+        logging.debug("About to remove vm alone on the switch:%s" % bridge)
+        res = lib.do_cmd([vmAloneSetupPath, "-D", "-b", bridge, "-i", isolated, "-v", vm_mac])
+	if res:
+	    result = "FAILURE:%s" % res
+	    return result;
+	logging.debug("Remove vm alone on switch program done")
+
+    result = "true"
+    logging.debug("Setup_pvlan_vm_alone completed with result:%s" % result)
+    return result
+
+@echo
+def setup_pvlan_vm_dhcp(session, args):
+    op = args.pop("op")
+    bridge = args.pop("bridge")
+    isolated = args.pop("isolated-pvlan")
+    vm_mac = args.pop("vm-mac")
+    dhcp_mac = args.pop("dhcp-mac");
+    trunk_port = 1
+
+    res = lib.check_switch()
+    if res != "SUCCESS":
+        return "FAILURE:%s" % res
+
+    if op == "add":
+        logging.debug("About to setup vm dhcp on the switch:%s" % bridge)
+        res = lib.do_cmd([vmDhcpSetupPath, "-A", "-b", bridge, "-i", isolated,
+            "-v", vm_mac, "-m", dhcp_mac])
+	if res:
+	    result = "FAILURE:%s" % res
+	    return result;
+	logging.debug("Setup vm dhcp on switch program done")
+    elif op == "delete":
+        logging.debug("About to remove vm dhcp on the switch:%s" % bridge)
+        res = lib.do_cmd([vmDhcpSetupPath, "-D", "-b", bridge, "-i", isolated,
+            "-v", vm_mac, "-m", dhcp_mac])
+	if res:
+	    result = "FAILURE:%s" % res
+	    return result;
+	logging.debug("Remove vm dhcp on switch program done")
+
+    result = "true"
+    logging.debug("Setup_pvlan_vm_dhcp completed with result:%s" % result)
+    return result
+
+@echo
+def cleanup(session, args):
+    res = lib.check_switch()
+    if res != "SUCCESS":
+        return "FAILURE:%s" % res
+
+    res = lib.do_cmd([pvlanCleanUpPath])
+    if res:
+        result = "FAILURE:%s" % res
+        return result;
+
+    result = "true"
+    logging.debug("Setup_pvlan_vm_dhcp completed with result:%s" % result)
+    return result
+
+if __name__ == "__main__":
+    XenAPIPlugin.dispatch({"setup-pvlan-dhcp": setup_pvlan_dhcp,
+                           "setup-pvlan-vm-alone": setup_pvlan_vm_alone,
+                           "setup-pvlan-vm-dhcp": setup_pvlan_vm_dhcp,
+                           "cleanup":cleanup})

scripts/vm/hypervisor/xenserver/xenserver60/patch

@@ -67,4 +67,8 @@ bumpUpPriority.sh=../../../../network/domr/,0755,/opt/xensource/bin
 swift=..,0755,/opt/xensource/bin
 swiftxen=..,0755,/etc/xapi.d/plugins
 s3xen=..,0755,/etc/xapi.d/plugins
-
+ovs-pvlan=..,0755,/etc/xapi.d/plugins
+ovs-pvlan-dhcp-host.sh=../../../network,0755,/opt/xensource/bin
+ovs-pvlan-vm-in-dhcp-host.sh=../../../network,0755,/opt/xensource/bin
+ovs-pvlan-vm.sh=../../../network,0755,/opt/xensource/bin
+ovs-pvlan-cleanup.sh=../../../network,0755,/opt/xensource/bin

scripts/vm/network/ovs-pvlan-cleanup.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#!/bin/bash
+
+ovs-ofctl del-flows xenbr0
+ovs-ofctl add-flow xenbr0 priority=0,actions=NORMAL
+

scripts/vm/network/ovs-pvlan-dhcp-host.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+usage() {
+  printf "Usage: %s: (-A|-D) -b <bridge/switch> -p <primary vlan> -i <secondary isolated vlan> -d <DHCP server IP> -m <DHCP server MAC> -v <VM MAC> -h \n" $(basename $0) >&2
+  exit 2
+}
+
+br=
+pri_vlan=
+sec_iso_vlan=
+dhcp_ip=
+dhcp_mac=
+vm_mac=
+op=
+
+while getopts 'ADb:p:i:d:m:v:h' OPTION
+do
+  case $OPTION in
+  A)  op="add"
+      ;;
+  D)  op="del"
+      ;;
+  b)  br="$OPTARG"
+      ;;
+  p)  pri_vlan="$OPTARG"
+      ;;
+  i)  sec_iso_vlan="$OPTARG"
+      ;;
+  d)  dhcp_ip="$OPTARG"
+      ;;
+  m)  dhcp_mac="$OPTARG"
+      ;;
+  v)  vm_mac="$OPTARG"
+      ;;
+  h)  usage
+      exit 1
+      ;;
+  esac
+done
+
+if [ -z "$op" ]
+then
+    echo Missing operation pararmeter!
+    exit 1
+fi
+
+if [ -z "$br" ]
+then
+    echo Missing parameter bridge!
+    exit 1
+fi
+
+if [ -z "$pri_vlan" ]
+then
+    echo Missing parameter primary vlan!
+    exit 1
+fi
+
+if [ -z "$sec_iso_vlan" ]
+then
+    echo Missing parameter secondary isolate vlan!
+    exit 1
+fi
+
+if [ -z "$dhcp_ip" ]
+then
+    echo Missing parameter DHCP IP!
+    exit 1
+fi
+
+if [ -z "$dhcp_mac" ]
+then
+    echo Missing parameter DHCP MAC!
+    exit 1
+fi
+
+if [ "$op" == "add" ]
+then
+    ovs-ofctl add-flow $br priority=200,arp,dl_vlan=$sec_iso_vlan,nw_dst=$dhcp_ip,actions=mod_vlan_vid:$pri_vlan,NORMAL
+    ovs-ofctl add-flow $br priority=180,arp,nw_dst=$dhcp_ip,actions=NORMAL
+    ovs-ofctl add-flow $br priority=150,dl_vlan=$sec_iso_vlan,dl_dst=$dhcp_mac,actions=mod_vlan_vid:$pri_vlan,NORMAL
+    ovs-ofctl add-flow $br priority=100,udp,dl_vlan=$sec_iso_vlan,nw_dst=255.255.255.255,tp_dst=67,actions=mod_vlan_vid:$pri_vlan,NORMAL
+else
+    ovs-ofctl del-flows --strict $br priority=200,arp,dl_vlan=$sec_iso_vlan,nw_dst=$dhcp_ip
+    ovs-ofctl del-flows --strict $br priority=180,arp,nw_dst=$dhcp_ip
+    ovs-ofctl del-flows --strict $br priority=150,dl_vlan=$sec_iso_vlan,dl_dst=$dhcp_mac
+    ovs-ofctl del-flows --strict $br priority=100,udp,dl_vlan=$sec_iso_vlan,nw_dst=255.255.255.255,tp_dst=67
+fi

scripts/vm/network/ovs-pvlan-vm-in-dhcp-host.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+usage() {
+  printf "Usage: %s: (-A|-D) -b <bridge/switch> -p <primary vlan> -i <secondary isolated vlan> -d <DHCP server IP> -m <DHCP server MAC> -v <VM MAC> -h \n" $(basename $0) >&2
+  exit 2
+}
+
+br=
+pri_vlan=
+sec_iso_vlan=
+dhcp_ip=
+dhcp_mac=
+vm_mac=
+op=
+
+while getopts 'ADb:p:i:d:m:v:h' OPTION
+do
+  case $OPTION in
+  A)  op="add"
+      ;;
+  D)  op="del"
+      ;;
+  b)  br="$OPTARG"
+      ;;
+  p)  pri_vlan="$OPTARG"
+      ;;
+  i)  sec_iso_vlan="$OPTARG"
+      ;;
+  d)  dhcp_ip="$OPTARG"
+      ;;
+  m)  dhcp_mac="$OPTARG"
+      ;;
+  v)  vm_mac="$OPTARG"
+      ;;
+  h)  usage
+      exit 1
+      ;;
+  esac
+done
+
+if [ -z "$op" ]
+then
+    echo Missing operation pararmeter!
+    exit 1
+fi
+
+if [ -z "$br" ]
+then
+    echo Missing parameter bridge!
+    exit 1
+fi
+
+if [ -z "$vm_mac" ]
+then
+    echo Missing parameter VM MAC!
+    exit 1
+fi
+
+if [ -z "$dhcp_mac" ]
+then
+    echo Missing parameter DHCP MAC!
+    exit 1
+fi
+
+if [ "$op" == "add" ]
+then
+    ovs-ofctl add-flow $br priority=120,dl_src=$vm_mac,dl_dst=$dhcp_mac,actions=NORMAL
+    ovs-ofctl add-flow $br priority=80,udp,dl_src=$vm_mac,nw_dst=255.255.255.255,tp_dst=67,actions=NORMAL
+else
+    ovs-ofctl del-flows --strict $br priority=120,dl_src=$vm_mac,dl_dst=$dhcp_mac
+    ovs-ofctl del-flows --strict $br priority=80,udp,dl_src=$vm_mac,nw_dst=255.255.255.255,tp_dst=67
+fi

scripts/vm/network/ovs-pvlan-vm.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+usage() {
+  printf "Usage: %s: (-A|-D) -b <bridge/switch> -p <primary vlan> -i <secondary isolated vlan> -d <DHCP server IP> -m <DHCP server MAC> -v <VM MAC> -h \n" $(basename $0) >&2
+  exit 2
+}
+
+br=
+pri_vlan=
+sec_iso_vlan=
+dhcp_ip=
+dhcp_mac=
+vm_mac=
+op=
+
+while getopts 'ADb:p:i:d:m:v:h' OPTION
+do
+  case $OPTION in
+  A)  op="add"
+      ;;
+  D)  op="del"
+      ;;
+  b)  br="$OPTARG"
+      ;;
+  p)  pri_vlan="$OPTARG"
+      ;;
+  i)  sec_iso_vlan="$OPTARG"
+      ;;
+  d)  dhcp_ip="$OPTARG"
+      ;;
+  m)  dhcp_mac="$OPTARG"
+      ;;
+  v)  vm_mac="$OPTARG"
+      ;;
+  h)  usage
+      exit 1
+      ;;
+  esac
+done
+
+if [ -z "$op" ]
+then
+    echo Missing operation pararmeter!
+    exit 1
+fi
+
+if [ -z "$br" ]
+then
+    echo Missing parameter bridge!
+    exit 1
+fi
+
+if [ -z "$vm_mac" ]
+then
+    echo Missing parameter VM MAC!
+    exit 1
+fi
+
+if [ -z "$sec_iso_vlan" ]
+then
+    echo Missing parameter secondary isolate vlan!
+    exit 1
+fi
+
+trunk_port=1
+
+if [ "$op" == "add" ]
+then
+    ovs-ofctl add-flow $br priority=50,dl_src=$vm_mac,actions=mod_vlan_vid:$sec_iso_vlan,output:$trunk_port
+else
+    # it would delete any rule related to this vm, not only the rule added above
+    ovs-ofctl del-flows $br dl_src=$vm_mac
+fi
+

server/src/com/cloud/network/element/VirtualRouterElement.java

@@ -31,6 +31,7 @@ import org.apache.cloudstack.api.command.admin.router.CreateVirtualRouterElement
 import org.apache.cloudstack.api.command.admin.router.ListVirtualRouterElementsCmd;
 import org.apache.log4j.Logger;
 
+import com.cloud.agent.api.PvlanSetupCommand;
 import com.cloud.agent.api.to.LoadBalancerTO;
 import com.cloud.configuration.ConfigurationManager;
 import com.cloud.configuration.dao.ConfigurationDao;
@@ -48,6 +49,7 @@ import com.cloud.network.Network.Capability;
 import com.cloud.network.Network.Provider;
 import com.cloud.network.Network.Service;
 import com.cloud.network.NetworkModel;
+import com.cloud.network.Networks.BroadcastDomainType;
 import com.cloud.network.Networks.TrafficType;
 import com.cloud.network.PhysicalNetworkServiceProvider;
 import com.cloud.network.PublicIpAddress;
@@ -214,6 +216,15 @@ public class VirtualRouterElement extends AdapterBase implements VirtualRouterEl
                     DataCenter.class, network.getDataCenterId());
         }
         
+        // Setup PVlan for vm if necessary
+        if (network.getTrafficType() == TrafficType.Guest && network.getBroadcastDomainType() == BroadcastDomainType.Pvlan) {
+        	assert routers.size() == 1;
+        	DomainRouterVO router = routers.get(0);
+        	if (router.getHostId() == dest.getHost().getId()) {
+        		_routerMgr.setupVmWithDhcpHostForPvlan(true, router, nic);
+        	}
+        }
+        
         return true;      
     }
 

server/src/com/cloud/network/router/VirtualNetworkApplianceManager.java

@@ -19,6 +19,7 @@ package com.cloud.network.router;
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.agent.api.PvlanSetupCommand;
 import com.cloud.deploy.DeployDestination;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
@@ -35,6 +36,7 @@ import com.cloud.user.User;
 import com.cloud.uservm.UserVm;
 import com.cloud.utils.component.Manager;
 import com.cloud.vm.DomainRouterVO;
+import com.cloud.vm.Nic;
 import com.cloud.vm.NicProfile;
 import com.cloud.vm.VirtualMachineProfile;
 
@@ -103,4 +105,6 @@ public interface VirtualNetworkApplianceManager extends Manager, VirtualNetworkA
 	
 	boolean applyUserData(Network config, NicProfile nic, VirtualMachineProfile<UserVm> vm, DeployDestination dest, 
 	        List<DomainRouterVO> routers) throws ResourceUnavailableException;
+
+	void setupVmWithDhcpHostForPvlan(boolean add, DomainRouterVO router, NicProfile profile) throws ResourceUnavailableException;
 }

server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -62,6 +62,7 @@ import com.cloud.agent.api.GetDomRVersionCmd;
 import com.cloud.agent.api.ModifySshKeysCommand;
 import com.cloud.agent.api.NetworkUsageAnswer;
 import com.cloud.agent.api.NetworkUsageCommand;
+import com.cloud.agent.api.PvlanSetupCommand;
 import com.cloud.agent.api.StartupCommand;
 import com.cloud.agent.api.StopAnswer;
 import com.cloud.agent.api.check.CheckSshAnswer;
@@ -2210,6 +2211,72 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V
         return dhcpRange;
     }
 
+    private boolean setupDhcpForPvlanOnHost(boolean add, DomainRouterVO router, Nic routerNic) {
+    	if (!routerNic.getBroadcastUri().getScheme().equals("pvlan")) {
+    		return false;
+    	}
+    	setupDhcpForPvlan(add, router, routerNic);
+    	Long hostId = router.getHostId();
+    	List<UserVmVO> vms = _userVmDao.listByHostId(hostId);
+    	for (UserVmVO vm : vms) {
+    		if (vm.getState() != State.Running) {
+    			continue;
+    		}
+    		List<NicVO> nics = _nicDao.listByVmId(vm.getId());
+    		for (NicVO nic : nics) {
+    			if (nic.getNetworkId() == routerNic.getNetworkId()) {
+    				try {
+    					Network network = _networkDao.findById(routerNic.getNetworkId());
+    					NicProfile profile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), 
+    							null, _networkModel.isSecurityGroupSupportedInNetwork(network), _networkModel.getNetworkTag(vm.getHypervisorType(), network));
+						setupVmWithDhcpHostForPvlan(add, router, profile);
+					} catch (ResourceUnavailableException e) {
+						s_logger.warn("Fail to program pvlan on nic " + nic.getMacAddress(), e);
+						return false;
+					}
+    			}
+    		}
+    	}
+    	return true;
+    }
+    
+    private boolean setupDhcpForPvlan(boolean add, DomainRouterVO router, Nic nic) {
+    	if (!nic.getBroadcastUri().getScheme().equals("pvlan")) {
+    		return false;
+    	}
+    	String op = "add";
+    	if (!add) {
+    		op = "delete";
+    	}
+    	PvlanSetupCommand cmd = PvlanSetupCommand.createDhcpSetup(op, "xenbr0", nic.getBroadcastUri(), nic.getMacAddress(), nic.getIp4Address());
+    	Commands cmds = new Commands(cmd);
+    	// In fact we send command to the host of router, we're not programming router but the host
+    	try {
+			sendCommandsToRouter(router, cmds);
+		} catch (AgentUnavailableException e) {
+            s_logger.warn("Agent Unavailable ", e);
+			return false;
+		}
+    	return true;
+    }
+    
+    @Override
+    public void setupVmWithDhcpHostForPvlan(boolean add, DomainRouterVO router, NicProfile profile) throws ResourceUnavailableException
+    {
+    	if (!profile.getBroadCastUri().getScheme().equals("pvlan")) {
+    		return;
+    	}
+    	String op = "add";
+    	if (!add) {
+    		op = "delete";
+    	}
+    	NicVO routerNic = _nicDao.findByInstanceIdAndNetworkId(profile.getNetworkId(), router.getId());
+    	PvlanSetupCommand cmd = PvlanSetupCommand.createVmInDhcpHostSetup(op, "xenbr0", profile.getBroadCastUri(), routerNic.getMacAddress(), profile.getMacAddress());
+    	Commands cmds = new Commands(cmd);
+    	// In fact we send command to the host of router, we're not programming router but the host
+    	sendCommandsToRouter(router, cmds);
+    }
+    
     @Override
     public boolean finalizeDeployment(Commands cmds, VirtualMachineProfile<DomainRouterVO> profile, 
             DeployDestination dest, ReservationContext context) throws ResourceUnavailableException {
@@ -2505,13 +2572,20 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V
         List<Network> guestNetworks = new ArrayList<Network>();
         
         List<? extends Nic> routerNics = _nicDao.listByVmId(profile.getId());
-        for (Nic routerNic : routerNics) {
-            Network network = _networkModel.getNetwork(routerNic.getNetworkId());
+        for (Nic nic : routerNics) {
+        	Network network = _networkModel.getNetwork(nic.getNetworkId());
             if (network.getTrafficType() == TrafficType.Guest) {
                 guestNetworks.add(network);
+                if (nic.getBroadcastUri().getScheme().equals("pvlan")) {
+                	result = setupDhcpForPvlanOnHost(true, router, nic);
+                }
             }
         }
         
+        if (!result) {
+        	return result;
+        }
+        
         answer = cmds.getAnswer("getDomRVersion");
         if (answer != null && answer instanceof GetDomRVersionAnswer) {
             GetDomRVersionAnswer versionAnswer = (GetDomRVersionAnswer)answer;
@@ -2537,6 +2611,13 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V
             VMInstanceVO vm = profile.getVirtualMachine();
             DomainRouterVO domR = _routerDao.findById(vm.getId());
             processStopOrRebootAnswer(domR, answer);
+            List<? extends Nic> routerNics = _nicDao.listByVmId(profile.getId());
+            for (Nic nic : routerNics) {
+            	Network network = _networkModel.getNetwork(nic.getNetworkId());
+            	if (network.getTrafficType() == TrafficType.Guest && nic.getBroadcastUri().getScheme().equals("pvlan")) {
+            		setupDhcpForPvlanOnHost(false, domR, nic);
+            	}
+            }
         }
     }
 

server/src/com/cloud/vm/UserVmManagerImpl.java

@@ -69,6 +69,7 @@ import com.cloud.agent.api.GetVmStatsAnswer;
 import com.cloud.agent.api.GetVmStatsCommand;
 import com.cloud.agent.api.PlugNicAnswer;
 import com.cloud.agent.api.PlugNicCommand;
+import com.cloud.agent.api.PvlanSetupCommand;
 import com.cloud.agent.api.StartAnswer;
 import com.cloud.agent.api.StopAnswer;
 import com.cloud.agent.api.UnPlugNicAnswer;
@@ -2751,6 +2752,34 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Use
         return true;
     }
 
+    private boolean setupVmForPvlan(boolean add, Long hostId, NicVO nic) {
+        if (!nic.getBroadcastUri().getScheme().equals("pvlan")) {
+    		return false;
+    	}
+        String op = "add";
+        if (!add) {
+        	// "delete" would remove all the rules(if using ovs) related to this vm
+        	op = "delete";
+        }
+    	PvlanSetupCommand cmd = PvlanSetupCommand.createVmSetup(op, "xenbr0", nic.getBroadcastUri(), nic.getMacAddress());
+        Answer answer = null;
+        try {
+            answer = _agentMgr.send(hostId, cmd);
+        } catch (OperationTimedoutException e) {
+            s_logger.warn("Timed Out", e);
+            return false;
+        } catch (AgentUnavailableException e) {
+            s_logger.warn("Agent Unavailable ", e);
+            return false;
+        }
+
+        boolean result = true;
+        if (answer == null || !answer.getResult()) {
+        	result = false;
+        }
+        return result;
+    }
+    
     @Override
     public boolean finalizeDeployment(Commands cmds,
             VirtualMachineProfile<UserVmVO> profile, DeployDestination dest,
@@ -2812,6 +2841,11 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Use
                 originalIp = nic.getIp4Address();
                 guestNic = nic;
                 guestNetwork = network;
+                if (nic.getBroadcastUri().getScheme().equals("pvlan")) {
+                	if (!setupVmForPvlan(true, hostId, nic)) {
+                		return false;
+                	}
+                }
             }
         }
         boolean ipChanged = false;
@@ -2942,6 +2976,17 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Use
                                 + " stop due to exception ", ex);
             }
         }
+        
+        VMInstanceVO vm = profile.getVirtualMachine();
+        List<NicVO> nics = _nicDao.listByVmId(vm.getId());
+        for (NicVO nic : nics) {
+            NetworkVO network = _networkDao.findById(nic.getNetworkId());
+            if (network.getTrafficType() == TrafficType.Guest) {
+                if (nic.getBroadcastUri().getScheme().equals("pvlan")) {
+                	setupVmForPvlan(false, vm.getHostId(), nic);
+                }
+            }
+        }
     }
 
     public String generateRandomPassword() {

server/test/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java

@@ -402,4 +402,11 @@ VpcVirtualNetworkApplianceService {
         return null;
     }
 
+	@Override
+	public void setupVmWithDhcpHostForPvlan(boolean add,
+			DomainRouterVO router, NicProfile nic) throws ResourceUnavailableException {
+		// TODO Auto-generated method stub
+		
+	}
+
 }