From 03ad702c1be3466b982099ec8764a045411cf3bf Mon Sep 17 00:00:00 2001
From: davidjumani <dj.davidjumani1994@gmail.com>
Date: Tue, 30 Mar 2021 14:05:46 +0530
Subject: [PATCH 01/12] server: Set free memory to zero if greater than total
 memory (#4571)

Fixes https://github.com/apache/cloudstack/issues/4566

Sets `memoryintfreekbs` to zero if it is greater than `memorykbs`. Caused by KVM returning the RSS memory of the process running the VM rather than the free memory inside the VM.

Co-authored-by: dahn <daan.hoogland@gmail.com>
---
 .../org/apache/cloudstack/api/response/UserVmResponse.java | 2 +-
 .../java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java    | 7 +++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java
index 44eaba7f313..ea5300e7875 100644
--- a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java
+++ b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java
@@ -209,7 +209,7 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co
     private Long memoryKBs;
 
     @SerializedName("memoryintfreekbs")
-    @Param(description = "the internal memory thats free in vm")
+    @Param(description = "the internal memory that's free in vm or zero if it can not be calculated")
     private Long memoryIntFreeKBs;
 
     @SerializedName("memorytargetkbs")
diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java
index 8e489f89eca..1589f0151fe 100644
--- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java
+++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java
@@ -222,8 +222,11 @@ public class UserVmJoinDaoImpl extends GenericDaoBaseWithTagInformation<UserVmJo
                 userVmResponse.setDiskKbsWrite((long)vmStats.getDiskWriteKBs());
                 userVmResponse.setDiskIORead((long)vmStats.getDiskReadIOs());
                 userVmResponse.setDiskIOWrite((long)vmStats.getDiskWriteIOs());
-                userVmResponse.setMemoryKBs((long)vmStats.getMemoryKBs());
-                userVmResponse.setMemoryIntFreeKBs((long)vmStats.getIntFreeMemoryKBs());
+                long totalMemory = (long)vmStats.getMemoryKBs();
+                long freeMemory = (long)vmStats.getIntFreeMemoryKBs();
+                long correctedFreeMemory = freeMemory >= totalMemory ? 0 : freeMemory;
+                userVmResponse.setMemoryKBs(totalMemory);
+                userVmResponse.setMemoryIntFreeKBs(correctedFreeMemory);
                 userVmResponse.setMemoryTargetKBs((long)vmStats.getTargetMemoryKBs());
 
             }

From 5b986be5ba6915ed85a6430f19fccf6af3787cfc Mon Sep 17 00:00:00 2001
From: Hoang Nguyen <hoangnm@unitech.vn>
Date: Tue, 30 Mar 2021 15:39:56 +0700
Subject: [PATCH 02/12] ui: fix the style of zone wizard on mobile view (#4764)

This PR for fixes create zone form in mobile view
---
 .../infra/zone/AdvancedGuestTrafficForm.vue   |   3 +-
 .../views/infra/zone/IpAddressRangeForm.vue   | 146 ++++++++++--------
 ui/src/views/infra/zone/StaticInputsForm.vue  |   3 +-
 ui/src/views/infra/zone/ZoneWizard.vue        |  37 ++++-
 .../infra/zone/ZoneWizardAddResources.vue     |  39 ++++-
 .../infra/zone/ZoneWizardNetworkSetupStep.vue |  36 ++++-
 .../ZoneWizardPhysicalNetworkSetupStep.vue    |  10 +-
 7 files changed, 190 insertions(+), 84 deletions(-)

diff --git a/ui/src/views/infra/zone/AdvancedGuestTrafficForm.vue b/ui/src/views/infra/zone/AdvancedGuestTrafficForm.vue
index 8a96677859d..ca2cb0ae6cb 100644
--- a/ui/src/views/infra/zone/AdvancedGuestTrafficForm.vue
+++ b/ui/src/views/infra/zone/AdvancedGuestTrafficForm.vue
@@ -194,11 +194,10 @@ export default {
     min-height: 200px;
     text-align: center;
     vertical-align: center;
-    padding-top: 16px;
-    padding-top: 16px;
     margin-top: 8px;
     max-height: 300px;
     overflow-y: auto;
+    padding: 16px 20px 0;
 
     /deep/.has-error {
       .ant-form-explain {
diff --git a/ui/src/views/infra/zone/IpAddressRangeForm.vue b/ui/src/views/infra/zone/IpAddressRangeForm.vue
index f1082a37d9f..07b2ef9c956 100644
--- a/ui/src/views/infra/zone/IpAddressRangeForm.vue
+++ b/ui/src/views/infra/zone/IpAddressRangeForm.vue
@@ -24,79 +24,94 @@
     </a-card>
     <a-table
       bordered
+      :scroll="{ x: 500 }"
       :dataSource="ipRanges"
       :columns="columns"
       :pagination="false"
-      style="margin-bottom: 24px;" >
+      style="margin-bottom: 24px; width: 100%" >
       <template slot="actions" slot-scope="text, record">
         <a-button type="danger" shape="circle" icon="delete" @click="onDelete(record.key)" />
       </template>
       <template slot="footer">
         <a-form
-          layout="inline"
+          :layout="isMobile() ? 'horizontal': 'inline'"
           :form="form"
           @submit="handleAddRange">
-          <a-form-item :style="{ display: 'inline-block', width: '14%' }">
-            <a-input
-              v-decorator="[ 'gateway', {
-                rules: [{ required: true, message: $t('message.error.gateway') }]
-              }]"
-              :placeholder="$t('label.gateway')"
-              autoFocus
-            />
-          </a-form-item>
-          <a-form-item :style="{ display: 'inline-block', width: '14%' }">
-            <a-input
-              v-decorator="[ 'netmask', {
-                rules: [{ required: true, message: $t('message.error.netmask') }]
-              }]"
-              :placeholder="$t('label.netmask')"
-            />
-          </a-form-item>
-          <a-form-item :style="{ display: 'inline-block', width: '14%' }">
-            <a-input
-              v-decorator="[ 'vlan', { rules: [{ required: false }] }]"
-              :placeholder="$t('label.vlan')"
-            />
-          </a-form-item>
-          <a-form-item :style="{ display: 'inline-block', width: '14%' }">
-            <a-input
-              v-decorator="[ 'startIp', {
-                rules: [
-                  {
-                    required: true,
-                    message: $t('message.error.startip')
-                  },
-                  {
-                    validator: checkIpFormat,
-                    ipV4: true,
-                    message: $t('message.error.ipv4.address')
-                  }
-                ]
-              }]"
-              :placeholder="$t('label.start.ip')"
-            />
-          </a-form-item>
-          <a-form-item :style="{ display: 'inline-block', width: '14%' }">
-            <a-input
-              v-decorator="[ 'endIp', {
-                rules: [
-                  {
-                    required: true,
-                    message: $t('message.error.endip')
-                  },
-                  {
-                    validator: checkIpFormat,
-                    ipV4: true,
-                    message: $t('message.error.ipv4.address')
-                  }]
-              }]"
-              :placeholder="$t('label.end.ip')"
-            />
-          </a-form-item>
-          <a-form-item :style="{ display: 'inline-block', width: '14%' }">
-            <a-button type="primary" html-type="submit">{{ $t('label.add') }}</a-button>
-          </a-form-item>
+          <a-row :gutter="12">
+            <a-col :md="4" :lg="4">
+              <a-form-item>
+                <a-input
+                  v-decorator="[ 'gateway', {
+                    rules: [{ required: true, message: $t('message.error.gateway') }]
+                  }]"
+                  :placeholder="$t('label.gateway')"
+                  autoFocus
+                />
+              </a-form-item>
+            </a-col>
+            <a-col :md="4" :lg="4">
+              <a-form-item>
+                <a-input
+                  v-decorator="[ 'netmask', {
+                    rules: [{ required: true, message: $t('message.error.netmask') }]
+                  }]"
+                  :placeholder="$t('label.netmask')"
+                />
+              </a-form-item>
+            </a-col>
+            <a-col :md="4" :lg="4">
+              <a-form-item>
+                <a-input
+                  v-decorator="[ 'vlan', { rules: [{ required: false }] }]"
+                  :placeholder="$t('label.vlan')"
+                />
+              </a-form-item>
+            </a-col>
+            <a-col :md="4" :lg="4">
+              <a-form-item>
+                <a-input
+                  v-decorator="[ 'startIp', {
+                    rules: [
+                      {
+                        required: true,
+                        message: $t('message.error.startip')
+                      },
+                      {
+                        validator: checkIpFormat,
+                        ipV4: true,
+                        message: $t('message.error.ipv4.address')
+                      }
+                    ]
+                  }]"
+                  :placeholder="$t('label.start.ip')"
+                />
+              </a-form-item>
+            </a-col>
+            <a-col :md="4" :lg="4">
+              <a-form-item>
+                <a-input
+                  v-decorator="[ 'endIp', {
+                    rules: [
+                      {
+                        required: true,
+                        message: $t('message.error.endip')
+                      },
+                      {
+                        validator: checkIpFormat,
+                        ipV4: true,
+                        message: $t('message.error.ipv4.address')
+                      }]
+                  }]"
+                  :placeholder="$t('label.end.ip')"
+                />
+              </a-form-item>
+            </a-col>
+            <a-col :md="4" :lg="4">
+              <a-form-item :style="{ display: 'inline-block', float: 'right' }">
+                <a-button type="primary" html-type="submit">{{ $t('label.add') }}</a-button>
+              </a-form-item>
+            </a-col>
+          </a-row>
         </a-form>
       </template>
     </a-table>
@@ -126,7 +141,10 @@
   </div>
 </template>
 <script>
+import { mixinDevice } from '@/utils/mixin.js'
+
 export default {
+  mixins: [mixinDevice],
   props: {
     traffic: {
       type: String,
@@ -183,7 +201,7 @@ export default {
           title: '',
           dataIndex: 'actions',
           scopedSlots: { customRender: 'actions' },
-          width: 50
+          width: 70
         }
       ],
       showError: false,
diff --git a/ui/src/views/infra/zone/StaticInputsForm.vue b/ui/src/views/infra/zone/StaticInputsForm.vue
index 6a9903f0871..45b1a8a2480 100644
--- a/ui/src/views/infra/zone/StaticInputsForm.vue
+++ b/ui/src/views/infra/zone/StaticInputsForm.vue
@@ -251,11 +251,10 @@ export default {
     min-height: 200px;
     text-align: center;
     vertical-align: center;
-    padding-top: 16px;
-    padding-top: 16px;
     margin-top: 8px;
     max-height: 300px;
     overflow-y: auto;
+    padding: 16px 20px 0;
 
     /deep/.has-error {
       .ant-form-explain {
diff --git a/ui/src/views/infra/zone/ZoneWizard.vue b/ui/src/views/infra/zone/ZoneWizard.vue
index 8fd0852c0fa..862b1f841f8 100644
--- a/ui/src/views/infra/zone/ZoneWizard.vue
+++ b/ui/src/views/infra/zone/ZoneWizard.vue
@@ -18,13 +18,15 @@
 <template>
   <div class="form">
     <a-steps
+      ref="zoneStep"
       labelPlacement="vertical"
       size="small"
       :current="currentStep">
       <a-step
-        v-for="(item) in steps"
+        v-for="(item, index) in steps"
         :key="item.title"
-        :title="$t(item.title)">
+        :title="$t(item.title)"
+        :ref="`step${index}`">
       </a-step>
     </a-steps>
     <div>
@@ -79,6 +81,7 @@
   </div>
 </template>
 <script>
+import { mixinDevice } from '@/utils/mixin.js'
 import ZoneWizardZoneTypeStep from '@views/infra/zone/ZoneWizardZoneTypeStep'
 import ZoneWizardZoneDetailsStep from '@views/infra/zone/ZoneWizardZoneDetailsStep'
 import ZoneWizardNetworkSetupStep from '@views/infra/zone/ZoneWizardNetworkSetupStep'
@@ -93,6 +96,7 @@ export default {
     ZoneWizardAddResources,
     ZoneWizardLaunchZone
   },
+  mixins: [mixinDevice],
   data () {
     return {
       currentStep: 0,
@@ -138,9 +142,26 @@ export default {
   methods: {
     nextPressed () {
       this.currentStep++
+      this.scrollToStepActive()
     },
     backPressed (data) {
       this.currentStep--
+      this.scrollToStepActive()
+    },
+    scrollToStepActive () {
+      if (!this.isMobile()) {
+        return
+      }
+      this.$nextTick(() => {
+        if (!this.$refs.zoneStep) {
+          return
+        }
+        if (this.currentStep === 0) {
+          this.$refs.zoneStep.$el.scrollLeft = 0
+          return
+        }
+        this.$refs.zoneStep.$el.scrollLeft = this.$refs['step' + (this.currentStep - 1)][0].$el.offsetLeft
+      })
     },
     onFieldsChanged (data) {
       if (data.zoneType &&
@@ -177,7 +198,8 @@ export default {
 
 <style scoped lang="scss">
   .form {
-    width: 95vw;
+    width: 100%;
+
     @media (min-width: 1000px) {
       width: 800px;
     }
@@ -197,6 +219,15 @@ export default {
       position: absolute;
       right: 0;
     }
+
+    /deep/.ant-steps {
+      overflow-x: auto;
+      padding: 10px 0;
+    }
+
+    /deep/.submit-btn {
+      display: none;
+    }
   }
 
   /deep/.ant-form-text {
diff --git a/ui/src/views/infra/zone/ZoneWizardAddResources.vue b/ui/src/views/infra/zone/ZoneWizardAddResources.vue
index 4ba878ef7d5..e2b566646e1 100644
--- a/ui/src/views/infra/zone/ZoneWizardAddResources.vue
+++ b/ui/src/views/infra/zone/ZoneWizardAddResources.vue
@@ -17,9 +17,15 @@
 
 <template>
   <div style="width: auto;">
-    <a-steps progressDot :current="currentStep" size="small" style="margin-left: 0; margin-top: 16px;">
+    <a-steps
+      ref="resourceStep"
+      progressDot
+      :current="currentStep"
+      size="small"
+      style="margin-left: 0; margin-top: 16px;">
       <a-step
-        v-for="step in steps"
+        v-for="(step, index) in steps"
+        :ref="`resourceStep${index}`"
         :key="step.title"
         :title="$t(step.title)"></a-step>
     </a-steps>
@@ -97,13 +103,15 @@
   </div>
 </template>
 <script>
-import StaticInputsForm from '@views/infra/zone/StaticInputsForm'
 import { api } from '@/api'
+import { mixinDevice } from '@/utils/mixin.js'
+import StaticInputsForm from '@views/infra/zone/StaticInputsForm'
 
 export default {
   components: {
     StaticInputsForm
   },
+  mixins: [mixinDevice],
   props: {
     prefillContent: {
       type: Object,
@@ -687,14 +695,16 @@ export default {
       primaryStorageScopes: [],
       primaryStorageProtocols: [],
       storageProviders: [],
-      currentStep: 0,
+      currentStep: null,
       options: ['primaryStorageScope', 'primaryStorageProtocol', 'provider']
     }
   },
   mounted () {
+    this.currentStep = this.prefillContent.resourceStep ? this.prefillContent.resourceStep : 0
     if (this.stepChild && this.stepChild !== '') {
       this.currentStep = this.steps.findIndex(item => item.fromKey === this.stepChild)
     }
+    this.scrollToStepActive()
     if (this.prefillContent.hypervisor.value === 'BareMetal') {
       this.$emit('nextPressed')
     } else {
@@ -719,14 +729,35 @@ export default {
         this.$emit('nextPressed')
       } else {
         this.currentStep++
+        this.$emit('fieldsChanged', { resourceStep: this.currentStep })
       }
+
+      this.scrollToStepActive()
     },
     handleBack (e) {
       if (this.currentStep === 0) {
         this.$emit('backPressed')
       } else {
         this.currentStep--
+        this.$emit('fieldsChanged', { resourceStep: this.currentStep })
       }
+
+      this.scrollToStepActive()
+    },
+    scrollToStepActive () {
+      if (!this.isMobile()) {
+        return
+      }
+      this.$nextTick(() => {
+        if (!this.$refs.resourceStep) {
+          return
+        }
+        if (this.currentStep === 0) {
+          this.$refs.resourceStep.$el.scrollLeft = 0
+          return
+        }
+        this.$refs.resourceStep.$el.scrollLeft = this.$refs['resourceStep' + (this.currentStep - 1)][0].$el.offsetLeft
+      })
     },
     fieldsChanged (changed) {
       this.$emit('fieldsChanged', changed)
diff --git a/ui/src/views/infra/zone/ZoneWizardNetworkSetupStep.vue b/ui/src/views/infra/zone/ZoneWizardNetworkSetupStep.vue
index c7175600df4..d1c317229e3 100644
--- a/ui/src/views/infra/zone/ZoneWizardNetworkSetupStep.vue
+++ b/ui/src/views/infra/zone/ZoneWizardNetworkSetupStep.vue
@@ -17,12 +17,18 @@
 
 <template>
   <div style="width: auto;">
-    <a-steps progressDot :current="currentStep" size="small" style="margin-left: 0px; margin-top: 16px;">
+    <a-steps
+      ref="zoneNetStep"
+      progressDot
+      :current="currentStep"
+      size="small"
+      style="margin-left: 0px; margin-top: 16px;">
       <a-step
-        v-for="step in steps"
+        v-for="(step, index) in steps"
         :key="step.title"
         :title="$t(step.title)"
-        :style="stepScales"></a-step>
+        :style="stepScales"
+        :ref="`netStep${index}`"></a-step>
     </a-steps>
     <zone-wizard-physical-network-setup-step
       v-if="steps && steps[currentStep].formKey === 'physicalNetwork'"
@@ -110,6 +116,7 @@
 
 <script>
 import { api } from '@/api'
+import { mixinDevice } from '@/utils/mixin.js'
 import ZoneWizardPhysicalNetworkSetupStep from '@views/infra/zone/ZoneWizardPhysicalNetworkSetupStep'
 import IpAddressRangeForm from '@views/infra/zone/IpAddressRangeForm'
 import StaticInputsForm from '@views/infra/zone/StaticInputsForm'
@@ -122,6 +129,7 @@ export default {
     StaticInputsForm,
     AdvancedGuestTrafficForm
   },
+  mixins: [mixinDevice],
   props: {
     prefillContent: {
       type: Object,
@@ -187,8 +195,8 @@ export default {
       return steps
     },
     stepScales () {
-      if (this.allSteps.length > 4) {
-        return { width: 'calc(100% / ' + this.allSteps.length + ')' }
+      if (!this.isMobile() && this.steps.length > 4) {
+        return { width: 'calc(100% / ' + this.steps.length + ')' }
       }
       return {}
     },
@@ -368,6 +376,7 @@ export default {
     if (this.stepChild && this.stepChild !== '') {
       this.currentStep = this.steps.findIndex(item => item.formKey === this.stepChild)
     }
+    this.scrollToStepActive()
     if (this.zoneType === 'Basic' ||
       (this.zoneType === 'Advanced' && this.sgEnabled)) {
       this.skipGuestTrafficStep = false
@@ -401,6 +410,7 @@ export default {
       } else {
         this.currentStep++
         this.$emit('fieldsChanged', { networkStep: this.currentStep })
+        this.scrollToStepActive()
       }
     },
     handleBack (e) {
@@ -409,8 +419,24 @@ export default {
       } else {
         this.currentStep--
         this.$emit('fieldsChanged', { networkStep: this.currentStep })
+        this.scrollToStepActive()
       }
     },
+    scrollToStepActive () {
+      if (!this.isMobile()) {
+        return
+      }
+      this.$nextTick(() => {
+        if (!this.$refs.zoneNetStep) {
+          return
+        }
+        if (this.currentStep === 0) {
+          this.$refs.zoneNetStep.$el.scrollLeft = 0
+          return
+        }
+        this.$refs.zoneNetStep.$el.scrollLeft = this.$refs['netStep' + (this.currentStep - 1)][0].$el.offsetLeft
+      })
+    },
     submitLaunchZone () {
       this.$emit('submitLaunchZone')
     },
diff --git a/ui/src/views/infra/zone/ZoneWizardPhysicalNetworkSetupStep.vue b/ui/src/views/infra/zone/ZoneWizardPhysicalNetworkSetupStep.vue
index 40b749f7698..08ed2382af0 100644
--- a/ui/src/views/infra/zone/ZoneWizardPhysicalNetworkSetupStep.vue
+++ b/ui/src/views/infra/zone/ZoneWizardPhysicalNetworkSetupStep.vue
@@ -24,10 +24,11 @@
     </a-card>
     <a-table
       bordered
+      :scroll="{ x: 500 }"
       :dataSource="physicalNetworks"
       :columns="columns"
       :pagination="false"
-      style="margin-bottom: 24px;">
+      style="margin-bottom: 24px; width: 100%">
       <template slot="name" slot-scope="text, record">
         <a-input :value="text" @change="e => onCellChange(record.key, 'name', e.target.value)" autoFocus />
       </template>
@@ -212,19 +213,20 @@ export default {
       columns.push({
         title: this.$t('label.network.name'),
         dataIndex: 'name',
-        width: '30%',
+        width: 175,
         scopedSlots: { customRender: 'name' }
       })
       columns.push({
         title: this.$t('label.isolation.method'),
         dataIndex: 'isolationMethod',
-        width: '20%',
+        width: 150,
         scopedSlots: { customRender: 'isolationMethod' }
       })
       columns.push({
         title: this.$t('label.traffic.types'),
         key: 'traffics',
         dataIndex: 'traffics',
+        width: 250,
         scopedSlots: { customRender: 'traffics' }
       })
       if (this.isAdvancedZone) {
@@ -232,7 +234,7 @@ export default {
           title: '',
           dataIndex: 'actions',
           scopedSlots: { customRender: 'actions' },
-          width: 50
+          width: 70
         })
       }
 

From fd4bc612d7f04656ef1f4fbf8a96c726bc5e83e2 Mon Sep 17 00:00:00 2001
From: davidjumani <dj.davidjumani1994@gmail.com>
Date: Thu, 1 Apr 2021 12:29:57 +0530
Subject: [PATCH 03/12] ui: assignVM: Set isrecursive to false when fetching
 accounts (#4867)

* Set isrecursive to false when fetching accounts

* Adding exception handling
---
 ui/src/views/compute/AssignInstance.vue | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/ui/src/views/compute/AssignInstance.vue b/ui/src/views/compute/AssignInstance.vue
index 000dee5b2be..e77cfc4d0a1 100644
--- a/ui/src/views/compute/AssignInstance.vue
+++ b/ui/src/views/compute/AssignInstance.vue
@@ -134,6 +134,10 @@ export default {
         this.selectedDomain = this.domains[0].id
         this.fetchAccounts()
         this.fetchProjects()
+      }).catch(error => {
+        this.$notifyError(error)
+      }).finally(() => {
+        this.loading = false
       })
     },
     fetchAccounts () {
@@ -142,9 +146,12 @@ export default {
         response: 'json',
         domainId: this.selectedDomain,
         state: 'Enabled',
-        listAll: true
+        isrecursive: false
       }).then(response => {
         this.accounts = response.listaccountsresponse.account
+      }).catch(error => {
+        this.$notifyError(error)
+      }).finally(() => {
         this.loading = false
       })
     },
@@ -155,9 +162,12 @@ export default {
         domainId: this.selectedDomain,
         state: 'Active',
         details: 'min',
-        listAll: true
+        isrecursive: false
       }).then(response => {
         this.projects = response.listprojectsresponse.project
+      }).catch(error => {
+        this.$notifyError(error)
+      }).finally(() => {
         this.loading = false
       })
     },
@@ -172,6 +182,9 @@ export default {
         projectid: this.selectedProject
       }).then(response => {
         this.networks = response.listnetworksresponse.network
+      }).catch(error => {
+        this.$notifyError(error)
+      }).finally(() => {
         this.loading = false
       })
     },

From 69825f6cb646ad4f821d643397fa3b9538419b09 Mon Sep 17 00:00:00 2001
From: Pearl Dsilva <pearl1594@gmail.com>
Date: Thu, 1 Apr 2021 12:31:07 +0530
Subject: [PATCH 04/12] ui: Display root disk size in Compute offering details
 (#4862)

Co-authored-by: Pearl Dsilva <pearl.dsilva@shapeblue.com>
---
 ui/public/locales/en.json             | 1 +
 ui/src/components/view/DetailsTab.vue | 5 +++++
 ui/src/config/section/offering.js     | 4 ++++
 3 files changed, 10 insertions(+)

diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json
index 476bd89e61a..ba0d2b13d22 100644
--- a/ui/public/locales/en.json
+++ b/ui/public/locales/en.json
@@ -1810,6 +1810,7 @@
 "label.rolename": "Role",
 "label.roles": "Roles",
 "label.roletype": "Role Type",
+"label.rootdisksize": "Root disk size (GB)",
 "label.root.certificate": "Root certificate",
 "label.root.disk.offering": "Root Disk Offering",
 "label.root.disk.size": "Root disk size (GB)",
diff --git a/ui/src/components/view/DetailsTab.vue b/ui/src/components/view/DetailsTab.vue
index 64809b2546a..e47f3c58a3f 100644
--- a/ui/src/components/view/DetailsTab.vue
+++ b/ui/src/components/view/DetailsTab.vue
@@ -33,6 +33,11 @@
             <router-link :to="{ path: '/volume/' + volume.uuid }">{{ volume.type }} - {{ volume.path }}</router-link> ({{ parseFloat(volume.size / (1024.0 * 1024.0 * 1024.0)).toFixed(1) }} GB)
           </div>
         </div>
+        <div v-else-if="$route.meta.name === 'computeoffering' && item === 'rootdisksize'">
+          <div>
+            {{ parseFloat( resource.rootdisksize / (1024.0 * 1024.0 * 1024.0)).toFixed(1) }} GB
+          </div>
+        </div>
         <div v-else-if="['name', 'type'].includes(item)">
           <span v-if="['USER.LOGIN', 'USER.LOGOUT', 'ROUTER.HEALTH.CHECKS', 'FIREWALL.CLOSE', 'ALERT.SERVICE.DOMAINROUTER'].includes(resource[item])">{{ $t(resource[item].toLowerCase()) }}</span>
           <span v-else>{{ resource[item] }}</span>
diff --git a/ui/src/config/section/offering.js b/ui/src/config/section/offering.js
index 085d85989f6..1ec2d40d181 100644
--- a/ui/src/config/section/offering.js
+++ b/ui/src/config/section/offering.js
@@ -36,6 +36,10 @@ export default {
           store.getters.apis.createServiceOffering.params.filter(x => x.name === 'storagepolicy').length > 0) {
           fields.splice(6, 0, 'vspherestoragepolicy')
         }
+        if (store.getters.apis.createServiceOffering &&
+          store.getters.apis.createServiceOffering.params.filter(x => x.name === 'rootdisksize').length > 0) {
+          fields.splice(12, 0, 'rootdisksize')
+        }
         return fields
       },
       related: [{

From 02abb87f12211e7aaf4801e2921e31ccf7449f9a Mon Sep 17 00:00:00 2001
From: Wei Zhou <w.zhou@global.leaseweb.com>
Date: Thu, 1 Apr 2021 09:02:52 +0200
Subject: [PATCH 05/12] server: allow copy cross-zone templates to other zone
 (#4804)

---
 .../src/main/java/com/cloud/template/TemplateManagerImpl.java   | 2 --
 1 file changed, 2 deletions(-)

diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java
index 550f53b37f8..3bdea7c89c1 100755
--- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java
+++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java
@@ -920,8 +920,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
             if (template.isCrossZones()) {
                 // sync template from cache store to region store if it is not there, for cases where we are going to migrate existing NFS to S3.
                 _tmpltSvr.syncTemplateToRegionStore(templateId, srcSecStore);
-                s_logger.debug("Template " + templateId + " is cross-zone, don't need to copy");
-                return template;
             }
             for (Long destZoneId : destZoneIds) {
                 DataStore dstSecStore = getImageStore(destZoneId, templateId);

From 95cd937d7b32dd84c1cf62bc82539256196dbbab Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek.mrt22@gmail.com>
Date: Thu, 1 Apr 2021 12:33:48 +0530
Subject: [PATCH 06/12] server: fix hahost value in listHosts (#4789)

* api/server: fix hahost value in listHosts

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* fix

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* Update server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java

Co-authored-by: dahn <daan.hoogland@gmail.com>

* Update server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java

Co-authored-by: Rohit Yadav <rohit.yadav@shapeblue.com>
Co-authored-by: dahn <daan.hoogland@gmail.com>
---
 .../response/HostForMigrationResponse.java    |  7 ++-
 .../cloud/api/query/dao/HostJoinDaoImpl.java  | 57 ++++++++++---------
 2 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java
index 8501f71eda1..4ed0cdd8d74 100644
--- a/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java
+++ b/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java
@@ -18,8 +18,6 @@ package org.apache.cloudstack.api.response;
 
 import java.util.Date;
 
-import com.google.gson.annotations.SerializedName;
-
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseResponse;
 import org.apache.cloudstack.api.EntityReference;
@@ -28,6 +26,7 @@ import com.cloud.host.Host;
 import com.cloud.host.Status;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.serializer.Param;
+import com.google.gson.annotations.SerializedName;
 
 @EntityReference(value = Host.class)
 public class HostForMigrationResponse extends BaseResponse {
@@ -452,6 +451,10 @@ public class HostForMigrationResponse extends BaseResponse {
         this.hypervisorVersion = hypervisorVersion;
     }
 
+    public Boolean getHaHost() {
+        return haHost;
+    }
+
     public void setHaHost(Boolean haHost) {
         this.haHost = haHost;
     }
diff --git a/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java
index f7373a7687a..de87a0a0d5f 100644
--- a/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java
+++ b/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java
@@ -26,16 +26,18 @@ import java.util.Set;
 
 import javax.inject.Inject;
 
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
 import org.apache.cloudstack.api.ApiConstants.HostDetails;
 import org.apache.cloudstack.api.response.GpuResponse;
 import org.apache.cloudstack.api.response.HostForMigrationResponse;
 import org.apache.cloudstack.api.response.HostResponse;
 import org.apache.cloudstack.api.response.VgpuResponse;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.ha.HAResource;
+import org.apache.cloudstack.ha.dao.HAConfigDao;
 import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
 
 import com.cloud.api.ApiDBUtils;
 import com.cloud.api.query.vo.HostJoinVO;
@@ -52,9 +54,6 @@ import com.cloud.utils.db.GenericDaoBase;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 
-import org.apache.cloudstack.ha.HAResource;
-import org.apache.cloudstack.ha.dao.HAConfigDao;
-
 @Component
 public class HostJoinDaoImpl extends GenericDaoBase<HostJoinVO, Long> implements HostJoinDao {
     public static final Logger s_logger = Logger.getLogger(HostJoinDaoImpl.class);
@@ -178,17 +177,13 @@ public class HostJoinDaoImpl extends GenericDaoBase<HostJoinVO, Long> implements
                 hostResponse.setMemoryAllocatedPercentage(memoryAllocatedPercentage);
 
                 String hostTags = host.getTag();
-                hostResponse.setHostTags(host.getTag());
+                hostResponse.setHostTags(hostTags);
 
+                hostResponse.setHaHost(false);
                 String haTag = ApiDBUtils.getHaTag();
-                if (haTag != null && !haTag.isEmpty() && hostTags != null && !hostTags.isEmpty()) {
-                    if (haTag.equalsIgnoreCase(hostTags)) {
-                        hostResponse.setHaHost(true);
-                    } else {
-                        hostResponse.setHaHost(false);
-                    }
-                } else {
-                    hostResponse.setHaHost(false);
+                if (StringUtils.isNotEmpty(haTag) && StringUtils.isNotEmpty(hostTags) &&
+                        haTag.equalsIgnoreCase(hostTags)) {
+                    hostResponse.setHaHost(true);
                 }
 
                 hostResponse.setHypervisorVersion(host.getHypervisorVersion());
@@ -274,12 +269,19 @@ public class HostJoinDaoImpl extends GenericDaoBase<HostJoinVO, Long> implements
     @Override
     public HostResponse setHostResponse(HostResponse response, HostJoinVO host) {
         String tag = host.getTag();
-        if (tag != null) {
-            if (response.getHostTags() != null && response.getHostTags().length() > 0) {
+        if (StringUtils.isNotEmpty(tag)) {
+            if (StringUtils.isNotEmpty(response.getHostTags())) {
                 response.setHostTags(response.getHostTags() + "," + tag);
             } else {
                 response.setHostTags(tag);
             }
+
+            if (Boolean.FALSE.equals(response.getHaHost())) {
+                String haTag = ApiDBUtils.getHaTag();
+                if (StringUtils.isNotEmpty(haTag) && haTag.equalsIgnoreCase(tag)) {
+                    response.setHaHost(true);
+                }
+            }
         }
         return response;
     }
@@ -334,17 +336,13 @@ public class HostJoinDaoImpl extends GenericDaoBase<HostJoinVO, Long> implements
                 hostResponse.setMemoryAllocatedBytes(mem);
 
                 String hostTags = host.getTag();
-                hostResponse.setHostTags(host.getTag());
+                hostResponse.setHostTags(hostTags);
 
+                hostResponse.setHaHost(false);
                 String haTag = ApiDBUtils.getHaTag();
-                if (haTag != null && !haTag.isEmpty() && hostTags != null && !hostTags.isEmpty()) {
-                    if (haTag.equalsIgnoreCase(hostTags)) {
-                        hostResponse.setHaHost(true);
-                    } else {
-                        hostResponse.setHaHost(false);
-                    }
-                } else {
-                    hostResponse.setHaHost(false);
+                if (StringUtils.isNotEmpty(haTag) && StringUtils.isNotEmpty(hostTags) &&
+                        haTag.equalsIgnoreCase(hostTags)) {
+                    hostResponse.setHaHost(true);
                 }
 
                 hostResponse.setHypervisorVersion(host.getHypervisorVersion());
@@ -419,6 +417,13 @@ public class HostJoinDaoImpl extends GenericDaoBase<HostJoinVO, Long> implements
             } else {
                 response.setHostTags(tag);
             }
+
+            if (Boolean.FALSE.equals(response.getHaHost())) {
+                String haTag = ApiDBUtils.getHaTag();
+                if (StringUtils.isNotEmpty(haTag) && haTag.equalsIgnoreCase(tag)) {
+                    response.setHaHost(true);
+                }
+            }
         }
         return response;
     }

From 76ba5c62d98c81c4de72ee53ac47036a5a28af04 Mon Sep 17 00:00:00 2001
From: Rakesh <rakeshv.com@gmail.com>
Date: Thu, 1 Apr 2021 09:09:01 +0200
Subject: [PATCH 07/12] server: Fix displaying public IP address of shared
 networks (#4675)

Public IP addresses dedicated to one domain should not be accessed
by other domains. Also, root admin should be able to display all
public ip addresses in system.

Currently following issues exist

1. Public IP address assigned to one domain can be accessed by
other sibling domains

If use.system.public.ip is false then child domains should not
see public ip of ROOT domain

Before fix
```
(test1) mgt01 > list publicipaddresses listall=true fordisplay=true allocatedonly=false forvirtualnetwork=true filter=ipaddress,
{
  "count": 59,
  "publicipaddress": [
```

After fix

```
(test) mgt01 > list publicipaddresses listall=true fordisplay=true allocatedonly=false forvirtualnetwork=true filter=ipaddress,
{
  "count": 10,
```
---
 .travis.yml                                   |   1 +
 .../com/cloud/network/IpAddressManager.java   |  17 +
 .../main/java/com/cloud/api/ApiDBUtils.java   |  14 +
 .../java/com/cloud/api/ApiResponseHelper.java |  39 +
 .../cloud/network/IpAddressManagerImpl.java   | 116 ++-
 .../cloud/server/ManagementServerImpl.java    | 232 ++++-
 test/integration/component/test_public_ip.py  | 870 ++++++++++++++++++
 test/integration/smoke/test_network.py        |   9 +-
 8 files changed, 1206 insertions(+), 92 deletions(-)
 create mode 100644 test/integration/component/test_public_ip.py

diff --git a/.travis.yml b/.travis.yml
index 8b2859a2a10..0648b27e95c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -164,6 +164,7 @@ env:
 
     - TESTS="component/test_project_usage
              component/test_protocol_number_security_group
+             component/test_public_ip
              component/test_resource_limits"
 
     - TESTS="component/test_regions_accounts
diff --git a/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java b/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java
index b62a8e9a838..0a50e4b29df 100644
--- a/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java
+++ b/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java
@@ -208,5 +208,22 @@ public interface IpAddressManager {
     void releasePodIp(Long id) throws CloudRuntimeException;
 
     boolean isUsageHidden(IPAddressVO address);
+
+    List<IPAddressVO> listAvailablePublicIps(final long dcId,
+                                             final Long podId,
+                                             final List<Long> vlanDbIds,
+                                             final Account owner,
+                                             final VlanType vlanUse,
+                                             final Long guestNetworkId,
+                                             final boolean sourceNat,
+                                             final boolean assign,
+                                             final boolean allocate,
+                                             final String requestedIp,
+                                             final boolean isSystem,
+                                             final Long vpcId,
+                                             final Boolean displayIp,
+                                             final boolean forSystemVms,
+                                             final boolean lockOneRow)
+            throws InsufficientAddressCapacityException;
 }
 
diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java
index 87b9f36fd23..d7445927f37 100644
--- a/server/src/main/java/com/cloud/api/ApiDBUtils.java
+++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java
@@ -27,6 +27,8 @@ import java.util.Set;
 import javax.annotation.PostConstruct;
 import javax.inject.Inject;
 
+import com.cloud.vm.NicVO;
+import com.cloud.vm.dao.NicDao;
 import org.apache.cloudstack.acl.Role;
 import org.apache.cloudstack.acl.RoleService;
 import org.apache.cloudstack.affinity.AffinityGroup;
@@ -458,6 +460,7 @@ public class ApiDBUtils {
     static BackupDao s_backupDao;
     static BackupScheduleDao s_backupScheduleDao;
     static BackupOfferingDao s_backupOfferingDao;
+    static NicDao s_nicDao;
 
     @Inject
     private ManagementServer ms;
@@ -702,6 +705,8 @@ public class ApiDBUtils {
     private BackupOfferingDao backupOfferingDao;
     @Inject
     private BackupScheduleDao backupScheduleDao;
+    @Inject
+    private NicDao nicDao;
 
     @PostConstruct
     void init() {
@@ -811,6 +816,7 @@ public class ApiDBUtils {
         s_hostDetailsDao = hostDetailsDao;
         s_clusterDetailsDao = clusterDetailsDao;
         s_vmSnapshotDao = vmSnapshotDao;
+        s_nicDao = nicDao;
         s_nicSecondaryIpDao = nicSecondaryIpDao;
         s_vpcProvSvc = vpcProvSvc;
         s_affinityGroupDao = affinityGroupDao;
@@ -2079,4 +2085,12 @@ public class ApiDBUtils {
     public static BackupOfferingResponse newBackupOfferingResponse(BackupOffering policy) {
         return s_backupOfferingDao.newBackupOfferingResponse(policy);
     }
+
+    public static NicVO findByIp4AddressAndNetworkId(String ip4Address, long networkId) {
+        return s_nicDao.findByIp4AddressAndNetworkId(ip4Address, networkId);
+    }
+
+    public static NicSecondaryIpVO findSecondaryIpByIp4AddressAndNetworkId(String ip4Address, long networkId) {
+        return s_nicSecondaryIpDao.findByIp4AddressAndNetworkId(ip4Address, networkId);
+    }
 }
diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java
index c159c5c8d53..05593fd1f84 100644
--- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java
+++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java
@@ -920,6 +920,9 @@ public class ApiResponseHelper implements ResponseGenerator {
             }
         }
 
+        // show vm info for shared networks
+        showVmInfoForSharedNetworks(forVirtualNetworks, ipAddr, ipResponse);
+
         // show this info to full view only
         if (view == ResponseView.Full) {
             VlanVO vl = ApiDBUtils.findVlanById(ipAddr.getVlanId());
@@ -954,6 +957,42 @@ public class ApiResponseHelper implements ResponseGenerator {
         return ipResponse;
     }
 
+    private void showVmInfoForSharedNetworks(boolean forVirtualNetworks, IpAddress ipAddr, IPAddressResponse ipResponse) {
+        if (!forVirtualNetworks) {
+            NicVO nic = ApiDBUtils.findByIp4AddressAndNetworkId(ipAddr.getAddress().toString(), ipAddr.getNetworkId());
+
+            if (nic == null) {  // find in nic_secondary_ips, user vm only
+                NicSecondaryIpVO secondaryIp =
+                        ApiDBUtils.findSecondaryIpByIp4AddressAndNetworkId(ipAddr.getAddress().toString(), ipAddr.getNetworkId());
+                if (secondaryIp != null) {
+                    UserVm vm = ApiDBUtils.findUserVmById(secondaryIp.getVmId());
+                    if (vm != null) {
+                        ipResponse.setVirtualMachineId(vm.getUuid());
+                        ipResponse.setVirtualMachineName(vm.getHostName());
+                        if (vm.getDisplayName() != null) {
+                            ipResponse.setVirtualMachineDisplayName(vm.getDisplayName());
+                        } else {
+                            ipResponse.setVirtualMachineDisplayName(vm.getHostName());
+                        }
+                    }
+                }
+            } else if (nic.getVmType() == VirtualMachine.Type.User) {
+                UserVm vm = ApiDBUtils.findUserVmById(nic.getInstanceId());
+                if (vm != null) {
+                    ipResponse.setVirtualMachineId(vm.getUuid());
+                    ipResponse.setVirtualMachineName(vm.getHostName());
+                    if (vm.getDisplayName() != null) {
+                        ipResponse.setVirtualMachineDisplayName(vm.getDisplayName());
+                    } else {
+                        ipResponse.setVirtualMachineDisplayName(vm.getHostName());
+                    }
+                }
+            } else if (nic.getVmType() == VirtualMachine.Type.DomainRouter) {
+                ipResponse.setIsSystem(true);
+            }
+        }
+    }
+
     @Override
     public LoadBalancerResponse createLoadBalancerResponse(LoadBalancer loadBalancer) {
         LoadBalancerResponse lbResponse = new LoadBalancerResponse();
diff --git a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java
index e5f06c69f3e..2f630cfa682 100644
--- a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java
+++ b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java
@@ -322,31 +322,30 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
                 }
             }
 
-            for (final IPAddressVO possibleAddr : addressVOS) {
+            for (IPAddressVO possibleAddr : addressVOS) {
                 if (possibleAddr.getState() != State.Free) {
                     continue;
                 }
-                final IPAddressVO addressVO = possibleAddr;
-                addressVO.setSourceNat(sourceNat);
-                addressVO.setAllocatedTime(new Date());
-                addressVO.setAllocatedInDomainId(owner.getDomainId());
-                addressVO.setAllocatedToAccountId(owner.getId());
-                addressVO.setSystem(isSystem);
+                possibleAddr.setSourceNat(sourceNat);
+                possibleAddr.setAllocatedTime(new Date());
+                possibleAddr.setAllocatedInDomainId(owner.getDomainId());
+                possibleAddr.setAllocatedToAccountId(owner.getId());
+                possibleAddr.setSystem(isSystem);
 
                 if (displayIp != null) {
-                    addressVO.setDisplay(displayIp);
+                    possibleAddr.setDisplay(displayIp);
                 }
 
                 if (vlanUse != VlanType.DirectAttached) {
-                    addressVO.setAssociatedWithNetworkId(guestNetworkId);
-                    addressVO.setVpcId(vpcId);
+                    possibleAddr.setAssociatedWithNetworkId(guestNetworkId);
+                    possibleAddr.setVpcId(vpcId);
                 }
                 if (_ipAddressDao.lockRow(possibleAddr.getId(), true) != null) {
-                    final IPAddressVO userIp = _ipAddressDao.findById(addressVO.getId());
+                    final IPAddressVO userIp = _ipAddressDao.findById(possibleAddr.getId());
                     if (userIp.getState() == State.Free) {
-                        addressVO.setState(State.Allocating);
-                        if (_ipAddressDao.update(addressVO.getId(), addressVO)) {
-                            finalAddress = addressVO;
+                        possibleAddr.setState(State.Allocating);
+                        if (_ipAddressDao.update(possibleAddr.getId(), possibleAddr)) {
+                            finalAddress = possibleAddr;
                             break;
                         }
                     }
@@ -783,9 +782,22 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
     public PublicIp fetchNewPublicIp(final long dcId, final Long podId, final List<Long> vlanDbIds, final Account owner, final VlanType vlanUse, final Long guestNetworkId,
             final boolean sourceNat, final boolean assign, final boolean allocate, final String requestedIp, final boolean isSystem, final Long vpcId, final Boolean displayIp, final boolean forSystemVms)
                     throws InsufficientAddressCapacityException {
-        IPAddressVO addr = Transaction.execute(new TransactionCallbackWithException<IPAddressVO, InsufficientAddressCapacityException>() {
+        List<IPAddressVO> addrs = listAvailablePublicIps(dcId, podId, vlanDbIds, owner, vlanUse, guestNetworkId, sourceNat, assign, allocate, requestedIp, isSystem, vpcId, displayIp, forSystemVms, true);
+        IPAddressVO addr = addrs.get(0);
+        if (vlanUse == VlanType.VirtualNetwork) {
+            _firewallMgr.addSystemFirewallRules(addr, owner);
+        }
+
+        return PublicIp.createFromAddrAndVlan(addr, _vlanDao.findById(addr.getVlanId()));
+    }
+
+    @Override
+    public List<IPAddressVO> listAvailablePublicIps(final long dcId, final Long podId, final List<Long> vlanDbIds, final Account owner, final VlanType vlanUse, final Long guestNetworkId,
+                                                    final boolean sourceNat, final boolean assign, final boolean allocate, final String requestedIp, final boolean isSystem,
+                                                    final Long vpcId, final Boolean displayIp, final boolean forSystemVms, final boolean lockOneRow) throws InsufficientAddressCapacityException {
+        return Transaction.execute(new TransactionCallbackWithException<List<IPAddressVO>, InsufficientAddressCapacityException>() {
             @Override
-            public IPAddressVO doInTransaction(TransactionStatus status) throws InsufficientAddressCapacityException {
+            public List<IPAddressVO> doInTransaction(TransactionStatus status) throws InsufficientAddressCapacityException {
                 StringBuilder errorMessage = new StringBuilder("Unable to get ip address in ");
                 boolean fetchFromDedicatedRange = false;
                 List<Long> dedicatedVlanDbIds = new ArrayList<Long>();
@@ -823,23 +835,26 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
                     if (vlanDbIds == null || vlanDbIds.contains(nonDedicatedVlan.getId()))
                         nonDedicatedVlanDbIds.add(nonDedicatedVlan.getId());
                 }
-                if (dedicatedVlanDbIds != null && !dedicatedVlanDbIds.isEmpty()) {
-                    fetchFromDedicatedRange = true;
-                    sc.setParameters("vlanId", dedicatedVlanDbIds.toArray());
-                    errorMessage.append(", vlanId id=" + Arrays.toString(dedicatedVlanDbIds.toArray()));
-                } else if (nonDedicatedVlanDbIds != null && !nonDedicatedVlanDbIds.isEmpty()) {
-                    sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray());
-                    errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray()));
-                } else {
-                    if (podId != null) {
-                        InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", Pod.class, podId);
-                        ex.addProxyObject(ApiDBUtils.findPodById(podId).getUuid());
+
+                if (vlanUse == VlanType.VirtualNetwork) {
+                    if (dedicatedVlanDbIds != null && !dedicatedVlanDbIds.isEmpty()) {
+                        fetchFromDedicatedRange = true;
+                        sc.setParameters("vlanId", dedicatedVlanDbIds.toArray());
+                        errorMessage.append(", vlanId id=" + Arrays.toString(dedicatedVlanDbIds.toArray()));
+                    } else if (nonDedicatedVlanDbIds != null && !nonDedicatedVlanDbIds.isEmpty()) {
+                        sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray());
+                        errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray()));
+                    } else {
+                        if (podId != null) {
+                            InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", Pod.class, podId);
+                            ex.addProxyObject(ApiDBUtils.findPodById(podId).getUuid());
+                            throw ex;
+                        }
+                        s_logger.warn(errorMessage.toString());
+                        InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", DataCenter.class, dcId);
+                        ex.addProxyObject(ApiDBUtils.findZoneById(dcId).getUuid());
                         throw ex;
                     }
-                    s_logger.warn(errorMessage.toString());
-                    InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", DataCenter.class, dcId);
-                    ex.addProxyObject(ApiDBUtils.findZoneById(dcId).getUuid());
-                    throw ex;
                 }
 
                 sc.setParameters("dc", dcId);
@@ -864,21 +879,31 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
 
                 filter.addOrderBy(IPAddressVO.class,"vlanId", true);
 
-                List<IPAddressVO> addrs = _ipAddressDao.search(sc, filter, false);
+                List<IPAddressVO> addrs;
+
+                if (lockOneRow) {
+                    addrs = _ipAddressDao.lockRows(sc, filter, true);
+                } else {
+                    addrs = new ArrayList<>(_ipAddressDao.search(sc, null));
+                }
 
                 // If all the dedicated IPs of the owner are in use fetch an IP from the system pool
-                if (addrs.size() == 0 && fetchFromDedicatedRange) {
+                if ((!lockOneRow || (lockOneRow && addrs.size() == 0)) && fetchFromDedicatedRange && vlanUse == VlanType.VirtualNetwork) {
                     // Verify if account is allowed to acquire IPs from the system
                     boolean useSystemIps = UseSystemPublicIps.valueIn(owner.getId());
                     if (useSystemIps && nonDedicatedVlanDbIds != null && !nonDedicatedVlanDbIds.isEmpty()) {
                         fetchFromDedicatedRange = false;
                         sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray());
                         errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray()));
-                        addrs = _ipAddressDao.search(sc, filter, false);
+                        if (lockOneRow) {
+                            addrs = _ipAddressDao.lockRows(sc, filter, true);
+                        } else {
+                            addrs.addAll(_ipAddressDao.search(sc, null));
+                        }
                     }
                 }
 
-                if (addrs.size() == 0) {
+                if (lockOneRow && addrs.size() == 0) {
                     if (podId != null) {
                         InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", Pod.class, podId);
                         // for now, we hardcode the table names, but we should ideally do a lookup for the tablename from the VO object.
@@ -891,23 +916,16 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
                     throw ex;
                 }
 
-                assert(addrs.size() == 1) : "Return size is incorrect: " + addrs.size();
-                IPAddressVO finalAddr = null;
-                if (assign) {
-                    finalAddr = assignAndAllocateIpAddressEntry(owner, vlanUse, guestNetworkId, sourceNat, allocate,
-                            isSystem,vpcId, displayIp, fetchFromDedicatedRange, addrs);
-                } else {
-                    finalAddr = addrs.get(0);
+                if (lockOneRow) {
+                    assert (addrs.size() == 1) : "Return size is incorrect: " + addrs.size();
                 }
-                return finalAddr;
+                if (assign) {
+                    assignAndAllocateIpAddressEntry(owner, vlanUse, guestNetworkId, sourceNat, allocate,
+                            isSystem,vpcId, displayIp, fetchFromDedicatedRange, addrs);
+                }
+                return addrs;
             }
         });
-
-        if (vlanUse == VlanType.VirtualNetwork) {
-            _firewallMgr.addSystemFirewallRules(addr, owner);
-        }
-
-        return PublicIp.createFromAddrAndVlan(addr, _vlanDao.findById(addr.getVlanId()));
     }
 
     @DB
diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java
index 8f0007aca0c..080dadc74a6 100644
--- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java
+++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java
@@ -610,6 +610,7 @@ import com.cloud.event.EventTypes;
 import com.cloud.event.EventVO;
 import com.cloud.event.dao.EventDao;
 import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientAddressCapacityException;
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.ManagementServerException;
 import com.cloud.exception.OperationTimedoutException;
@@ -634,12 +635,20 @@ import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao;
 import com.cloud.hypervisor.kvm.dpdk.DpdkHelper;
 import com.cloud.info.ConsoleProxyInfo;
 import com.cloud.network.IpAddress;
+import com.cloud.network.IpAddressManager;
+import com.cloud.network.Network;
+import com.cloud.network.NetworkModel;
 import com.cloud.network.dao.IPAddressDao;
 import com.cloud.network.dao.IPAddressVO;
 import com.cloud.network.dao.LoadBalancerDao;
 import com.cloud.network.dao.LoadBalancerVO;
+import com.cloud.network.dao.NetworkAccountDao;
+import com.cloud.network.dao.NetworkAccountVO;
+import com.cloud.network.dao.NetworkDomainDao;
+import com.cloud.network.dao.NetworkDomainVO;
 import com.cloud.network.dao.NetworkDao;
 import com.cloud.network.dao.NetworkVO;
+import com.cloud.network.vpc.dao.VpcDao;
 import com.cloud.org.Cluster;
 import com.cloud.org.Grouping.AllocationState;
 import com.cloud.projects.Project;
@@ -854,6 +863,16 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
     private VolumeDataStoreDao _volumeStoreDao;
     @Inject
     private TemplateDataStoreDao _vmTemplateStoreDao;
+    @Inject
+    private IpAddressManager _ipAddressMgr;
+    @Inject
+    private NetworkAccountDao _networkAccountDao;
+    @Inject
+    private NetworkDomainDao _networkDomainDao;
+    @Inject
+    private NetworkModel _networkMgr;
+    @Inject
+    private VpcDao _vpcDao;
 
     private LockMasterListener _lockMasterListener;
     private final ScheduledExecutorService _eventExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("EventChecker"));
@@ -2004,29 +2023,90 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
 
     @Override
     public Pair<List<? extends IpAddress>, Integer> searchForIPAddresses(final ListPublicIpAddressesCmd cmd) {
-        final Object keyword = cmd.getKeyword();
-        final Long physicalNetworkId = cmd.getPhysicalNetworkId();
         final Long associatedNetworkId = cmd.getAssociatedNetworkId();
-        final Long sourceNetworkId = cmd.getNetworkId();
         final Long zone = cmd.getZoneId();
-        final String address = cmd.getIpAddress();
         final Long vlan = cmd.getVlanId();
         final Boolean forVirtualNetwork = cmd.isForVirtualNetwork();
-        final Boolean forLoadBalancing = cmd.isForLoadBalancing();
         final Long ipId = cmd.getId();
-        final Boolean sourceNat = cmd.isSourceNat();
-        final Boolean staticNat = cmd.isStaticNat();
+        final Long networkId = cmd.getNetworkId();
         final Long vpcId = cmd.getVpcId();
-        final Boolean forDisplay = cmd.getDisplay();
-        final Map<String, String> tags = cmd.getTags();
 
         final String state = cmd.getState();
         Boolean isAllocated = cmd.isAllocatedOnly();
         if (isAllocated == null) {
-            isAllocated = Boolean.TRUE;
-
-            if (state != null) {
+            if (state != null && state.equalsIgnoreCase(IpAddress.State.Free.name())) {
                 isAllocated = Boolean.FALSE;
+            } else {
+                isAllocated = Boolean.TRUE; // default
+            }
+        } else {
+            if (state != null && state.equalsIgnoreCase(IpAddress.State.Free.name())) {
+                if (isAllocated) {
+                    throw new InvalidParameterValueException("Conflict: allocatedonly is true but state is Free");
+                }
+            } else if (state != null && state.equalsIgnoreCase(IpAddress.State.Allocated.name())) {
+                isAllocated = Boolean.TRUE;
+            }
+        }
+
+        VlanType vlanType = null;
+        if (forVirtualNetwork != null) {
+            vlanType = forVirtualNetwork ? VlanType.VirtualNetwork : VlanType.DirectAttached;
+        } else {
+            vlanType = VlanType.VirtualNetwork;
+        }
+
+        final Account caller = getCaller();
+        List<IPAddressVO> addrs = new ArrayList<>();
+
+        if (vlanType == VlanType.DirectAttached && networkId == null && ipId == null) { // only root admin can list public ips in all shared networks
+            if (caller.getType() != Account.ACCOUNT_TYPE_ADMIN) {
+                isAllocated = true;
+            }
+        } else if (vlanType == VlanType.DirectAttached) {
+            // list public ip address on shared network
+            // access control. admin: all Ips, domain admin/user: all Ips in shared network in the domain/sub-domain/user
+            NetworkVO network = null;
+            if (networkId == null) {
+                IPAddressVO ip = _publicIpAddressDao.findById(ipId);
+                if (ip == null) {
+                    throw new InvalidParameterValueException("Please specify a valid ipaddress id");
+                }
+                network = _networkDao.findById(ip.getSourceNetworkId());
+            } else {
+                network = _networkDao.findById(networkId);
+            }
+            if (network == null || network.getGuestType() != Network.GuestType.Shared) {
+                throw new InvalidParameterValueException("Please specify a valid network id");
+            }
+            if (network.getAclType() == ControlledEntity.ACLType.Account) {
+                NetworkAccountVO networkMap = _networkAccountDao.getAccountNetworkMapByNetworkId(network.getId());
+                if (networkMap == null) {
+                    return new Pair<>(addrs, 0);
+                }
+                _accountMgr.checkAccess(caller, null, false, _accountDao.findById(networkMap.getAccountId()));
+            } else { // Domain level
+                NetworkDomainVO networkMap = _networkDomainDao.getDomainNetworkMapByNetworkId(network.getId());
+                if (networkMap == null) {
+                    return new Pair<>(addrs, 0);
+                }
+                if (caller.getType() == Account.ACCOUNT_TYPE_NORMAL || caller.getType() == Account.ACCOUNT_TYPE_PROJECT) {
+                    if (_networkMgr.isNetworkAvailableInDomain(network.getId(), caller.getDomainId())) {
+                        isAllocated = Boolean.TRUE;
+                    } else {
+                        return new Pair<>(addrs, 0);
+                    }
+                } else if (caller.getType() == Account.ACCOUNT_TYPE_DOMAIN_ADMIN || caller.getType() == Account.ACCOUNT_TYPE_RESOURCE_DOMAIN_ADMIN) {
+                    if (caller.getDomainId() == networkMap.getDomainId() || _domainDao.isChildDomain(caller.getDomainId(), networkMap.getDomainId())) {
+                        s_logger.debug("Caller " + caller.getUuid() + " has permission to access the network : " + network.getUuid());
+                    } else {
+                        if (_networkMgr.isNetworkAvailableInDomain(network.getId(), caller.getDomainId())) {
+                            isAllocated = Boolean.TRUE;
+                        } else {
+                            return new Pair<>(addrs, 0);
+                        }
+                    }
+                }
             }
         }
 
@@ -2034,12 +2114,10 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         final SearchBuilder<IPAddressVO> sb = _publicIpAddressDao.createSearchBuilder();
         Long domainId = null;
         Boolean isRecursive = null;
-        final List<Long> permittedAccounts = new ArrayList<Long>();
+        final List<Long> permittedAccounts = new ArrayList<>();
         ListProjectResourcesCriteria listProjectResourcesCriteria = null;
-        if (isAllocated) {
-            final Account caller = getCaller();
-
-            final Ternary<Long, Boolean, ListProjectResourcesCriteria> domainIdRecursiveListProject = new Ternary<Long, Boolean, ListProjectResourcesCriteria>(cmd.getDomainId(), cmd.isRecursive(),
+        if (isAllocated || (vlanType == VlanType.VirtualNetwork && (caller.getType() != Account.ACCOUNT_TYPE_ADMIN || cmd.getDomainId() != null))) {
+            final Ternary<Long, Boolean, ListProjectResourcesCriteria> domainIdRecursiveListProject = new Ternary<>(cmd.getDomainId(), cmd.isRecursive(),
                     null);
             _accountMgr.buildACLSearchParameters(caller, cmd.getId(), cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false);
             domainId = domainIdRecursiveListProject.first();
@@ -2048,12 +2126,93 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
             _accountMgr.buildACLSearchBuilder(sb, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
         }
 
+        buildParameters(sb, cmd);
+
+        SearchCriteria<IPAddressVO> sc = sb.create();
+        setParameters(sc, cmd, vlanType);
+
+        if (isAllocated || (vlanType == VlanType.VirtualNetwork && (caller.getType() != Account.ACCOUNT_TYPE_ADMIN || cmd.getDomainId() != null))) {
+            _accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
+        }
+
+        if (associatedNetworkId != null) {
+            _accountMgr.checkAccess(caller, null, false, _networkDao.findById(associatedNetworkId));
+            sc.setParameters("associatedNetworkIdEq", associatedNetworkId);
+        }
+        if (vpcId != null) {
+            _accountMgr.checkAccess(caller, null, false, _vpcDao.findById(vpcId));
+            sc.setParameters("vpcId", vpcId);
+        }
+
+        addrs = _publicIpAddressDao.search(sc, searchFilter); // Allocated
+
+        // Free IP addresses in system IP ranges
+        List<Long> freeAddrIds = new ArrayList<>();
+        if (!(isAllocated || vlanType == VlanType.DirectAttached)) {
+            Long zoneId = zone;
+            Account owner = _accountMgr.finalizeOwner(CallContext.current().getCallingAccount(), cmd.getAccountName(), cmd.getDomainId(), cmd.getProjectId());
+            if (associatedNetworkId != null) {
+                NetworkVO guestNetwork = _networkDao.findById(associatedNetworkId);
+                if (zoneId == null) {
+                    zoneId = guestNetwork.getDataCenterId();
+                } else if (zoneId != guestNetwork.getDataCenterId()) {
+                    InvalidParameterValueException ex = new InvalidParameterValueException("Please specify a valid associated network id in the specified zone.");
+                    throw ex;
+                }
+                owner = _accountDao.findById(guestNetwork.getAccountId());
+            }
+            List<DataCenterVO> dcList = new ArrayList<>();
+            if (zoneId == null){
+                dcList = ApiDBUtils.listZones();
+            } else {
+                dcList.add(ApiDBUtils.findZoneById(zoneId));
+            }
+            List<Long> vlanDbIds = null;
+            if (vlan != null) {
+                vlanDbIds = new ArrayList<>();
+                vlanDbIds.add(vlan);
+            }
+            List<IPAddressVO> freeAddrs = new ArrayList<>();
+            for (DataCenterVO dc : dcList) {
+                long dcId = dc.getId();
+                try {
+                    freeAddrs.addAll(_ipAddressMgr.listAvailablePublicIps(dcId, null, vlanDbIds, owner, VlanType.VirtualNetwork, associatedNetworkId,
+                            false, false, false, null, false, cmd.getVpcId(), cmd.isDisplay(), false, false)); // Free
+                } catch (InsufficientAddressCapacityException e) {
+                    s_logger.warn("no free address is found in zone " + dcId);
+                }
+            }
+            for (IPAddressVO addr: freeAddrs) {
+                freeAddrIds.add(addr.getId());
+            }
+        }
+        if (freeAddrIds.size() > 0) {
+            final SearchBuilder<IPAddressVO> sb2 = _publicIpAddressDao.createSearchBuilder();
+            buildParameters(sb2, cmd);
+            sb2.and("ids", sb2.entity().getId(), SearchCriteria.Op.IN);
+
+            SearchCriteria<IPAddressVO> sc2 = sb2.create();
+            setParameters(sc2, cmd, vlanType);
+            sc2.setParameters("ids", freeAddrIds.toArray());
+            addrs.addAll(_publicIpAddressDao.search(sc2, searchFilter)); // Allocated + Free
+        }
+
+        return new Pair<>(addrs, addrs.size());
+    }
+
+    private void buildParameters(final SearchBuilder<IPAddressVO> sb, final ListPublicIpAddressesCmd cmd) {
+        final Object keyword = cmd.getKeyword();
+        final String address = cmd.getIpAddress();
+        final Boolean forLoadBalancing = cmd.isForLoadBalancing();
+        Boolean isAllocated = cmd.isAllocatedOnly();
+        final Map<String, String> tags = cmd.getTags();
+
         sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
         sb.and("address", sb.entity().getAddress(), SearchCriteria.Op.EQ);
         sb.and("vlanDbId", sb.entity().getVlanId(), SearchCriteria.Op.EQ);
         sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ);
         sb.and("physicalNetworkId", sb.entity().getPhysicalNetworkId(), SearchCriteria.Op.EQ);
-        sb.and("associatedNetworkId", sb.entity().getAssociatedWithNetworkId(), SearchCriteria.Op.EQ);
+        sb.and("associatedNetworkIdEq", sb.entity().getAssociatedWithNetworkId(), SearchCriteria.Op.EQ);
         sb.and("sourceNetworkId", sb.entity().getSourceNetworkId(), SearchCriteria.Op.EQ);
         sb.and("isSourceNat", sb.entity().isSourceNat(), SearchCriteria.Op.EQ);
         sb.and("isStaticNat", sb.entity().isOneToOneNat(), SearchCriteria.Op.EQ);
@@ -2089,21 +2248,24 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         vlanSearch.and("removed", vlanSearch.entity().getRemoved(), SearchCriteria.Op.NULL);
         sb.join("vlanSearch", vlanSearch, sb.entity().getVlanId(), vlanSearch.entity().getId(), JoinBuilder.JoinType.INNER);
 
-        if (isAllocated != null && isAllocated == true) {
+        if (isAllocated != null && isAllocated) {
             sb.and("allocated", sb.entity().getAllocatedTime(), SearchCriteria.Op.NNULL);
         }
+    }
 
-        VlanType vlanType = null;
-        if (forVirtualNetwork != null) {
-            vlanType = forVirtualNetwork ? VlanType.VirtualNetwork : VlanType.DirectAttached;
-        } else {
-            vlanType = VlanType.VirtualNetwork;
-        }
-
-        final SearchCriteria<IPAddressVO> sc = sb.create();
-        if (isAllocated) {
-            _accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
-        }
+    private void setParameters(SearchCriteria<IPAddressVO> sc, final ListPublicIpAddressesCmd cmd, VlanType vlanType) {
+        final Object keyword = cmd.getKeyword();
+        final Long physicalNetworkId = cmd.getPhysicalNetworkId();
+        final Long sourceNetworkId = cmd.getNetworkId();
+        final Long zone = cmd.getZoneId();
+        final String address = cmd.getIpAddress();
+        final Long vlan = cmd.getVlanId();
+        final Long ipId = cmd.getId();
+        final Boolean sourceNat = cmd.isSourceNat();
+        final Boolean staticNat = cmd.isStaticNat();
+        final Boolean forDisplay = cmd.getDisplay();
+        final String state = cmd.getState();
+        final Map<String, String> tags = cmd.getTags();
 
         sc.setJoinParameters("vlanSearch", "vlanType", vlanType);
 
@@ -2121,10 +2283,6 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
             sc.setParameters("dataCenterId", zone);
         }
 
-        if (vpcId != null) {
-            sc.setParameters("vpcId", vpcId);
-        }
-
         if (ipId != null) {
             sc.setParameters("id", ipId);
         }
@@ -2153,10 +2311,6 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
             sc.setParameters("physicalNetworkId", physicalNetworkId);
         }
 
-        if (associatedNetworkId != null) {
-            sc.setParameters("associatedNetworkId", associatedNetworkId);
-        }
-
         if (sourceNetworkId != null) {
             sc.setParameters("sourceNetworkId", sourceNetworkId);
         }
@@ -2170,8 +2324,6 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         }
 
         sc.setParameters( "forsystemvms", false);
-        final Pair<List<IPAddressVO>, Integer> result = _publicIpAddressDao.searchAndCount(sc, searchFilter);
-        return new Pair<List<? extends IpAddress>, Integer>(result.first(), result.second());
     }
 
     @Override
diff --git a/test/integration/component/test_public_ip.py b/test/integration/component/test_public_ip.py
new file mode 100644
index 00000000000..a37226f32fc
--- /dev/null
+++ b/test/integration/component/test_public_ip.py
@@ -0,0 +1,870 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+""" Tests for acquiring/listing public ip address
+"""
+from nose.plugins.attrib import attr
+from marvin.cloudstackTestCase import cloudstackTestCase
+from marvin.lib.utils import cleanup_resources
+from marvin.lib.base import (Account,
+                             Configurations,
+                             Domain,
+                             PublicIpRange,
+                             PublicIPAddress,
+                             Network,
+                             NetworkOffering,
+                             ServiceOffering,
+                             VPC,
+                             VpcOffering,
+                             Zone)
+from marvin.lib.common import (get_zone,
+                               get_domain,
+                               get_template,
+                               get_free_vlan
+                              )
+import logging
+import random
+
+class TestPublicIp(cloudstackTestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.testClient = super(TestPublicIp, cls).getClsTestClient()
+        cls.apiclient = cls.testClient.getApiClient()
+        cls.services = cls.testClient.getParsedTestDataConfig()
+
+        # Get Zone, Domain and templates
+        cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests())
+        cls.services['mode'] = cls.zone.networktype
+        cls.logger = logging.getLogger("TestPublicIp")
+        cls.domain = get_domain(cls.apiclient)
+        cls.services["virtual_machine"]["zoneid"] = cls.zone.id
+
+        cls.template = get_template(
+            cls.apiclient,
+            cls.zone.id,
+            cls.services["ostype"]
+        )
+
+        cls.use_system_ips_config_name = "use.system.public.ips"
+        cls.use_system_ips_config = Configurations.list(
+            cls.apiclient,
+            name=cls.use_system_ips_config_name
+        )
+        cls.use_system_ips_config_value = cls.use_system_ips_config[0].value
+        Configurations.update(
+            cls.apiclient,
+            name=cls.use_system_ips_config_name,
+            value="false"
+        )
+
+        cls._cleanup = []
+        cls.unsupportedHypervisor = False
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
+        if cls.hypervisor.lower() in ['lxc']:
+            cls.unsupportedHypervisor = True
+            return
+
+        # Create new domain1
+        cls.domain1 = Domain.create(
+            cls.apiclient,
+            services=cls.services["acl"]["domain1"],
+            parentdomainid=cls.domain.id)
+
+        # Create account1
+        cls.account1 = Account.create(
+            cls.apiclient,
+            cls.services["account"],
+            # cls.services["acl"]["accountD1"],
+            admin=True,
+            domainid=cls.domain1.id
+        )
+
+        # Create new sub-domain
+        cls.sub_domain = Domain.create(
+            cls.apiclient,
+            services=cls.services["acl"]["domain11"],
+            parentdomainid=cls.domain1.id)
+
+        # Create account for sub-domain
+        cls.sub_account = Account.create(
+            cls.apiclient,
+            cls.services["acl"]["accountD11"],
+            domainid=cls.sub_domain.id
+        )
+
+        # Create new domain2
+        cls.domain2 = Domain.create(
+            cls.apiclient,
+            services=cls.services["acl"]["domain2"],
+            parentdomainid=cls.domain.id)
+
+        # Create account2
+        cls.account2 = Account.create(
+            cls.apiclient,
+            cls.services["acl"]["accountD2"],
+            domainid=cls.domain2.id
+        )
+
+        cls.services["publiciprange"]["zoneid"] = cls.zone.id
+        cls.services["publiciprange"]["forvirtualnetwork"] = "true"
+
+        # Create public ip range 1
+        cls.services["publiciprange"]["vlan"] = get_free_vlan(
+            cls.apiclient,
+            cls.zone.id)[1]
+        random_subnet_number = random.randrange(10,20)
+        cls.services["publiciprange"]["gateway"] = "10.100." + \
+                                                   str(random_subnet_number) + ".254"
+        cls.services["publiciprange"]["startip"] = "10.100." + \
+                                                   str(random_subnet_number) + ".1"
+        cls.services["publiciprange"]["endip"] = "10.100." + \
+                                                 str(random_subnet_number) + ".10"
+        cls.services["publiciprange"]["netmask"] = "255.255.255.0"
+        cls.public_ip_range1 = PublicIpRange.create(
+            cls.apiclient,
+            cls.services["publiciprange"],
+            account=cls.account1.name,
+            domainid=cls.account1.domainid
+        )
+
+        # dedicate ip range to sub domain
+        cls.services["publiciprange"]["vlan"] = get_free_vlan(
+            cls.apiclient,
+            cls.zone.id)[1]
+        random_subnet_number = random.randrange(10,20)
+        cls.services["publiciprange"]["gateway"] = "10.110." + \
+                                                    str(random_subnet_number) + ".254"
+        cls.services["publiciprange"]["startip"] = "10.110." + \
+                                                    str(random_subnet_number) + ".1"
+        cls.services["publiciprange"]["endip"] = "10.110." + \
+                                                  str(random_subnet_number) + ".10"
+        cls.services["publiciprange"]["netmask"] = "255.255.255.0"
+        cls.public_ip_range2 = PublicIpRange.create(
+            cls.apiclient,
+            cls.services["publiciprange"],
+            account=cls.sub_account.name,
+            domainid=cls.sub_account.domainid
+        )
+
+        # dedicate ip range to second domain
+        cls.services["publiciprange"]["vlan"] = get_free_vlan(
+            cls.apiclient,
+            cls.zone.id)[1]
+        random_subnet_number = random.randrange(10,20)
+        cls.services["publiciprange"]["gateway"] = "10.120." + \
+                                                   str(random_subnet_number) + ".254"
+        cls.services["publiciprange"]["startip"] = "10.120." + \
+                                                   str(random_subnet_number) + ".1"
+        cls.services["publiciprange"]["endip"] = "10.120." + \
+                                                 str(random_subnet_number) + ".10"
+        cls.services["publiciprange"]["netmask"] = "255.255.255.0"
+        cls.public_ip_range3 = PublicIpRange.create(
+            cls.apiclient,
+            cls.services["publiciprange"],
+            account=cls.account2.name,
+            domainid=cls.account2.domainid
+        )
+
+        # create vpc offering and VPC
+        cls.vpc_off = VpcOffering.create(
+            cls.apiclient,
+            cls.services["vpc_offering"]
+        )
+        cls.vpc_off.update(cls.apiclient, state='Enabled')
+
+        # create network offering
+        cls.isolated_network_offering = NetworkOffering.create(
+            cls.apiclient,
+            cls.services["isolated_network_offering"],
+            conservemode=False
+        )
+
+        NetworkOffering.update(
+            cls.isolated_network_offering,
+            cls.apiclient,
+            id=cls.isolated_network_offering.id,
+            state="enabled"
+        )
+
+        physical_network, shared_vlan = get_free_vlan(
+            cls.apiclient, cls.zone.id)
+        if shared_vlan is None:
+            cls.fail("Failed to get free vlan id for shared network")
+
+        cls.services["shared_network_offering"]["specifyVlan"] = "True"
+        cls.services["shared_network_offering"]["specifyIpRanges"] = "True"
+
+        cls.shared_network_offering = NetworkOffering.create(
+            cls.apiclient,
+            cls.services["shared_network_offering"],
+            conservemode=False
+        )
+
+        NetworkOffering.update(
+            cls.shared_network_offering,
+            cls.apiclient,
+            id=cls.shared_network_offering.id,
+            state="enabled"
+        )
+
+        # create network using the shared network offering created
+        cls.services["shared_network"]["acltype"] = "Domain"
+        cls.services["shared_network"][
+            "networkofferingid"] = cls.shared_network_offering.id
+        cls.services["shared_network"][
+            "physicalnetworkid"] = physical_network.id
+        cls.services["shared_network"]["vlan"] = shared_vlan
+        shared_network_subnet_number = random.randrange(1, 254)
+
+        cls.services["shared_network"]["netmask"] = "255.255.255.0"
+        cls.services["shared_network"]["gateway"] = "172.16." + \
+                                                     str(shared_network_subnet_number) + ".254"
+        cls.services["shared_network"]["startip"] = "172.16." + \
+                                                     str(shared_network_subnet_number) + ".1"
+        cls.services["shared_network"]["endip"] = "172.16." + \
+                                                   str(shared_network_subnet_number) + ".10"
+
+        cls.guest_network = Network.create(
+            cls.apiclient,
+            cls.services["shared_network"],
+            networkofferingid=cls.shared_network_offering.id,
+            zoneid=cls.zone.id
+        )
+
+        cls._cleanup.append(cls.guest_network)
+        cls._cleanup.append(cls.shared_network_offering)
+        cls._cleanup.append(cls.account1)
+        cls._cleanup.append(cls.account2)
+        cls._cleanup.append(cls.sub_account)
+        cls._cleanup.append(cls.sub_domain)
+        cls._cleanup.append(cls.domain1)
+        cls._cleanup.append(cls.domain2)
+        cls._cleanup.append(cls.public_ip_range1)
+        cls._cleanup.append(cls.public_ip_range2)
+        cls._cleanup.append(cls.public_ip_range3)
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            cls.apiclient = super(
+                TestPublicIp,
+                cls).getClsTestClient().getApiClient()
+
+            Configurations.update(
+                cls.apiclient,
+                name=cls.use_system_ips_config_name,
+                value=cls.use_system_ips_config_value
+            )
+            # Cleanup resources used
+            cleanup_resources(cls.apiclient, cls._cleanup)
+
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+
+        return
+
+    def setUp(self):
+        self.apiclient = self.testClient.getApiClient()
+        self.cleanup = []
+        return
+
+    def tearDown(self):
+        try:
+            # Clean up, terminate the resources created
+            cleanup_resources(self.apiclient, self.cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    @attr(tags=["advanced", "basic", "sg"], required_hardware="false")
+    def test_01_list_publicip_root_domain(self):
+        """
+        ROOT domain should be able to list public IP address of all
+        sub domains
+
+        Step 1: List all public ip address
+        Step 2: Ensure that the count is greater than or equal to 30
+        Step 3: Display only allocated ip address
+        Step 4: Ensure that the count is greater than or equal to 0
+        Step 5: Display ip address from shared networks
+        Step 6: Ensure that the count is greater than or equal to 30
+        :return:
+        """
+        # Step 1
+        ipAddresses = PublicIPAddress.list(
+            self.apiclient,
+            allocatedonly=False,
+            isrecursive=True,
+            listall=True,
+            forvirtualnetwork=True)
+
+        # Step 2
+        self.assertGreaterEqual(
+            len(ipAddresses),
+            30,
+            "Unable to display all public ip for ROOT domain"
+        )
+
+        # Step 4
+        ipAddresses = PublicIPAddress.list(
+            self.apiclient,
+            allocatedonly=True,
+            isrecursive=True,
+            listall=True,
+            forvirtualnetwork=True)
+
+        self.assertGreaterEqual(
+            len(ipAddresses),
+            0,
+            "Unable to display all public ip for ROOT domain"
+        )
+
+        # Step 5
+        ipAddresses = PublicIPAddress.list(
+            self.apiclient,
+            allocatedonly=False,
+            isrecursive=True,
+            listall=True,
+            forvirtualnetwork=False)
+
+        # Step 6
+        self.assertGreaterEqual(
+            len(ipAddresses),
+            10,
+            "Unable to display all public ip for ROOT domain"
+        )
+
+    @attr(tags=["advanced", "basic", "sg"], required_hardware="false")
+    def test_02_list_publicip_domain_admin(self):
+        """
+        A domain admin should be able to list public IP address of
+        his/her domain and all sub domains
+
+        Step 1: Create an isolated network in the user domain and sub domain
+        Step 2: Associate IP in the range, dedicated to domain1
+        Step 3: Display all public ip address in domain1
+        Step 4: Ensure that the count is greater than or equal to 10
+        Step 5: Display only the allocated Ip address in domain1
+        Step 6: Ensure that the count is 1
+        Step 7: Try to display public ip address from shared networks
+        Step 8: It should not return any result
+        Step 9: Try to display ip address of domain1 from domain2
+        Step 10: Ensure that it doesnt not return any result
+        :return:
+        """
+        # Step 1. Create isolated network
+        self.isolated_network1 = Network.create(
+            self.apiclient,
+            self.services["isolated_network"],
+            accountid=self.account1.name,
+            domainid=self.account1.domainid,
+            networkofferingid=self.isolated_network_offering.id,
+            zoneid=self.zone.id
+        )
+
+        # Step 2. Associate IP in range dedicated to domain1
+        ip_address_1 = self.get_free_ipaddress(self.public_ip_range1.vlan.id)
+        ipaddress = PublicIPAddress.create(
+            self.apiclient,
+            zoneid=self.zone.id,
+            networkid=self.isolated_network1.id,
+            ipaddress=ip_address_1
+        )
+        self.assertIsNotNone(
+            ipaddress,
+            "Failed to Associate IP Address"
+        )
+
+        # Step 3: Display all public ip address in domain1
+        user = self.account1.user[0]
+        user_api_client = self.testClient.getUserApiClient(user.username, self.domain1.name)
+
+        ipAddresses = PublicIPAddress.list(
+            user_api_client,
+            allocatedonly=False,
+            isrecursive=True,
+            listall=True,
+            forvirtualnetwork=True)
+
+        # Step 4: Ensure that the count is equal to 10
+        self.assertEqual(
+            len(ipAddresses),
+            10,
+            "Failed to display all public ip address is domain %s" % self.domain1.name
+        )
+
+        # Display public ip address using network id
+        ipAddresses = PublicIPAddress.list(
+            self.apiclient,
+            associatednetworkid=self.isolated_network1.id,
+            account=self.account1.name,
+            domainid=self.account1.domainid,
+            allocatedonly=False,
+            listall=True,
+            forvirtualnetwork=True)
+
+        # Step 4: Ensure that the count is greater than or equal to 10
+        self.assertEqual(
+            len(ipAddresses),
+            10,
+            "Failed to display all public ip address using network id"
+        )
+
+        # Step 5: Display all allocated public ip address in domain1
+        ipAddresses = PublicIPAddress.list(
+            user_api_client,
+            allocatedonly=True,
+            isrecursive=True,
+            listall=True,
+            forvirtualnetwork=True)
+
+        # Step 6: Ensure that the count is greater than or equal to 1
+        self.assertEqual(
+            len(ipAddresses),
+            1,
+            "Allocated IP address is greater than 1"
+        )
+
+        # Step 7: Display public ip address from shared networks
+        ipAddresses = PublicIPAddress.list(
+            user_api_client,
+            allocatedonly=True,
+            isrecursive=True,
+            listall=True,
+            forvirtualnetwork=False)
+
+        # Step 8: Ensure that the result is empty
+        self.assertIsNone(
+            ipAddresses,
+            "Users should not display ip from shared networks"
+        )
+
+        try:
+            # Step 9
+            user = self.account2.user[0]
+            user_api_client = self.testClient.getUserApiClient(user.username, self.domain2.name)
+
+            ipAddresses = PublicIPAddress.list(
+                user_api_client,
+                allocatedonly=False,
+                listall=True,
+                associatednetworkid=self.isolated_network1.id,
+                forvirtualnetwork=True)
+
+            # Step 10
+            self.fail("Domain should not access public ip of sibling domain")
+        except Exception as e:
+            self.info("Got exception as expected since domain2 cant access network of domain1")
+
+    @attr(tags=["advanced", "basic", "sg"], required_hardware="false")
+    def test_03_list_publicip_user_domain(self):
+        """
+        A regular user should be able to display public ip address
+        only in his domain
+
+        Step 1: Create an isolated network in the user domain
+        Step 2: Display all public ip address in that domain
+        Step 3: Ensure that the count is 10
+        Step 4: Associate IP in the range, dedicated to domain2
+        Step 5: Display only the allocated Ip address in domain2
+        Step 6: Ensure that the count is 1
+        Step 7: Try to display public ip address from shared networks
+        Step 8: It should not return any result
+        Step 9: Try to display allocated public ip address of child domain using networkid
+        Step 10: It should display all allocated ip address from child domain
+        Step 11: Try to display all public ip address from child domain
+        Step 12: It should display all public ip of child domain
+        Step 13: Try to display ip of domain2 from a different domain
+        Step 14: Ensure that we get exception when trying to do so
+        :return:
+        """
+        user = self.sub_account.user[0]
+        sub_user_api_client = self.testClient.getUserApiClient(user.username, self.sub_domain.name)
+
+        # Step 1: create network in child domain
+        self.isolated_network2 = Network.create(
+            self.apiclient,
+            self.services["isolated_network"],
+            accountid=self.sub_account.name,
+            domainid=self.sub_account.domainid,
+            networkofferingid=self.isolated_network_offering.id,
+            zoneid=self.zone.id
+        )
+
+        # Step 2: Display all public ip address in sub domain
+        ipAddresses = PublicIPAddress.list(
+            sub_user_api_client,
+            allocatedonly=False,
+            listall=True,
+            forvirtualnetwork=True)
+
+        # Step 3: Ensure that sub domain can list only the ip address in his domain
+        self.assertEqual(
+            len(ipAddresses),
+            10,
+            "Allocated IP address is greater than 1"
+        )
+
+        # Step 4: Associate IP in range dedicated to sub domain
+        ip_address_1 = self.get_free_ipaddress(self.public_ip_range2.vlan.id)
+        ipaddress = PublicIPAddress.create(
+            sub_user_api_client,
+            zoneid=self.zone.id,
+            networkid=self.isolated_network2.id,
+            ipaddress=ip_address_1
+        )
+
+        # Step 5: Display all allocated ip address in sub domain
+        ipAddresses = PublicIPAddress.list(
+            sub_user_api_client,
+            allocatedonly=True,
+            listall=True,
+            forvirtualnetwork=True)
+
+        # Step 6: Ensure that the count is 1
+        self.assertEqual(
+            len(ipAddresses),
+            1,
+            "Allocated IP address is greater than 1"
+        )
+
+        # Step 7: Display ip address from shared networks
+        ipAddresses = PublicIPAddress.list(
+            sub_user_api_client,
+            allocatedonly=True,
+            listall=True,
+            forvirtualnetwork=False)
+
+        # Step 8: It should not return any result
+        self.assertIsNone(
+            ipAddresses,
+            "Users should not display ip from shared networks"
+        )
+
+        user = self.account1.user[0]
+        user_api_client = self.testClient.getUserApiClient(user.username, self.domain1.name)
+
+        # Step 9: display ip of child domain using network id
+        ipAddresses = PublicIPAddress.list(
+            user_api_client,
+            allocatedonly=True,
+            listall=True,
+            isrecursive=True,
+            associatednetworkid=self.isolated_network2.id,
+            forvirtualnetwork=True)
+
+        # Step 10: Ensure that the count is 1 as only 1 ip is acquired in test 3
+        self.assertEqual(
+            len(ipAddresses),
+            1,
+            "Unable to display IP address of child domain using network id"
+        )
+
+        # Step 11: display ip of child domain using network id
+        ipAddresses = PublicIPAddress.list(
+            user_api_client,
+            allocatedonly=False,
+            listall=True,
+            isrecursive=True,
+            associatednetworkid=self.isolated_network2.id,
+            forvirtualnetwork=True)
+
+        # Step 12: Ensure that the count is 1 as only 1 ip is acquired in test 3
+        self.assertEqual(
+            len(ipAddresses),
+            10,
+            "Unable to display IP address of child domain using network id"
+        )
+
+        try:
+            # Step 13
+            user = self.account2.user[0]
+            user_api_client = self.testClient.getUserApiClient(user.username, self.domain2.name)
+
+            PublicIPAddress.list(
+                user_api_client,
+                allocatedonly=False,
+                listall=True,
+                associatednetworkid=self.isolated_network2.id,
+                forvirtualnetwork=True)
+
+            # Step 14
+            self.fail("Domain should not access public ip of sibling domain")
+        except Exception as e:
+            self.info("Got exception as expected since domain2 cant access network of domain1")
+
+    @attr(tags=["advanced", "basic", "sg"], required_hardware="false")
+    def test_04_list_publicip_all_subdomains(self):
+        """
+        A domain admin should be able to display public ip address
+        in his domain and also all child domains
+
+        Step 1: Display all public ip address in that domain and sub domain
+        Step 2: Ensure that the count is 11 (all ip from parent domain and allocated from sub domain)
+        Step 3: Display only the allocated Ip address
+        Step 4: Ensure that the count is 2
+        Step 5: Display public ip of child domain using domain/account
+        Step 6: Ensure that the count is 10
+        Step 7: Display public ip of child domain using network id
+        Step 8: Ensure that the count is 1 as only one IP is allocated
+        :return:
+        """
+        user = self.account1.user[0]
+        user_api_client = self.testClient.getUserApiClient(user.username, self.domain1.name)
+
+        # Step 1: Display all public ip
+        ipAddresses = PublicIPAddress.list(
+            user_api_client,
+            allocatedonly=False,
+            listall=True,
+            isrecursive=True,
+            forvirtualnetwork=True)
+
+        # Step 2: Ensure that it can display all ip from current domain and
+        # only allocated ip from child domains
+        self.assertEqual(
+            len(ipAddresses),
+            11,
+            "Unable to display all IP address is domain %s" % self.domain1.name
+        )
+
+        # Step 3: display only allocated ip for parent and sub domain
+        ipAddresses = PublicIPAddress.list(
+            user_api_client,
+            allocatedonly=True,
+            listall=True,
+            isrecursive=True,
+            forvirtualnetwork=True)
+
+        # Step 4: Ensure that the count is 2
+        self.assertEqual(
+            len(ipAddresses),
+            2,
+            "Unable to display all allocated IP address is domain %s" % self.domain1.name
+        )
+
+        # Step 5: display ip of child domain using domainid/account
+        ipAddresses = PublicIPAddress.list(
+            user_api_client,
+            allocatedonly=False,
+            listall=True,
+            isrecursive=True,
+            domainid=self.sub_domain.id,
+            account=self.sub_account.name,
+            forvirtualnetwork=True)
+
+        # Step 6: Ensure that the count is 10
+        self.assertEqual(
+            len(ipAddresses),
+            10,
+            "Unable to display IP address of child domain"
+        )
+
+    @attr(tags=["advanced", "basic", "sg"], required_hardware="false")
+    def test_05_list_publicip_user_domain(self):
+        """
+        A domain admin should be able to display public ip address
+        in his domain and also all child domains
+
+        Step 1: Display all public ip address in that domain and sub domain
+        Step 2: Ensure that the count is 20
+        Step 3: Display only the allocated Ip address
+        Step 4: Ensure that the count is 2
+        Step 5: Try to display public ip of vpc from different domain
+        Step 6: Ensure that we get exception when trying to do so
+        :return:
+        """
+        user = self.account2.user[0]
+        user_api_client = self.testClient.getUserApiClient(user.username, self.domain2.name)
+
+        self.services["vpc"]["cidr"] = "10.1.1.1/16"
+        vpc_1 = VPC.create(
+            user_api_client,
+            self.services["vpc"],
+            vpcofferingid=self.vpc_off.id,
+            zoneid=self.zone.id,
+            account=self.account2.name,
+            domainid=self.account2.domainid
+        )
+        self.validate_vpc_network(vpc_1)
+
+        ipAddresses = PublicIPAddress.list(
+            user_api_client,
+            allocatedonly=False,
+            listall=True,
+            forvirtualnetwork=True)
+
+        self.assertGreaterEqual(
+            len(ipAddresses),
+            10,
+            "Unable to display all public ip in VPC"
+        )
+
+        # List ip address using vpcid
+        ipAddresses = PublicIPAddress.list(
+            self.apiclient,
+            vpcid=vpc_1.id,
+            allocatedonly=False,
+            account=self.account2.name,
+            domainid=self.domain2.id,
+            forvirtualnetwork=True)
+
+        self.assertGreaterEqual(
+            len(ipAddresses),
+            10,
+            "Unable to display all public ip in VPC"
+        )
+
+        # Acquire public ip address from VPC
+        ip_address_1 = self.get_free_ipaddress(self.public_ip_range3.vlan.id)
+        PublicIPAddress.create(
+            user_api_client,
+            zoneid=self.zone.id,
+            vpcid=vpc_1.id,
+            ipaddress=ip_address_1
+        )
+
+        ipAddresses = PublicIPAddress.list(
+            user_api_client,
+            allocatedonly=True,
+            listall=True,
+            forvirtualnetwork=True)
+
+        self.assertGreaterEqual(
+            len(ipAddresses),
+            2,
+            "Unable to display allocated public ip in VPC"
+        )
+
+        try:
+            # Step 5
+            user = self.account1.user[0]
+            user_api_client = self.testClient.getUserApiClient(user.username, self.domain1.name)
+
+            PublicIPAddress.list(
+                user_api_client,
+                allocatedonly=False,
+                listall=True,
+                vpcid=vpc_1.id,
+                forvirtualnetwork=True)
+
+            # Step 6
+            self.fail("Domain should not access public ip of sibling domain")
+        except Exception as e:
+            self.info("Got exception as expected since domain2 cant access network of domain1")
+
+        self.vpc_off.update(self.apiclient, state='Disabled')
+        self.cleanup.append(vpc_1)
+        self.cleanup.append(self.vpc_off)
+
+    @attr(tags=["advanced", "basic", "sg"], required_hardware="false")
+    def test_06_list_publicip_user_domain(self):
+        """
+        Display public ip address from shared networks
+        1. List public ip address of shared network as root admin
+        2. Ensure that it can display all public ip address
+        3. List public ip address of shared networks as domain admin
+        4. It should not return any result
+        5. Try to display public ip by passing network id
+        6. Ensure that we get exception when trying to do so
+        :return:
+        """
+        # Step 1
+        ipAddresses = PublicIPAddress.list(
+            self.apiclient,
+            allocatedonly=False,
+            listall=True,
+            networkid=self.guest_network.id,
+            forvirtualnetwork=False)
+
+        # Step 2
+        self.assertGreaterEqual(
+            len(ipAddresses),
+            10,
+            "Unable to display allocated public ip in VPC"
+        )
+
+        user = self.account1.user[0]
+        user_api_client = self.testClient.getUserApiClient(user.username, self.domain1.name)
+
+        # Step 3
+        ipAddresses = PublicIPAddress.list(
+            user_api_client,
+            allocatedonly=False,
+            listall=True,
+            networkid=self.guest_network.id,
+            forvirtualnetwork=False)
+
+        # Step 4
+        self.assertIsNone(
+            ipAddresses,
+            "Unable to display allocated public ip in VPC"
+        )
+
+        try:
+            # Step 5
+            PublicIPAddress.list(
+                user_api_client,
+                allocatedonly=False,
+                listall=True,
+                associatednetworkid=self.guest_network.id,
+                forvirtualnetwork=True)
+
+            # Step 6
+            self.fail("Domain should not access public ip of sibling domain")
+        except Exception as e:
+            self.info("Got exception as expected since domain2 cant access network of domain1")
+
+    def get_free_ipaddress(self, vlanId):
+        ipaddresses = PublicIPAddress.list(
+            self.apiclient,
+            vlanid=vlanId,
+            state='Free'
+        )
+        self.assertEqual(
+            isinstance(ipaddresses, list),
+            True,
+            "List ipaddresses should return a valid response for Free ipaddresses"
+        )
+        random.shuffle(ipaddresses)
+        return ipaddresses[0].ipaddress
+
+    def validate_vpc_network(self, network, state=None):
+        """Validates the VPC network"""
+
+        self.debug("Check if the VPC network is created successfully?")
+        vpc_networks = VPC.list(
+            self.apiclient,
+            id=network.id
+        )
+        self.assertEqual(
+            isinstance(vpc_networks, list),
+            True,
+            "List VPC network should return a valid list"
+        )
+        self.assertEqual(
+            network.name,
+            vpc_networks[0].name,
+            "Name of the VPC network should match with listVPC data"
+        )
+        if state:
+            self.assertEqual(
+                vpc_networks[0].state,
+                state,
+                "VPC state should be '%s'" % state
+            )
+        self.debug("VPC network validated - %s" % network.name)
+        return
diff --git a/test/integration/smoke/test_network.py b/test/integration/smoke/test_network.py
index 99d96d7bdef..d98e8228417 100644
--- a/test/integration/smoke/test_network.py
+++ b/test/integration/smoke/test_network.py
@@ -224,7 +224,8 @@ class TestPublicIP(cloudstackTestCase):
         # 1.listPublicIpAddresses should no more return the released address
         list_pub_ip_addr_resp = list_publicIP(
             self.apiclient,
-            id=ip_address.ipaddress.id
+            id=ip_address.ipaddress.id,
+            allocatedonly=True
         )
         if list_pub_ip_addr_resp is None:
             return
@@ -276,7 +277,8 @@ class TestPublicIP(cloudstackTestCase):
 
         list_pub_ip_addr_resp = list_publicIP(
             self.apiclient,
-            id=ip_address.ipaddress.id
+            id=ip_address.ipaddress.id,
+            allocatedonly=True
         )
 
         self.assertEqual(
@@ -880,7 +882,8 @@ class TestReleaseIP(cloudstackTestCase):
         while retriesCount > 0:
             listResponse = list_publicIP(
                 self.apiclient,
-                id=self.ip_addr.id
+                id=self.ip_addr.id,
+                state="Allocated"
             )
             if listResponse is None:
                 isIpAddressDisassociated = True

From 63c91c145858387e0e03e2365b50eb2848b77147 Mon Sep 17 00:00:00 2001
From: Wei Zhou <w.zhou@global.leaseweb.com>
Date: Thu, 1 Apr 2021 09:13:06 +0200
Subject: [PATCH 08/12] server: Fix network statistics for vpc (#3944)

This contains 3 main changes
(1) add NETWORK_STATS_ethX for all nics with public ips in VPC VRs (current: NETWORK_STATS_eth1)
(2) DO NOT create records in user_statistics for each VPC tier (only one record per public nic per VPC VR)
(3) send NetworkUsageCommand before unplugging a NIC with public IPs from VPC VR
---
 .../VirtualNetworkApplianceService.java       |   3 +
 .../com/cloud/vm/dao/DomainRouterDaoImpl.java |  14 ++-
 .../VirtualNetworkApplianceManagerImpl.java   | 113 +++---------------
 .../network/rules/NicPlugInOutRules.java      |   8 ++
 .../rules/VirtualNetworkApplianceFactory.java |   7 ++
 ...MockVpcVirtualNetworkApplianceManager.java |   9 +-
 systemvm/debian/opt/cloud/bin/cs/CsAddress.py |   9 +-
 7 files changed, 56 insertions(+), 107 deletions(-)

diff --git a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java
index 98fb8be7c7a..39756ef259a 100644
--- a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java
+++ b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java
@@ -26,6 +26,7 @@ import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.network.router.VirtualRouter;
 import com.cloud.user.Account;
+import com.cloud.vm.Nic;
 
 public interface VirtualNetworkApplianceService {
     /**
@@ -74,4 +75,6 @@ public interface VirtualNetworkApplianceService {
      * @return
      */
     boolean performRouterHealthChecks(long routerId);
+
+    <T extends VirtualRouter> void collectNetworkStatistics(T router, Nic nic);
 }
diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/DomainRouterDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/DomainRouterDaoImpl.java
index 135f96f28f3..dc71f898970 100644
--- a/engine/schema/src/main/java/com/cloud/vm/dao/DomainRouterDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/vm/dao/DomainRouterDaoImpl.java
@@ -383,12 +383,14 @@ public class DomainRouterDaoImpl extends GenericDaoBase<DomainRouterVO, Long> im
                 final RouterNetworkVO routerNtwkMap = new RouterNetworkVO(router.getId(), guestNetwork.getId(), guestNetwork.getGuestType());
                 _routerNetworkDao.persist(routerNtwkMap);
                 //2) create user stats entry for the network
-                UserStatisticsVO stats =
-                        _userStatsDao.findBy(router.getAccountId(), router.getDataCenterId(), guestNetwork.getId(), null, router.getId(), router.getType().toString());
-                if (stats == null) {
-                    stats =
-                            new UserStatisticsVO(router.getAccountId(), router.getDataCenterId(), null, router.getId(), router.getType().toString(), guestNetwork.getId());
-                    _userStatsDao.persist(stats);
+                if (router.getVpcId() == null) {
+                    UserStatisticsVO stats =
+                            _userStatsDao.findBy(router.getAccountId(), router.getDataCenterId(), guestNetwork.getId(), null, router.getId(), router.getType().toString());
+                    if (stats == null) {
+                        stats =
+                                new UserStatisticsVO(router.getAccountId(), router.getDataCenterId(), null, router.getId(), router.getType().toString(), guestNetwork.getId());
+                        _userStatsDao.persist(stats);
+                    }
                 }
                 txn.commit();
             }
diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
index 1103ff9c058..2b6032f42d3 100644
--- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
+++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
@@ -732,100 +732,7 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
                 s_logger.debug("Found " + routers.size() + " running routers. ");
 
                 for (final DomainRouterVO router : routers) {
-                    final String privateIP = router.getPrivateIpAddress();
-
-                    if (privateIP != null) {
-                        final boolean forVpc = router.getVpcId() != null;
-                        final List<? extends Nic> routerNics = _nicDao.listByVmId(router.getId());
-                        for (final Nic routerNic : routerNics) {
-                            final Network network = _networkModel.getNetwork(routerNic.getNetworkId());
-                            // Send network usage command for public nic in VPC VR
-                            // Send network usage command for isolated guest nic of non) VPC VR
-
-                            //[TODO] Avoiding the NPE now, but I have to find out what is going on with the network. - Wilder Rodrigues
-                            if (network == null) {
-                                s_logger.error("Could not find a network with ID => " + routerNic.getNetworkId() + ". It might be a problem!");
-                                continue;
-                            }
-                            if (forVpc && network.getTrafficType() == TrafficType.Public || !forVpc && network.getTrafficType() == TrafficType.Guest
-                                    && network.getGuestType() == Network.GuestType.Isolated) {
-                                final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), forVpc, routerNic.getIPv4Address());
-                                final String routerType = router.getType().toString();
-                                final UserStatisticsVO previousStats = _userStatsDao.findBy(router.getAccountId(), router.getDataCenterId(), network.getId(),
-                                        forVpc ? routerNic.getIPv4Address() : null, router.getId(), routerType);
-                                NetworkUsageAnswer answer = null;
-                                try {
-                                    answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd);
-                                } catch (final Exception e) {
-                                    s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId(), e);
-                                    continue;
-                                }
-
-                                if (answer != null) {
-                                    if (!answer.getResult()) {
-                                        s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId()
-                                                + "; details: " + answer.getDetails());
-                                        continue;
-                                    }
-                                    try {
-                                        if (answer.getBytesReceived() == 0 && answer.getBytesSent() == 0) {
-                                            s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics");
-                                            continue;
-                                        }
-                                        final NetworkUsageAnswer answerFinal = answer;
-                                        Transaction.execute(new TransactionCallbackNoReturn() {
-                                            @Override
-                                            public void doInTransactionWithoutResult(final TransactionStatus status) {
-                                                final UserStatisticsVO stats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterId(), network.getId(),
-                                                        forVpc ? routerNic.getIPv4Address() : null, router.getId(), routerType);
-                                                if (stats == null) {
-                                                    s_logger.warn("unable to find stats for account: " + router.getAccountId());
-                                                    return;
-                                                }
-
-                                                if (previousStats != null
-                                                        && (previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived() || previousStats.getCurrentBytesSent() != stats
-                                                        .getCurrentBytesSent())) {
-                                                    s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + "Ignoring current answer. Router: "
-                                                            + answerFinal.getRouterName() + " Rcvd: " + answerFinal.getBytesReceived() + "Sent: " + answerFinal.getBytesSent());
-                                                    return;
-                                                }
-
-                                                if (stats.getCurrentBytesReceived() > answerFinal.getBytesReceived()) {
-                                                    if (s_logger.isDebugEnabled()) {
-                                                        s_logger.debug("Received # of bytes that's less than the last one.  "
-                                                                + "Assuming something went wrong and persisting it. Router: " + answerFinal.getRouterName() + " Reported: "
-                                                                + toHumanReadableSize(answerFinal.getBytesReceived()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesReceived()));
-                                                    }
-                                                    stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived());
-                                                }
-                                                stats.setCurrentBytesReceived(answerFinal.getBytesReceived());
-                                                if (stats.getCurrentBytesSent() > answerFinal.getBytesSent()) {
-                                                    if (s_logger.isDebugEnabled()) {
-                                                        s_logger.debug("Received # of bytes that's less than the last one.  "
-                                                                + "Assuming something went wrong and persisting it. Router: " + answerFinal.getRouterName() + " Reported: "
-                                                                + toHumanReadableSize(answerFinal.getBytesSent()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesSent()));
-                                                    }
-                                                    stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent());
-                                                }
-                                                stats.setCurrentBytesSent(answerFinal.getBytesSent());
-                                                if (!_dailyOrHourly) {
-                                                    // update agg bytes
-                                                    stats.setAggBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent());
-                                                    stats.setAggBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived());
-                                                }
-                                                _userStatsDao.update(stats.getId(), stats);
-                                            }
-                                        });
-
-                                    } catch (final Exception e) {
-                                        s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + toHumanReadableSize(answer.getBytesReceived()) + "; Tx: "
-                                                + toHumanReadableSize(answer.getBytesSent()));
-                                    }
-                                }
-                            }
-                        }
-                    }
+                    collectNetworkStatistics(router, null);
                 }
             } catch (final Exception e) {
                 s_logger.warn("Error while collecting network stats", e);
@@ -3057,6 +2964,11 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
         // Collect network usage before stopping Vm
 
         final DomainRouterVO router = _routerDao.findById(profile.getVirtualMachine().getId());
+        collectNetworkStatistics(router, null);
+    }
+
+    @Override
+    public <T extends VirtualRouter> void collectNetworkStatistics(final T router, final Nic nic) {
         if (router == null) {
             return;
         }
@@ -3065,12 +2977,23 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
 
         if (privateIP != null) {
             final boolean forVpc = router.getVpcId() != null;
-            final List<? extends Nic> routerNics = _nicDao.listByVmId(router.getId());
+            List<Nic> routerNics = new ArrayList<Nic>();
+            if (nic != null) {
+                routerNics.add(nic);
+            } else {
+                routerNics.addAll(_nicDao.listByVmId(router.getId()));
+            }
             for (final Nic routerNic : routerNics) {
                 final Network network = _networkModel.getNetwork(routerNic.getNetworkId());
                 // Send network usage command for public nic in VPC VR
                 // Send network usage command for isolated guest nic of non VPC
                 // VR
+
+                //[TODO] Avoiding the NPE now, but I have to find out what is going on with the network. - Wilder Rodrigues
+                if (network == null) {
+                    s_logger.error("Could not find a network with ID => " + routerNic.getNetworkId() + ". It might be a problem!");
+                    continue;
+                }
                 if (forVpc && network.getTrafficType() == TrafficType.Public || !forVpc && network.getTrafficType() == TrafficType.Guest
                         && network.getGuestType() == Network.GuestType.Isolated) {
                     final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), forVpc, routerNic.getIPv4Address());
diff --git a/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java b/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java
index 6ee5e85f271..b671e33df08 100644
--- a/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java
+++ b/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java
@@ -40,6 +40,7 @@ import com.cloud.network.NetworkModel;
 import com.cloud.network.Networks.BroadcastDomainType;
 import com.cloud.network.Networks.IsolationType;
 import com.cloud.network.PublicIpAddress;
+import com.cloud.network.VpcVirtualNetworkApplianceService;
 import com.cloud.network.dao.FirewallRulesDao;
 import com.cloud.network.dao.IPAddressDao;
 import com.cloud.network.dao.IPAddressVO;
@@ -84,6 +85,7 @@ public class NicPlugInOutRules extends RuleApplier {
         NetworkModel networkModel = visitor.getVirtualNetworkApplianceFactory().getNetworkModel();
         VirtualMachineManager itMgr = visitor.getVirtualNetworkApplianceFactory().getItMgr();
         NicDao nicDao = visitor.getVirtualNetworkApplianceFactory().getNicDao();
+        VpcVirtualNetworkApplianceService routerService = visitor.getVirtualNetworkApplianceFactory().getRouterService();
 
         // de-associate IPs before unplugging nics
         if (!nicsToUnplug.isEmpty()) {
@@ -107,6 +109,12 @@ public class NicPlugInOutRules extends RuleApplier {
 
         // 1) Unplug the nics
         for (Entry<String, PublicIpAddress> entry : nicsToUnplug.entrySet()) {
+            PublicIpAddress ip = entry.getValue();
+            NicVO nic = nicDao.findByIp4AddressAndNetworkIdAndInstanceId(ip.getNetworkId(), _router.getId(), ip.getAddress().addr());
+            if (nic != null) {
+                s_logger.info("Collect network statistics for nic " + nic + " from router " + _router);
+                routerService.collectNetworkStatistics(_router, nic);
+            }
             Network publicNtwk = null;
             try {
                 publicNtwk = networkModel.getNetwork(entry.getValue().getNetworkId());
diff --git a/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java b/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java
index 9d4660da557..d96673d42d3 100644
--- a/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java
+++ b/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java
@@ -26,6 +26,7 @@ import com.cloud.dc.dao.HostPodDao;
 import com.cloud.dc.dao.VlanDao;
 import com.cloud.network.IpAddressManager;
 import com.cloud.network.NetworkModel;
+import com.cloud.network.VpcVirtualNetworkApplianceService;
 import com.cloud.network.dao.FirewallRulesDao;
 import com.cloud.network.dao.IPAddressDao;
 import com.cloud.network.dao.LoadBalancerDao;
@@ -98,6 +99,8 @@ public class VirtualNetworkApplianceFactory {
 
     @Inject
     private NetworkTopologyContext _networkTopologyContext;
+    @Inject
+    private VpcVirtualNetworkApplianceService _routerService;
 
     public NetworkModel getNetworkModel() {
         return _networkModel;
@@ -190,4 +193,8 @@ public class VirtualNetworkApplianceFactory {
     public FirewallRulesDao getFirewallRulesDao() {
         return _rulesDao;
     }
+
+    public VpcVirtualNetworkApplianceService getRouterService() {
+        return _routerService;
+    }
 }
diff --git a/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java b/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java
index 45bf4c1763b..176fd8e68ea 100644
--- a/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java
+++ b/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java
@@ -41,6 +41,7 @@ import com.cloud.user.Account;
 import com.cloud.user.User;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.vm.DomainRouterVO;
+import com.cloud.vm.Nic;
 import com.cloud.vm.VirtualMachineProfile;
 
 @Component
@@ -280,4 +281,10 @@ public class MockVpcVirtualNetworkApplianceManager extends ManagerBase implement
         // TODO Auto-generated method stub
         return false;
     }
-}
\ No newline at end of file
+
+    @Override
+    public <T extends VirtualRouter> void collectNetworkStatistics(T router, Nic nic) {
+        // TODO Auto-generated method stub
+        return;
+    }
+}
diff --git a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py
index 44b69500b4c..dfea7019f7a 100755
--- a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py
+++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py
@@ -476,11 +476,6 @@ class CsIP:
                             "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j ACL_OUTBOUND_%s" %
                             (self.dev, guestNetworkCidr, self.address['gateway'], self.dev)])
 
-            self.fw.append(["", "front", "-A NETWORK_STATS_%s -i %s -d %s" %
-                            ("eth1", "eth1", guestNetworkCidr)])
-            self.fw.append(["", "front", "-A NETWORK_STATS_%s -o %s -s %s" %
-                            ("eth1", "eth1", guestNetworkCidr)])
-
         if self.is_private_gateway():
             self.fw.append(["filter", "", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" %
                             (self.address['network'], self.dev, self.dev)])
@@ -518,6 +513,10 @@ class CsIP:
                 ["mangle", "", "-A VPN_STATS_%s -i %s -m mark --mark 0x524/0xffffffff" % (self.dev, self.dev)])
             self.fw.append(
                 ["", "front", "-A FORWARD -j NETWORK_STATS_%s" % self.dev])
+            self.fw.append(
+                ["", "front", "-A NETWORK_STATS_%s -s %s -o %s" % (self.dev, self.cl.get_vpccidr(), self.dev)])
+            self.fw.append(
+                ["", "front", "-A NETWORK_STATS_%s -d %s -i %s" % (self.dev, self.cl.get_vpccidr(), self.dev)])
 
         self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"])
         self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"])

From dc5b9ec7c891ebeb23ec43de5cf0158e2ef378eb Mon Sep 17 00:00:00 2001
From: Wei Zhou <w.zhou@global.leaseweb.com>
Date: Thu, 1 Apr 2021 09:21:17 +0200
Subject: [PATCH 09/12] systemvm: remove logrotate config for wtmp and btmp
 (#4872)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

logrotate in systemvms run every day. it exits with failure.
```
root@r-100-VM:~# systemctl status logrotate
● logrotate.service - Rotate log files
   Loaded: loaded (/lib/systemd/system/logrotate.service; static; vendor preset: enabled)
   Active: failed (Result: exit-code) since Thu 2021-03-23 00:00:01 UTC; 2 days ago
     Docs: man:logrotate(8)
           man:logrotate.conf(5)
  Process: 25001 ExecStart=/usr/sbin/logrotate /etc/logrotate.conf (code=exited, status=1/FAILURE)
 Main PID: 25001 (code=exited, status=1/FAILURE)

Warning: Journal has been rotated since unit was started. Log output is incomplete or unavailable.
```

it is because the logrotate for wtmp and btmp already exist in 4.15 systemvm template.
```
root@r-100-VM:~# cat /etc/logrotate.d/wtmp
/var/log/wtmp {
    missingok
    monthly
    create 0664 root utmp
    minsize 1M
    rotate 1
}
root@r-100-VM:~# cat /etc/logrotate.d/btmp
/var/log/btmp {
    missingok
    monthly
    create 0660 root utmp
    rotate 1
}
```

remove them from /etc/logrotate.conf fixes the issue.
```
root@r-100-VM:~# systemctl status logrotate
● logrotate.service - Rotate log files
   Loaded: loaded (/lib/systemd/system/logrotate.service; static; vendor preset: enabled)
   Active: inactive (dead) since Thu 2021-03-25 00:00:01 UTC; 9h ago
     Docs: man:logrotate(8)
           man:logrotate.conf(5)
  Process: 28211 ExecStart=/usr/sbin/logrotate /etc/logrotate.conf (code=exited, status=0/SUCCESS)
 Main PID: 28211 (code=exited, status=0/SUCCESS)

Mar 25 00:00:01 r-100-VM systemd[1]: Starting Rotate log files...
Mar 25 00:00:01 r-100-VM systemd[1]: logrotate.service: Succeeded.
Mar 25 00:00:01 r-100-VM systemd[1]: Started Rotate log files.
```
---
 systemvm/debian/etc/logrotate.conf   | 12 ------------
 systemvm/debian/etc/logrotate.d/btmp |  7 +++++++
 systemvm/debian/etc/logrotate.d/wtmp |  8 ++++++++
 3 files changed, 15 insertions(+), 12 deletions(-)
 create mode 100644 systemvm/debian/etc/logrotate.d/btmp
 create mode 100644 systemvm/debian/etc/logrotate.d/wtmp

diff --git a/systemvm/debian/etc/logrotate.conf b/systemvm/debian/etc/logrotate.conf
index d213d4e9ce8..d33f2dae2e6 100644
--- a/systemvm/debian/etc/logrotate.conf
+++ b/systemvm/debian/etc/logrotate.conf
@@ -11,15 +11,3 @@ size 50M
 compress
 # RPM packages drop log rotation information into this directory
 include /etc/logrotate.d
-# no packages own wtmp and btmp -- we'll rotate them here
-/var/log/wtmp {
-    monthly
-    create 0664 root utmp
-    rotate 1
-}
-/var/log/btmp {
-    missingok
-    monthly
-    create 0600 root utmp
-    rotate 1
-}
diff --git a/systemvm/debian/etc/logrotate.d/btmp b/systemvm/debian/etc/logrotate.d/btmp
new file mode 100644
index 00000000000..0aa1ae1ade1
--- /dev/null
+++ b/systemvm/debian/etc/logrotate.d/btmp
@@ -0,0 +1,7 @@
+# no packages own btmp -- we'll rotate it here
+/var/log/btmp {
+    missingok
+    monthly
+    create 0660 root utmp
+    rotate 1
+}
diff --git a/systemvm/debian/etc/logrotate.d/wtmp b/systemvm/debian/etc/logrotate.d/wtmp
new file mode 100644
index 00000000000..cc8a151e00b
--- /dev/null
+++ b/systemvm/debian/etc/logrotate.d/wtmp
@@ -0,0 +1,8 @@
+# no packages own wtmp -- we'll rotate it here
+/var/log/wtmp {
+    missingok
+    monthly
+    create 0664 root utmp
+    minsize 1M
+    rotate 1
+}

From 251368361ef6c49183932c8b4a89e734af9bfab9 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek.mrt22@gmail.com>
Date: Thu, 1 Apr 2021 13:10:00 +0530
Subject: [PATCH 10/12] ui: allow docHelp override using config.json (#4745)

config.json can have a property named 'docHelpMappings' which can be used to override docHelp suffixes.
In config.json admin can add mappings as follows:

"docHelpMappings": {
  "virtual_machine.html": "some.html",
  "some_string": "override_string"
}

UI will use these mappings and will make appropriate replacements in the documentation links for different sections and forms.

Addresses #4731

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
---
 ui/public/config.json                   |  1 +
 ui/src/components/view/ActionButton.vue |  3 +++
 ui/src/config/router.js                 |  8 ++++----
 ui/src/main.js                          |  4 +++-
 ui/src/utils/plugins.js                 | 18 ++++++++++++++++++
 ui/tests/common/index.js                |  3 ++-
 6 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/ui/public/config.json b/ui/public/config.json
index f4fda411426..45cc974ffeb 100644
--- a/ui/public/config.json
+++ b/ui/public/config.json
@@ -46,5 +46,6 @@
     "jp": "label.japanese.keyboard",
     "sc": "label.simplified.chinese.keyboard"
   },
+  "docHelpMappings": {},
   "plugins": []
 }
diff --git a/ui/src/components/view/ActionButton.vue b/ui/src/components/view/ActionButton.vue
index fd25c70d557..bf9f63b0fdd 100644
--- a/ui/src/components/view/ActionButton.vue
+++ b/ui/src/components/view/ActionButton.vue
@@ -134,6 +134,9 @@ export default {
   methods: {
     execAction (action) {
       action.resource = this.resource
+      if (action.docHelp) {
+        action.docHelp = this.$applyDocHelpMappings(action.docHelp)
+      }
       this.$emit('exec-action', action)
     },
     handleShowBadge () {
diff --git a/ui/src/config/router.js b/ui/src/config/router.js
index 1b1ad0c2a06..a07f0d68bb5 100644
--- a/ui/src/config/router.js
+++ b/ui/src/config/router.js
@@ -42,7 +42,7 @@ function generateRouterMap (section) {
     name: section.name,
     path: '/' + section.name,
     hidden: section.hidden,
-    meta: { title: section.title, icon: section.icon, docHelp: section.docHelp, searchFilters: section.searchFilters },
+    meta: { title: section.title, icon: section.icon, docHelp: Vue.prototype.$applyDocHelpMappings(section.docHelp), searchFilters: section.searchFilters },
     component: RouteView
   }
 
@@ -63,7 +63,7 @@ function generateRouterMap (section) {
           title: child.title,
           name: child.name,
           icon: child.icon,
-          docHelp: child.docHelp,
+          docHelp: Vue.prototype.$applyDocHelpMappings(child.docHelp),
           permission: child.permission,
           resourceType: child.resourceType,
           filters: child.filters,
@@ -85,7 +85,7 @@ function generateRouterMap (section) {
               title: child.title,
               name: child.name,
               icon: child.icon,
-              docHelp: child.docHelp,
+              docHelp: Vue.prototype.$applyDocHelpMappings(child.docHelp),
               permission: child.permission,
               resourceType: child.resourceType,
               params: child.params ? child.params : {},
@@ -140,7 +140,7 @@ function generateRouterMap (section) {
         title: section.title,
         name: section.name,
         icon: section.icon,
-        docHelp: section.docHelp,
+        docHelp: Vue.prototype.$applyDocHelpMappings(section.docHelp),
         hidden: section.hidden,
         permission: section.permission,
         resourceType: section.resourceType,
diff --git a/ui/src/main.js b/ui/src/main.js
index acc41d2de33..064314b98de 100644
--- a/ui/src/main.js
+++ b/ui/src/main.js
@@ -26,7 +26,7 @@ import './core/lazy_use'
 import './core/ext'
 import './permission' // permission control
 import './utils/filter' // global filter
-import { pollJobPlugin, notifierPlugin, toLocaleDatePlugin } from './utils/plugins'
+import { pollJobPlugin, notifierPlugin, toLocaleDatePlugin, configUtilPlugin } from './utils/plugins'
 import { VueAxios } from './utils/request'
 
 Vue.config.productionTip = false
@@ -49,3 +49,5 @@ fetch('config.json').then(response => response.json()).then(config => {
     }).$mount('#app')
   })
 })
+
+Vue.use(configUtilPlugin)
diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js
index d32217f6ba1..905f0b1bbfd 100644
--- a/ui/src/utils/plugins.js
+++ b/ui/src/utils/plugins.js
@@ -167,3 +167,21 @@ export const toLocaleDatePlugin = {
     }
   }
 }
+
+export const configUtilPlugin = {
+  install (Vue) {
+    Vue.prototype.$applyDocHelpMappings = function (docHelp) {
+      var docHelpMappings = this.$config.docHelpMappings
+      if (docHelp && docHelpMappings &&
+        docHelpMappings.constructor === Object && Object.keys(docHelpMappings).length > 0) {
+        for (var key in docHelpMappings) {
+          if (docHelp.includes(key)) {
+            docHelp = docHelp.replace(key, docHelpMappings[key])
+            break
+          }
+        }
+      }
+      return docHelp
+    }
+  }
+}
diff --git a/ui/tests/common/index.js b/ui/tests/common/index.js
index 544c66eeff6..e9c8deb0321 100644
--- a/ui/tests/common/index.js
+++ b/ui/tests/common/index.js
@@ -21,10 +21,11 @@ import mockRouter from '../mock/mockRouter'
 
 import localVue from '../setup'
 import { mount } from '@vue/test-utils'
-import { pollJobPlugin, notifierPlugin } from '@/utils/plugins'
+import { pollJobPlugin, notifierPlugin, configUtilPlugin } from '@/utils/plugins'
 
 localVue.use(pollJobPlugin)
 localVue.use(notifierPlugin)
+localVue.use(configUtilPlugin)
 
 function createMockRouter (newRoutes = []) {
   let routes = []

From d4ba00434c4925446a7aa2faa0ea782ae2496a47 Mon Sep 17 00:00:00 2001
From: Wei Zhou <w.zhou@global.leaseweb.com>
Date: Thu, 1 Apr 2021 11:00:58 +0200
Subject: [PATCH 11/12] VR: fix rsyslog compresses log files but not release
 disk space in VR (#4869)

We had critical issue with VR recently. The VRs of shared network or vpc stops working after some days.
After investigation, I found that the disk space is full

```
root@r-10-VM:~# df
Filesystem     1K-blocks    Used Available Use% Mounted on
/dev/vda5        2086316 2069932         0 100% /
```

logrotate/ryslog has compresses the log files, but space is not released. see `lsof |grep deleted`

```
root@r-10-VM:~# lsof |grep deleted
rsyslogd    960                      root   12w      REG              254,5 493060096        137 /var/log/daemon.log.1 (deleted)
rsyslogd    960                      root   13w      REG              254,5  17715200        110 /var/log/messages.1 (deleted)
rsyslogd    960                      root   16w      REG              254,5 545968128        342 /var/log/auth.log.1 (deleted)
rsyslogd    960                      root   18w      REG              254,5  38313984        341 /var/log/cron.log.1 (deleted)
rsyslogd    960  962 in:imuxso       root   12w      REG              254,5 493060096        137 /var/log/daemon.log.1 (deleted)
rsyslogd    960  962 in:imuxso       root   13w      REG              254,5  17715200        110 /var/log/messages.1 (deleted)
rsyslogd    960  962 in:imuxso       root   16w      REG              254,5 545968128        342 /var/log/auth.log.1 (deleted)
rsyslogd    960  962 in:imuxso       root   18w      REG              254,5  38313984        341 /var/log/cron.log.1 (deleted)
rsyslogd    960  963 in:imklog       root   12w      REG              254,5 493060096        137 /var/log/daemon.log.1 (deleted)
rsyslogd    960  963 in:imklog       root   13w      REG              254,5  17715200        110 /var/log/messages.1 (deleted)
rsyslogd    960  963 in:imklog       root   16w      REG              254,5 545968128        342 /var/log/auth.log.1 (deleted)
rsyslogd    960  963 in:imklog       root   18w      REG              254,5  38313984        341 /var/log/cron.log.1 (deleted)
rsyslogd    960  964 in:imfile       root   12w      REG              254,5 493060096        137 /var/log/daemon.log.1 (deleted)
rsyslogd    960  964 in:imfile       root   13w      REG              254,5  17715200        110 /var/log/messages.1 (deleted)
rsyslogd    960  964 in:imfile       root   16w      REG              254,5 545968128        342 /var/log/auth.log.1 (deleted)
rsyslogd    960  964 in:imfile       root   18w      REG              254,5  38313984        341 /var/log/cron.log.1 (deleted)
rsyslogd    960  965 in:imudp        root   12w      REG              254,5 493060096        137 /var/log/daemon.log.1 (deleted)
rsyslogd    960  965 in:imudp        root   13w      REG              254,5  17715200        110 /var/log/messages.1 (deleted)
rsyslogd    960  965 in:imudp        root   16w      REG              254,5 545968128        342 /var/log/auth.log.1 (deleted)
rsyslogd    960  965 in:imudp        root   18w      REG              254,5  38313984        341 /var/log/cron.log.1 (deleted)
rsyslogd    960  966 rs:main         root   12w      REG              254,5 493060096        137 /var/log/daemon.log.1 (deleted)
rsyslogd    960  966 rs:main         root   13w      REG              254,5  17715200        110 /var/log/messages.1 (deleted)
rsyslogd    960  966 rs:main         root   16w      REG              254,5 545968128        342 /var/log/auth.log.1 (deleted)
rsyslogd    960  966 rs:main         root   18w      REG              254,5  38313984        341 /var/log/cron.log.1 (deleted)
```

workaround: restarting rsyslog to release the space.
```
systemctl restart rsyslog
```

The root cause is, the following command does not work in 4.15 template
```
root@r-10-VM:~# invoke-rc.d rsyslog rotate
[FAIL] Closing open files: rsyslogd failed!
```

Fix: use `/usr/lib/rsyslog/rsyslog-rotate` instead
```
root@r-10-VM:~# /usr/lib/rsyslog/rsyslog-rotate
root@r-10-VM:~# cat /usr/lib/rsyslog/rsyslog-rotate

if [ -d /run/systemd/system ]; then
    systemctl kill -s HUP rsyslog.service
else
    invoke-rc.d rsyslog rotate > /dev/null
fi

```
---
 systemvm/debian/etc/logrotate.d/rsyslog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/systemvm/debian/etc/logrotate.d/rsyslog b/systemvm/debian/etc/logrotate.d/rsyslog
index 9291494481e..cef88191606 100644
--- a/systemvm/debian/etc/logrotate.d/rsyslog
+++ b/systemvm/debian/etc/logrotate.d/rsyslog
@@ -6,7 +6,7 @@
 	notifempty
 	compress
 	postrotate
-		/usr/sbin/invoke-rc.d rsyslog rotate > /dev/null
+		/usr/lib/rsyslog/rsyslog-rotate
 	endscript
 }
 
@@ -30,6 +30,6 @@
 	compress
 	sharedscripts
 	postrotate
-		/usr/sbin/invoke-rc.d rsyslog rotate > /dev/null
+		/usr/lib/rsyslog/rsyslog-rotate
 	endscript
 }

From 575e18c00af80f15f2cfb7d19bea06d086dc9779 Mon Sep 17 00:00:00 2001
From: Hoang Nguyen <hoangnm@unitech.vn>
Date: Thu, 1 Apr 2021 16:01:57 +0700
Subject: [PATCH 12/12] ui: Restored the Basic Networking (#4882)

* zone wizard: restored the basic networking

* set enabled Basic Networking
---
 ui/public/config.json                             |  3 ++-
 .../views/infra/zone/ZoneWizardZoneTypeStep.vue   | 15 ++++++++++++---
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/ui/public/config.json b/ui/public/config.json
index 45cc974ffeb..9f3b723c27f 100644
--- a/ui/public/config.json
+++ b/ui/public/config.json
@@ -47,5 +47,6 @@
     "sc": "label.simplified.chinese.keyboard"
   },
   "docHelpMappings": {},
-  "plugins": []
+  "plugins": [],
+  "basicZoneEnabled": true
 }
diff --git a/ui/src/views/infra/zone/ZoneWizardZoneTypeStep.vue b/ui/src/views/infra/zone/ZoneWizardZoneTypeStep.vue
index 722df4fa46c..72b4a60dd19 100644
--- a/ui/src/views/infra/zone/ZoneWizardZoneTypeStep.vue
+++ b/ui/src/views/infra/zone/ZoneWizardZoneTypeStep.vue
@@ -30,10 +30,18 @@
               initialValue: zoneType
             }]
           }]">
+          <a-card :gutter="12" class="card-item" v-if="$config.basicZoneEnabled">
+            <a-col :md="6" :lg="6">
+              <a-radio class="card-form-item" value="Basic">{{ $t('label.basic') }}</a-radio>
+            </a-col>
+            <a-col :md="18" :lg="18">
+              <a-card class="ant-form-text zone-support">{{ $t(zoneDescription.Basic) }}</a-card>
+            </a-col>
+          </a-card>
           <a-card :gutter="12" class="card-item">
             <a-col :md="6" :lg="6">
-              <a-radio style="display: none;" class="card-form-item" value="Advanced">{{ $t('label.advanced') }}</a-radio>
-              <span style="margin-top: 20px" class="card-form-item">
+              <a-radio class="card-form-item" value="Advanced" v-if="$config.basicZoneEnabled">{{ $t('label.advanced') }}</a-radio>
+              <span style="margin-top: 20px;" class="card-form-item" v-else>
                 <a-icon type="setting" style="margin-right: 10px" />
                 {{ $t('label.advanced') }}
               </span>
@@ -86,6 +94,7 @@ export default {
       wrapperCol: { span: 14 }
     },
     zoneDescription: {
+      Basic: 'message.desc.basic.zone',
       Advanced: 'message.desc.advanced.zone',
       SecurityGroups: 'message.advanced.security.group'
     }
@@ -108,7 +117,7 @@ export default {
       return this.zoneType === 'Advanced'
     },
     zoneType () {
-      return this.prefillContent.zoneType ? this.prefillContent.zoneType.value : 'Advanced'
+      return this.prefillContent.zoneType ? this.prefillContent.zoneType.value : (this.$config.basicZoneEnabled ? 'Basic' : 'Advanced')
     },
     securityGroupsEnabled () {
       return this.isAdvancedZone && (this.prefillContent.securityGroupsEnabled ? this.prefillContent.securityGroupsEnabled.value : false)