server: Add option 'details' to listProjects and listAccounts (#3331)

If there are many projects and accounts, listing projects/accounts will take long time getting the resource limitation and resource count in the process. However resource count/limitation are not needed sometimes.

Add an option 'details' to listProjects and listAccounts. If you do not need the resource count/limitation, please add details=min to api call. The api execution time will be reduced significantly.
This commit is contained in:
Wei Zhou 2019-07-04 12:56:04 +02:00 committed by Rohit Yadav
parent a0097d83da
commit 6780930ecc
17 changed files with 330 additions and 34 deletions

View File

@ -16,16 +16,22 @@
// under the License.
package org.apache.cloudstack.api.command.user.account;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import org.apache.log4j.Logger;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiConstants.DomainDetails;
import org.apache.cloudstack.api.BaseListDomainResourcesCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ResponseObject.ResponseView;
import org.apache.cloudstack.api.response.AccountResponse;
import org.apache.cloudstack.api.response.ListResponse;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.user.Account;
@APICommand(name = "listAccounts", description = "Lists accounts and provides detailed account information for listed accounts", responseObject = AccountResponse.class, responseView = ResponseView.Restricted, entityType = {Account.class},
@ -55,6 +61,12 @@ public class ListAccountsCmd extends BaseListDomainResourcesCmd {
@Parameter(name = ApiConstants.STATE, type = CommandType.STRING, description = "list accounts by state. Valid states are enabled, disabled, and locked.")
private String state;
@Parameter(name = ApiConstants.DETAILS,
type = CommandType.LIST,
collectionType = CommandType.STRING,
description = "comma separated list of account details requested, value can be a list of [ all, resource, min]")
private List<String> viewDetails;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
@ -79,6 +91,25 @@ public class ListAccountsCmd extends BaseListDomainResourcesCmd {
return state;
}
public EnumSet<DomainDetails> getDetails() throws InvalidParameterValueException {
EnumSet<DomainDetails> dv;
if (viewDetails == null || viewDetails.size() <= 0) {
dv = EnumSet.of(DomainDetails.all);
} else {
try {
ArrayList<DomainDetails> dc = new ArrayList<DomainDetails>();
for (String detail : viewDetails) {
dc.add(DomainDetails.valueOf(detail));
}
dv = EnumSet.copyOf(dc);
} catch (IllegalArgumentException e) {
throw new InvalidParameterValueException("The details parameter contains a non permitted value. The allowed values are " +
EnumSet.allOf(DomainDetails.class));
}
}
return dv;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////

View File

@ -16,15 +16,19 @@
// under the License.
package org.apache.cloudstack.api.command.user.project;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.log4j.Logger;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiConstants.DomainDetails;
import org.apache.cloudstack.api.BaseListAccountResourcesCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.response.ListResponse;
@ -61,6 +65,12 @@ public class ListProjectsCmd extends BaseListAccountResourcesCmd {
@Parameter(name = ApiConstants.TAGS, type = CommandType.MAP, description = "List projects by tags (key/value pairs)")
private Map tags;
@Parameter(name = ApiConstants.DETAILS,
type = CommandType.LIST,
collectionType = CommandType.STRING,
description = "comma separated list of project details requested, value can be a list of [ all, resource, min]")
private List<String> viewDetails;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
@ -105,6 +115,25 @@ public class ListProjectsCmd extends BaseListAccountResourcesCmd {
return tagsMap;
}
public EnumSet<DomainDetails> getDetails() throws InvalidParameterValueException {
EnumSet<DomainDetails> dv;
if (viewDetails == null || viewDetails.size() <= 0) {
dv = EnumSet.of(DomainDetails.all);
} else {
try {
ArrayList<DomainDetails> dc = new ArrayList<DomainDetails>();
for (String detail : viewDetails) {
dc.add(DomainDetails.valueOf(detail));
}
dv = EnumSet.copyOf(dc);
} catch (IllegalArgumentException e) {
throw new InvalidParameterValueException("The details parameter contains a non permitted value. The allowed values are " +
EnumSet.allOf(DomainDetails.class));
}
}
return dv;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////

View File

@ -1767,8 +1767,8 @@ public class ApiDBUtils {
return s_userAccountJoinDao.newUserView(usr);
}
public static ProjectResponse newProjectResponse(ProjectJoinVO proj) {
return s_projectJoinDao.newProjectResponse(proj);
public static ProjectResponse newProjectResponse(EnumSet<DomainDetails> details, ProjectJoinVO proj) {
return s_projectJoinDao.newProjectResponse(details, proj);
}
public static List<ProjectJoinVO> newProjectView(Project proj) {
@ -1872,8 +1872,8 @@ public class ApiDBUtils {
return s_domainJoinDao.newDomainResponse(view, details, ve);
}
public static AccountResponse newAccountResponse(ResponseView view, AccountJoinVO ve) {
AccountResponse response = s_accountJoinDao.newAccountResponse(view, ve);
public static AccountResponse newAccountResponse(ResponseView view, EnumSet<DomainDetails> details, AccountJoinVO ve) {
AccountResponse response = s_accountJoinDao.newAccountResponse(view, details, ve);
// Populate account role information
if (ve.getRoleId() != null) {
Role role = s_roleService.findRole(ve.getRoleId());

View File

@ -35,6 +35,7 @@ import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.acl.ControlledEntity.ACLType;
import org.apache.cloudstack.affinity.AffinityGroup;
import org.apache.cloudstack.affinity.AffinityGroupResponse;
import org.apache.cloudstack.api.ApiConstants.DomainDetails;
import org.apache.cloudstack.api.ApiConstants.HostDetails;
import org.apache.cloudstack.api.ApiConstants.VMDetails;
import org.apache.cloudstack.api.ResponseGenerator;
@ -380,13 +381,13 @@ public class ApiResponseHelper implements ResponseGenerator {
// creates an account + user)
@Override
public AccountResponse createUserAccountResponse(ResponseView view, UserAccount user) {
return ApiDBUtils.newAccountResponse(view, ApiDBUtils.findAccountViewById(user.getAccountId()));
return ApiDBUtils.newAccountResponse(view, EnumSet.of(DomainDetails.all), ApiDBUtils.findAccountViewById(user.getAccountId()));
}
@Override
public AccountResponse createAccountResponse(ResponseView view, Account account) {
AccountJoinVO vUser = ApiDBUtils.newAccountView(account);
return ApiDBUtils.newAccountResponse(view, vUser);
return ApiDBUtils.newAccountResponse(view, EnumSet.of(DomainDetails.all), vUser);
}
@Override
@ -2292,7 +2293,7 @@ public class ApiResponseHelper implements ResponseGenerator {
@Override
public ProjectResponse createProjectResponse(Project project) {
List<ProjectJoinVO> viewPrjs = ApiDBUtils.newProjectView(project);
List<ProjectResponse> listPrjs = ViewResponseHelper.createProjectResponse(viewPrjs.toArray(new ProjectJoinVO[viewPrjs.size()]));
List<ProjectResponse> listPrjs = ViewResponseHelper.createProjectResponse(EnumSet.of(DomainDetails.all), viewPrjs.toArray(new ProjectJoinVO[viewPrjs.size()]));
assert listPrjs != null && listPrjs.size() == 1 : "There should be one project returned";
return listPrjs.get(0);
}

View File

@ -1271,7 +1271,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
public ListResponse<ProjectResponse> listProjects(ListProjectsCmd cmd) {
Pair<List<ProjectJoinVO>, Integer> projects = listProjectsInternal(cmd);
ListResponse<ProjectResponse> response = new ListResponse<ProjectResponse>();
List<ProjectResponse> projectResponses = ViewResponseHelper.createProjectResponse(projects.first().toArray(new ProjectJoinVO[projects.first().size()]));
List<ProjectResponse> projectResponses = ViewResponseHelper.createProjectResponse(cmd.getDetails(), projects.first().toArray(new ProjectJoinVO[projects.first().size()]));
response.setResponses(projectResponses, projects.second());
return response;
}
@ -1931,7 +1931,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
respView = ResponseView.Full;
}
List<AccountResponse> accountResponses = ViewResponseHelper.createAccountResponse(respView, result.first().toArray(new AccountJoinVO[result.first().size()]));
List<AccountResponse> accountResponses = ViewResponseHelper.createAccountResponse(respView, cmd.getDetails(), result.first().toArray(new AccountJoinVO[result.first().size()]));
response.setResponses(accountResponses, result.second());
return response;
}

View File

@ -195,14 +195,14 @@ public class ViewResponseHelper {
return new ArrayList<SecurityGroupResponse>(vrDataList.values());
}
public static List<ProjectResponse> createProjectResponse(ProjectJoinVO... projects) {
public static List<ProjectResponse> createProjectResponse(EnumSet<DomainDetails> details, ProjectJoinVO... projects) {
Hashtable<Long, ProjectResponse> prjDataList = new Hashtable<Long, ProjectResponse>();
// Initialise the prjdatalist with the input data
for (ProjectJoinVO p : projects) {
ProjectResponse pData = prjDataList.get(p.getId());
if (pData == null) {
// first time encountering this vm
pData = ApiDBUtils.newProjectResponse(p);
pData = ApiDBUtils.newProjectResponse(details, p);
prjDataList.put(p.getId(), pData);
}
}
@ -538,10 +538,10 @@ public class ViewResponseHelper {
}
}
public static List<AccountResponse> createAccountResponse(ResponseView view, AccountJoinVO... accounts) {
public static List<AccountResponse> createAccountResponse(ResponseView view, EnumSet<DomainDetails> details, AccountJoinVO... accounts) {
List<AccountResponse> respList = new ArrayList<AccountResponse>();
for (AccountJoinVO vt : accounts){
respList.add(ApiDBUtils.newAccountResponse(view, vt));
respList.add(ApiDBUtils.newAccountResponse(view, details, vt));
}
return respList;
}

View File

@ -16,6 +16,9 @@
// under the License.
package com.cloud.api.query.dao;
import java.util.EnumSet;
import org.apache.cloudstack.api.ApiConstants.DomainDetails;
import org.apache.cloudstack.api.ResponseObject.ResponseView;
import org.apache.cloudstack.api.response.AccountResponse;
import org.apache.cloudstack.api.response.ResourceLimitAndCountResponse;
@ -26,7 +29,7 @@ import com.cloud.utils.db.GenericDao;
public interface AccountJoinDao extends GenericDao<AccountJoinVO, Long> {
AccountResponse newAccountResponse(ResponseView view, AccountJoinVO vol);
AccountResponse newAccountResponse(ResponseView view, EnumSet<DomainDetails> details, AccountJoinVO vol);
AccountJoinVO newAccountView(Account vol);

View File

@ -16,6 +16,7 @@
// under the License.
package com.cloud.api.query.dao;
import java.util.EnumSet;
import java.util.List;
import javax.inject.Inject;
@ -23,6 +24,7 @@ import javax.inject.Inject;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.api.ApiConstants.DomainDetails;
import org.apache.cloudstack.api.ResponseObject.ResponseView;
import org.apache.cloudstack.api.response.AccountResponse;
import org.apache.cloudstack.api.response.ResourceLimitAndCountResponse;
@ -57,7 +59,7 @@ public class AccountJoinDaoImpl extends GenericDaoBase<AccountJoinVO, Long> impl
}
@Override
public AccountResponse newAccountResponse(ResponseView view, AccountJoinVO account) {
public AccountResponse newAccountResponse(ResponseView view, EnumSet<DomainDetails> details, AccountJoinVO account) {
AccountResponse accountResponse = new AccountResponse();
accountResponse.setId(account.getUuid());
accountResponse.setName(account.getAccountName());
@ -76,17 +78,19 @@ public class AccountJoinDaoImpl extends GenericDaoBase<AccountJoinVO, Long> impl
accountResponse.setBytesReceived(account.getBytesReceived());
accountResponse.setBytesSent(account.getBytesSent());
boolean fullView = (view == ResponseView.Full && _acctMgr.isRootAdmin(account.getId()));
setResourceLimits(account, fullView, accountResponse);
if (details.contains(DomainDetails.all) || details.contains(DomainDetails.resource)) {
boolean fullView = (view == ResponseView.Full && _acctMgr.isRootAdmin(account.getId()));
setResourceLimits(account, fullView, accountResponse);
//get resource limits for projects
long projectLimit = ApiDBUtils.findCorrectResourceLimit(account.getProjectLimit(), account.getId(), ResourceType.project);
String projectLimitDisplay = (fullView || projectLimit == -1) ? "Unlimited" : String.valueOf(projectLimit);
long projectTotal = (account.getProjectTotal() == null) ? 0 : account.getProjectTotal();
String projectAvail = (fullView || projectLimit == -1) ? "Unlimited" : String.valueOf(projectLimit - projectTotal);
accountResponse.setProjectLimit(projectLimitDisplay);
accountResponse.setProjectTotal(projectTotal);
accountResponse.setProjectAvailable(projectAvail);
//get resource limits for projects
long projectLimit = ApiDBUtils.findCorrectResourceLimit(account.getProjectLimit(), account.getId(), ResourceType.project);
String projectLimitDisplay = (fullView || projectLimit == -1) ? "Unlimited" : String.valueOf(projectLimit);
long projectTotal = (account.getProjectTotal() == null) ? 0 : account.getProjectTotal();
String projectAvail = (fullView || projectLimit == -1) ? "Unlimited" : String.valueOf(projectLimit - projectTotal);
accountResponse.setProjectLimit(projectLimitDisplay);
accountResponse.setProjectTotal(projectTotal);
accountResponse.setProjectAvailable(projectAvail);
}
// set async job
if (account.getJobId() != null) {

View File

@ -16,8 +16,10 @@
// under the License.
package com.cloud.api.query.dao;
import java.util.EnumSet;
import java.util.List;
import org.apache.cloudstack.api.ApiConstants.DomainDetails;
import org.apache.cloudstack.api.response.ProjectResponse;
import com.cloud.api.query.vo.ProjectJoinVO;
@ -26,7 +28,7 @@ import com.cloud.utils.db.GenericDao;
public interface ProjectJoinDao extends GenericDao<ProjectJoinVO, Long> {
ProjectResponse newProjectResponse(ProjectJoinVO proj);
ProjectResponse newProjectResponse(EnumSet<DomainDetails> details, ProjectJoinVO proj);
List<ProjectJoinVO> newProjectView(Project proj);

View File

@ -17,13 +17,16 @@
package com.cloud.api.query.dao;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import javax.inject.Inject;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.api.ApiConstants.DomainDetails;
import org.apache.cloudstack.api.response.ProjectResponse;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
@ -68,7 +71,7 @@ public class ProjectJoinDaoImpl extends GenericDaoBase<ProjectJoinVO, Long> impl
}
@Override
public ProjectResponse newProjectResponse(ProjectJoinVO proj) {
public ProjectResponse newProjectResponse(EnumSet<DomainDetails> details, ProjectJoinVO proj) {
ProjectResponse response = new ProjectResponse();
response.setId(proj.getUuid());
response.setName(proj.getName());
@ -89,9 +92,11 @@ public class ProjectJoinDaoImpl extends GenericDaoBase<ProjectJoinVO, Long> impl
//set resource limit/count information for the project (by getting the info of the project's account)
Account account = _accountDao.findByIdIncludingRemoved(proj.getProjectAccountId());
AccountJoinVO accountJn = ApiDBUtils.newAccountView(account);
_accountJoinDao.setResourceLimits(accountJn, false, response);
response.setProjectAccountName(accountJn.getAccountName());
if (details.contains(DomainDetails.all) || details.contains(DomainDetails.resource)) {
AccountJoinVO accountJn = ApiDBUtils.newAccountView(account);
_accountJoinDao.setResourceLimits(accountJn, false, response);
}
response.setProjectAccountName(account.getAccountName());
response.setObjectName("project");
return response;

View File

@ -0,0 +1,205 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
""" tests for create/list domain,account,project
"""
# Import Local Modules
from nose.plugins.attrib import attr
from marvin.cloudstackTestCase import cloudstackTestCase
from marvin.lib.utils import cleanup_resources
from marvin.lib.base import (Account,
Project,
Domain)
from marvin.lib.common import get_domain
class Services:
"""Test Project Services
"""
def __init__(self):
self.services = {
"domain": {
"name": "Test Domain",
},
"project": {
"name": "Test Project",
"displaytext": "Test project",
},
"account": {
"email": "administrator@cloudstack.apache.org",
"firstname": "Test",
"lastname": "User",
"username": "test",
"password": "password",
},
}
class TestDomainAccountProject(cloudstackTestCase):
@classmethod
def setUpClass(cls):
cls.testClient = super(
TestDomainAccountProject,
cls).getClsTestClient()
cls.api_client = cls.testClient.getApiClient()
cls.services = Services().services
cls.domain = get_domain(cls.api_client)
cls._cleanup = []
return
@classmethod
def tearDownClass(cls):
try:
# Cleanup resources used
cleanup_resources(cls.api_client, cls._cleanup)
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
return
def setUp(self):
self.apiclient = self.testClient.getApiClient()
self.cleanup = []
return
def tearDown(self):
try:
# Clean up, terminate the created accounts, domains etc
cleanup_resources(self.apiclient, self.cleanup)
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
return
def test_01_create_list_domain_account_project(self):
""" Verify list domain, account and project return expected response
"""
# Validate the following
# 1. Create domain
# 2. list domain, 'cpulimit' should be included in response
# 3. list domain with details=min, 'cpulimit' should not be included in response.
# 4. create account in the domain
# 5. list account, 'cpulimit' should be included in response
# 6. list account with details=min, 'cpulimit' should not be included in response.
# 7. create project in the domain
# 8. list project, 'cpulimit' should be included in response
# 9. list project with details=min, 'cpulimit' should not be included in response.
# Create new domain
self.user_domain = Domain.create(
self.apiclient,
self.services["domain"],
parentdomainid=self.domain.id)
list_domain_response = Domain.list(
self.apiclient,
id = self.user_domain.id)
self.assertEqual(
isinstance(list_domain_response, list),
True,
"Check list response returns a valid list"
)
self.assertIsNotNone(list_domain_response[0].cpulimit, "'cpulimit' should be included in response")
list_domain_response = Domain.list(
self.apiclient,
details="min",
id = self.user_domain.id)
self.assertEqual(
isinstance(list_domain_response, list),
True,
"Check list response returns a valid list"
)
self.assertIsNone(list_domain_response[0].cpulimit, "'cpulimit' should not be included in response")
# Create account
self.account = Account.create(
self.apiclient,
self.services["account"],
admin=True,
domainid=self.user_domain.id
)
list_account_response = Account.list(
self.apiclient,
id = self.account.id)
self.assertEqual(
isinstance(list_account_response, list),
True,
"Check list response returns a valid list"
)
self.assertIsNotNone(list_account_response[0].cpulimit, "'cpulimit' should be included in response")
list_account_response = Account.list(
self.apiclient,
details="min",
id = self.account.id)
self.assertEqual(
isinstance(list_account_response, list),
True,
"Check list response returns a valid list"
)
self.assertIsNone(list_account_response[0].cpulimit, "'cpulimit' should not be included in response")
# Create project
self.project = Project.create(
self.apiclient,
self.services["project"],
account=self.account.name,
domainid=self.account.domainid
)
list_project_response = Project.list(
self.apiclient,
listall="true",
id = self.project.id)
self.assertEqual(
isinstance(list_project_response, list),
True,
"Check list response returns a valid list"
)
self.assertIsNotNone(list_project_response[0].cpulimit, "'cpulimit' should be included in response")
list_project_response = Project.list(
self.apiclient,
details="min",
listall="true",
id = self.project.id)
self.assertEqual(
isinstance(list_project_response, list),
True,
"Check list response returns a valid list"
)
self.assertIsNone(list_project_response[0].cpulimit, "'cpulimit' should not be included in response")
self.cleanup.append(self.project)
self.cleanup.append(self.account)
self.cleanup.append(self.user_domain)

View File

@ -149,6 +149,10 @@
}
}
$.extend(data, {
details: 'min'
});
$.ajax({
url: createURL('listAccounts'),
data: data,

View File

@ -2387,6 +2387,7 @@
};
$.ajax({
url: createURL('listAccounts', {
details: 'min',
ignoreProject: true
}),
data: dataObj,
@ -2420,6 +2421,7 @@
var dataObj = {
domainId: args.domainid,
state: 'Active',
details: 'min',
listAll: true,
};
$.ajax({

View File

@ -36,6 +36,10 @@
}
}
$.extend(data, {
details: 'min'
});
$.ajax({
url: createURL('listAccounts'),
async: false,
@ -187,4 +191,4 @@
}
}
};
}(jQuery, cloudStack));
}(jQuery, cloudStack));

View File

@ -635,6 +635,7 @@
var page = 1;
var getNextPage = function() {
var data2 = $.extend({}, data1, {
details: 'min',
page: page,
pageSize: 500
});
@ -777,6 +778,10 @@
});
}
$.extend(data, {
details: 'min',
});
$.ajax({
url: createURL('listProjects', {
ignoreProject: true

View File

@ -512,7 +512,7 @@ var addGuestNetworkDialog = {
select: function(args) {
var items = [];
$.ajax({
url: createURL("listProjects&listAll=true"),
url: createURL("listProjects&listAll=true&details=min"),
dataType: "json",
async: false,
success: function(json) {

View File

@ -2044,6 +2044,7 @@
$.ajax({
url: createURL('listAccounts&domainid=' + args.domainId),
data: {
details: 'min',
listAll: true
},
success: function (json) {
@ -2068,7 +2069,7 @@
select: function(args) {
var items = [];
$.ajax({
url: createURL("listProjects&domainid=" + args.domainId),
url: createURL("listProjects&details=min&domainid=" + args.domainId),
dataType: "json",
async: false,
success: function(json) {
@ -11027,7 +11028,7 @@
}
}
$.ajax({
url: createURL("listAccounts&listAll=true&page=" + args.page + "&pagesize=" + pageSize + array1.join("")),
url: createURL("listAccounts&listAll=true&details=min&page=" + args.page + "&pagesize=" + pageSize + array1.join("")),
success: function (json) {
var accountObjs = json.listaccountsresponse.account;
if (accountObjs != null) {