apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-12-15 18:12:35 +01:00
Code Issues Packages Projects Releases Wiki Activity

CLOUDSTACK-3862: Remove S3/Swift-backed Secondary Storage Code.

Browse Source
This commit is contained in:
Min Chen 2013-07-29 14:31:33 -07:00
parent e6ec6137ab
commit 42da3efbc3
17 changed files with 63 additions and 1534 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
client/tomcatconf/applicationContext.xml.in
View File

@ -785,8 +785,7 @@
<bean id="remoteAccessVpnManagerImpl" class="com.cloud.network.vpn.RemoteAccessVpnManagerImpl" />
<bean id="resourceLimitManagerImpl" class="com.cloud.resourcelimit.ResourceLimitManagerImpl" />
<bean id="rulesManagerImpl" class="com.cloud.network.rules.RulesManagerImpl" />
<bean id="s3ManagerImpl" class="com.cloud.storage.s3.S3ManagerImpl" />
<bean id="site2SiteVpnManagerImpl" class="com.cloud.network.vpn.Site2SiteVpnManagerImpl" />
<bean id="site2SiteVpnManagerImpl" class="com.cloud.network.vpn.Site2SiteVpnManagerImpl" />
<bean id="snapshotManagerImpl" class="com.cloud.storage.snapshot.SnapshotManagerImpl" />
<bean id="snapshotSchedulerImpl" class="com.cloud.storage.snapshot.SnapshotSchedulerImpl" />
<bean id="storageNetworkManagerImpl" class="com.cloud.network.StorageNetworkManagerImpl" />

61
core/src/com/cloud/agent/api/DownloadSnapshotFromS3Command.java
View File

@ -1,61 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.agent.api;
import com.cloud.agent.api.to.S3TO;
public class DownloadSnapshotFromS3Command extends SnapshotCommand {
private S3TO s3;
private String parent;
protected DownloadSnapshotFromS3Command() {
super();
}
public DownloadSnapshotFromS3Command(S3TO s3, String parent,
String secondaryStorageUrl, Long dcId, Long accountId,
Long volumeId, String backupUuid, int wait) {
super(null, secondaryStorageUrl, backupUuid, "", dcId, accountId,
volumeId);
this.s3 = s3;
this.parent = parent;
setWait(wait);
}
public S3TO getS3() {
return s3;
}
public void setS3(S3TO s3) {
this.s3 = s3;
}
public String getParent() {
return parent;
}
public void setParent(String parent) {
this.parent = parent;
}
}

60
core/src/com/cloud/agent/api/DownloadSnapshotFromSwiftCommand.java
View File

@ -1,60 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.agent.api;
import com.cloud.agent.api.LogLevel.Log4jLevel;
import com.cloud.agent.api.to.SwiftTO;
/**
* This currently assumes that both primary and secondary storage are mounted on the XenServer.
*/
public class DownloadSnapshotFromSwiftCommand extends SnapshotCommand {
@LogLevel(Log4jLevel.Off)
private SwiftTO _swift;
private String _parent;
protected DownloadSnapshotFromSwiftCommand() {
}
public DownloadSnapshotFromSwiftCommand(SwiftTO swift, String secondaryStorageUrl, Long dcId, Long accountId, Long volumeId, String parent, String BackupUuid, int wait) {
super(null, secondaryStorageUrl, BackupUuid, "", dcId, accountId, volumeId);
setParent(parent);
setSwift(swift);
setWait(wait);
}
public SwiftTO getSwift() {
return this._swift;
}
public void setSwift(SwiftTO swift) {
this._swift = swift;
}
public String getParent() {
return _parent;
}
public void setParent(String parent) {
this._parent = parent;
}
}

66
core/src/com/cloud/agent/api/DownloadTemplateFromS3ToSecondaryStorageCommand.java
View File

@ -1,66 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.agent.api;
import com.cloud.agent.api.to.S3TO;
public final class DownloadTemplateFromS3ToSecondaryStorageCommand extends Command {
private final S3TO s3;
private final Long accountId;
private final Long templateId;
private final String storagePath;
public DownloadTemplateFromS3ToSecondaryStorageCommand(final S3TO s3,
final Long accountId, final Long templateId,
final String storagePath, final int wait) {
super();
this.s3 = s3;
this.accountId = accountId;
this.templateId = templateId;
this.storagePath = storagePath;
setWait(wait);
}
public S3TO getS3() {
return this.s3;
}
public Long getAccountId() {
return this.accountId;
}
public Long getTemplateId() {
return this.templateId;
}
public String getStoragePath() {
return this.storagePath;
}
@Override
public boolean executeInSequence() {
return true;
}
}

90
core/src/com/cloud/agent/api/DownloadTemplateFromSwiftToSecondaryStorageCommand.java
View File

@ -1,90 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.agent.api;
import com.cloud.agent.api.LogLevel.Log4jLevel;
import com.cloud.agent.api.to.SwiftTO;
/**
*
*
*/
public class DownloadTemplateFromSwiftToSecondaryStorageCommand extends Command {
@LogLevel(Log4jLevel.Off)
private SwiftTO swift;
private String secondaryStorageUrl;
private Long dcId;
private Long accountId;
private Long templateId;
private String path;
protected DownloadTemplateFromSwiftToSecondaryStorageCommand() {
}
public DownloadTemplateFromSwiftToSecondaryStorageCommand(SwiftTO swift, String secondaryStorageUrl, Long dcId, Long accountId, Long templateId, String path, int wait) {
this.swift = swift;
this.secondaryStorageUrl = secondaryStorageUrl;
this.dcId = dcId;
this.accountId = accountId;
this.templateId = templateId;
this.path = path;
setWait(wait);
}
public SwiftTO getSwift() {
return this.swift;
}
public void setSwift(SwiftTO swift) {
this.swift = swift;
}
public String getSecondaryStorageUrl() {
return secondaryStorageUrl;
}
public Long getDcId() {
return dcId;
}
public Long getAccountId() {
return accountId;
}
public Long getTemplateId() {
return templateId;
}
public String getPath() {
return path;
}
public void setPath(String path) {
this.path = path;
}
@Override
public boolean executeInSequence() {
// TODO Auto-generated method stub
return true;
}
}

121
core/src/com/cloud/agent/api/UploadTemplateToS3FromSecondaryStorageCommand.java
View File

@ -1,121 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.agent.api;
import com.cloud.agent.api.to.S3TO;
public class UploadTemplateToS3FromSecondaryStorageCommand extends Command {
private final S3TO s3;
private final String storagePath;
private final Long dataCenterId;
private final Long accountId;
private final Long templateId;
public UploadTemplateToS3FromSecondaryStorageCommand(final S3TO s3,
final String storagePath, final Long dataCenterId, final Long accountId,
final Long templateId) {
super();
this.s3 = s3;
this.storagePath = storagePath;
this.dataCenterId = dataCenterId;
this.accountId = accountId;
this.templateId = templateId;
}
@Override
public boolean executeInSequence() {
return false;
}
@Override
public boolean equals(final Object thatObject) {
if (this == thatObject) {
return true;
}
if (thatObject == null || getClass() != thatObject.getClass()) {
return false;
}
final UploadTemplateToS3FromSecondaryStorageCommand thatCommand =
(UploadTemplateToS3FromSecondaryStorageCommand) thatObject;
if (this.accountId != null ? !this.accountId.equals(thatCommand
.accountId) : thatCommand.accountId != null) {
return false;
}
if (this.dataCenterId != null ? !this.dataCenterId.equals(thatCommand
.dataCenterId) : thatCommand.dataCenterId != null) {
return false;
}
if (this.s3 != null ? !this.s3.equals(thatCommand.s3) : thatCommand.s3 != null) {
return false;
}
if (this.storagePath != null ? !this.storagePath.equals(thatCommand
.storagePath) : thatCommand.storagePath != null) {
return false;
}
if (this.templateId != null ? !this.templateId.equals(thatCommand.templateId) :
thatCommand.templateId != null) {
return false;
}
return true;
}
@Override
public int hashCode() {
int result = this.s3 != null ? this.s3.hashCode() : 0;
result = 31 * result + (this.storagePath != null ? this.storagePath.hashCode() : 0);
result = 31 * result + (this.dataCenterId != null ? this.dataCenterId.hashCode() : 0);
result = 31 * result + (this.accountId != null ? this.accountId.hashCode() : 0);
result = 31 * result + (this.templateId != null ? this.templateId.hashCode() : 0);
return result;
}
public S3TO getS3() {
return this.s3;
}
public String getStoragePath() {
return this.storagePath;
}
public Long getDataCenterId() {
return this.dataCenterId;
}
public Long getAccountId() {
return this.accountId;
}
public Long getTemplateId() {
return this.templateId;
}
}

80
core/src/com/cloud/agent/api/UploadTemplateToSwiftFromSecondaryStorageCommand.java
View File

@ -1,80 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.agent.api;
import com.cloud.agent.api.LogLevel.Log4jLevel;
import com.cloud.agent.api.to.SwiftTO;
/**
*
*
*/
public class UploadTemplateToSwiftFromSecondaryStorageCommand extends Command {
@LogLevel(Log4jLevel.Off)
private SwiftTO swift;
private String secondaryStorageUrl;
private Long dcId;
private Long accountId;
private Long templateId;
protected UploadTemplateToSwiftFromSecondaryStorageCommand() {
}
public UploadTemplateToSwiftFromSecondaryStorageCommand(SwiftTO swift, String secondaryStorageUrl, Long dcId, Long accountId, Long templateId, int wait) {
this.swift = swift;
this.secondaryStorageUrl = secondaryStorageUrl;
this.dcId = dcId;
this.accountId = accountId;
this.templateId = templateId;
setWait(wait);
}
public SwiftTO getSwift() {
return this.swift;
}
public void setSwift(SwiftTO swift) {
this.swift = swift;
}
public String getSecondaryStorageUrl() {
return secondaryStorageUrl;
}
public Long getDcId() {
return dcId;
}
public Long getAccountId() {
return accountId;
}
public Long getTemplateId() {
return templateId;
}
@Override
public boolean executeInSequence() {
// TODO Auto-generated method stub
return true;
}
}

5
engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java
View File

@ -76,7 +76,6 @@ import com.cloud.storage.dao.VMTemplateZoneDaoImpl;
import com.cloud.storage.dao.VolumeDaoImpl;
import com.cloud.storage.dao.VolumeHostDaoImpl;
import com.cloud.storage.download.DownloadMonitorImpl;
import com.cloud.storage.s3.S3Manager;
import com.cloud.storage.secondary.SecondaryStorageVmManager;
import com.cloud.storage.snapshot.SnapshotManager;
import com.cloud.tags.dao.ResourceTagsDaoImpl;
@ -186,10 +185,6 @@ public class ChildTestConfiguration extends TestConfiguration {
return Mockito.mock(VirtualMachineManager.class);
}
@Bean
public S3Manager s3Mgr() {
return Mockito.mock(S3Manager.class);
}
@Bean
public SnapshotManager snapshotMgr() {

1
plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/lifecycle/S3ImageStoreLifeCycleImpl.java
View File

@ -40,7 +40,6 @@ import com.cloud.resource.Discoverer;
import com.cloud.resource.ResourceManager;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.ScopeType;
import com.cloud.storage.s3.S3Manager;
public class S3ImageStoreLifeCycleImpl implements ImageStoreLifeCycle {

120
server/src/com/cloud/resource/ResourceManagerImpl.java
View File

@ -134,7 +134,6 @@ import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.dao.GuestOSCategoryDao;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.s3.S3Manager;
import com.cloud.storage.secondary.SecondaryStorageVmManager;
import com.cloud.template.VirtualMachineTemplate;
import com.cloud.user.Account;
@ -194,8 +193,6 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
@Inject
protected HostDao _hostDao;
@Inject
protected S3Manager _s3Mgr;
@Inject
protected HostDetailsDao _hostDetailsDao;
@Inject
protected ConfigurationDao _configDao;
@ -379,8 +376,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
// Check if the zone exists in the system
DataCenterVO zone = _dcDao.findById(dcId);
if (zone == null) {
InvalidParameterValueException ex = new InvalidParameterValueException(
"Can't find zone by the id specified");
InvalidParameterValueException ex = new InvalidParameterValueException(
"Can't find zone by the id specified");
ex.addProxyObject(String.valueOf(dcId), "dcId");
throw ex;
}
@ -404,8 +401,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
}
// check if pod belongs to the zone
if (!Long.valueOf(pod.getDataCenterId()).equals(dcId)) {
InvalidParameterValueException ex = new InvalidParameterValueException(
"Pod with specified id doesn't belong to the zone " + dcId);
InvalidParameterValueException ex = new InvalidParameterValueException(
"Pod with specified id doesn't belong to the zone " + dcId);
ex.addProxyObject(pod.getUuid(), "podId");
ex.addProxyObject(zone.getUuid(), "dcId");
throw ex;
@ -548,8 +545,9 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
@Override
public Discoverer getMatchingDiscover(Hypervisor.HypervisorType hypervisorType) {
for (Discoverer discoverer : _discoverers) {
if (discoverer.getHypervisorType() == hypervisorType)
if (discoverer.getHypervisorType() == hypervisorType) {
return discoverer;
}
}
return null;
}
@ -575,8 +573,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
if (clusterId != null) {
ClusterVO cluster = _clusterDao.findById(clusterId);
if (cluster == null) {
InvalidParameterValueException ex = new InvalidParameterValueException(
"can not find cluster for specified clusterId");
InvalidParameterValueException ex = new InvalidParameterValueException(
"can not find cluster for specified clusterId");
ex.addProxyObject(clusterId.toString(), "clusterId");
throw ex;
} else {
@ -598,7 +596,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
@Override
public List<? extends Host> discoverHosts(AddSecondaryStorageCmd cmd) throws IllegalArgumentException, DiscoveryException,
InvalidParameterValueException {
InvalidParameterValueException {
Long dcId = cmd.getZoneId();
String url = cmd.getUrl();
return discoverHostsFull(dcId, null, null, null, url, null, null, "SecondaryStorage", null, null, false);
@ -631,11 +629,11 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
}
// check if pod belongs to the zone
if (!Long.valueOf(pod.getDataCenterId()).equals(dcId)) {
InvalidParameterValueException ex = new InvalidParameterValueException(
"Pod with specified podId"
+ podId
+ " doesn't belong to the zone with specified zoneId"
+ dcId);
InvalidParameterValueException ex = new InvalidParameterValueException(
"Pod with specified podId"
+ podId
+ " doesn't belong to the zone with specified zoneId"
+ dcId);
ex.addProxyObject(pod.getUuid(), "podId");
ex.addProxyObject(zone.getUuid(), "dcId");
throw ex;
@ -694,11 +692,11 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
} catch (Exception e) {
cluster = _clusterDao.findBy(clusterName, podId);
if (cluster == null) {
CloudRuntimeException ex = new CloudRuntimeException(
"Unable to create cluster "
+ clusterName
+ " in pod with specified podId and data center with specified dcID",
e);
CloudRuntimeException ex = new CloudRuntimeException(
"Unable to create cluster "
+ clusterName
+ " in pod with specified podId and data center with specified dcID",
e);
ex.addProxyObject(pod.getUuid(), "podId");
ex.addProxyObject(zone.getUuid(), "dcId");
throw ex;
@ -970,11 +968,11 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
if (hypervisorType == HypervisorType.VMware && Boolean.parseBoolean(_configDao.getValue(Config.VmwareUseNexusVSwitch.toString()))) {
_clusterVSMMapDao.removeByClusterId(cmd.getId());
}
// remove from dedicated resources
DedicatedResourceVO dr = _dedicatedDao.findByClusterId(cluster.getId());
if (dr != null) {
_dedicatedDao.remove(dr.getId());
}
// remove from dedicated resources
DedicatedResourceVO dr = _dedicatedDao.findByClusterId(cluster.getId());
if (dr != null) {
_dedicatedDao.remove(dr.getId());
}
}
txn.commit();
@ -1439,8 +1437,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
Iterator<Map.Entry<String, ResourceStateAdapter>> it = _resourceStateAdapters.entrySet().iterator();
Object result = null;
while (it.hasNext()) {
Map.Entry<String, ResourceStateAdapter> item = it
.next();
Map.Entry<String, ResourceStateAdapter> item = it
.next();
ResourceStateAdapter adapter = item.getValue();
String msg = new String("Dispatching resource state event " + event + " to " + item.getKey());
@ -1683,7 +1681,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
private void markHostAsDisconnected(HostVO host, StartupCommand[] cmds) {
if (host == null) { // in case host is null due to some errors, try
// reloading the host from db
// reloading the host from db
if (cmds != null) {
StartupCommand firstCmd = cmds[0];
host = findHostByGuid(firstCmd.getGuid());
@ -1733,13 +1731,13 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
host = findHostByGuid(firstCmd.getGuidWithoutResource());
}
if (host != null && host.getRemoved() == null) { // host already
// added, no
// need to add
// again
// added, no
// need to add
// again
s_logger.debug("Found the host " + host.getId() + " by guid: " + firstCmd.getGuid() + ", old host reconnected as new");
hostExists = true; // ensures that host status is left
// unchanged in case of adding same one
// again
// unchanged in case of adding same one
// again
return null;
}
}
@ -1805,13 +1803,13 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
host = findHostByGuid(firstCmd.getGuidWithoutResource());
}
if (host != null && host.getRemoved() == null) { // host already
// added, no
// need to add
// again
// added, no
// need to add
// again
s_logger.debug("Found the host " + host.getId() + " by guid: " + firstCmd.getGuid() + ", old host reconnected as new");
hostExists = true; // ensures that host status is left
// unchanged in case of adding same one
// again
// unchanged in case of adding same one
// again
return null;
}
}
@ -1820,29 +1818,29 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
GlobalLock addHostLock = GlobalLock.getInternLock("AddHostLock");
try {
if (addHostLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) { // to
// safely
// determine
// first
// host
// in
// cluster
// in
// multi-MS
// scenario
// safely
// determine
// first
// host
// in
// cluster
// in
// multi-MS
// scenario
try {
host = createHostVO(cmds, resource, details, hostTags, ResourceStateAdapter.Event.CREATE_HOST_VO_FOR_DIRECT_CONNECT);
if (host != null) {
deferAgentCreation = !isFirstHostInCluster(host); // if
// first
// host
// in
// cluster
// no
// need
// to
// defer
// agent
// creation
// first
// host
// in
// cluster
// no
// need
// to
// defer
// agent
// creation
}
} finally {
addHostLock.unlock();
@ -1854,8 +1852,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
if (host != null) {
if (!deferAgentCreation) { // if first host in cluster then
// create agent otherwise defer it to
// scan task
// create agent otherwise defer it to
// scan task
attache = _agentMgr.handleDirectConnectAgent(host, cmds, resource, forRebalance);
host = _hostDao.findById(host.getId()); // reload
} else {
@ -2533,7 +2531,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
sc.addAnd(sc.getEntity().getStatus(), Op.EQ, Status.Up);
sc.addAnd(sc.getEntity().getResourceState(), Op.EQ, ResourceState.Enabled);
return sc.list();
}
}
@Override
@DB

5
server/src/com/cloud/server/ManagementServerImpl.java
View File

@ -529,7 +529,6 @@ import com.cloud.storage.dao.GuestOSCategoryDao;
import com.cloud.storage.dao.GuestOSDao;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.s3.S3Manager;
import com.cloud.storage.secondary.SecondaryStorageVmManager;
import com.cloud.storage.snapshot.SnapshotManager;
import com.cloud.tags.ResourceTagVO;
@ -691,10 +690,6 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
DataStoreManager dataStoreMgr;
@Inject
HostTagsDao _hostTagsDao;
@Inject
S3Manager _s3Mgr;
@Inject
ConfigurationServer _configServer;
@Inject

10
server/src/com/cloud/storage/VolumeManagerImpl.java
View File

@ -31,11 +31,9 @@ import java.util.concurrent.ExecutionException;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd;
import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd;
@ -45,7 +43,6 @@ import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd;
import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd;
import org.apache.cloudstack.api.command.user.volume.UpdateVolumeCmd;
import org.apache.cloudstack.api.command.user.volume.UploadVolumeCmd;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
@ -62,7 +59,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult;
import org.apache.cloudstack.framework.async.AsyncCallFuture;
import org.apache.cloudstack.storage.command.AttachAnswer;
@ -77,12 +73,9 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.commons.lang.StringUtils;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.storage.CreateVolumeOVAAnswer;
import com.cloud.agent.api.storage.CreateVolumeOVACommand;
import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.VirtualMachineTO;
@ -148,7 +141,6 @@ import com.cloud.storage.dao.VMTemplateS3Dao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.dao.VolumeDetailsDao;
import com.cloud.storage.download.DownloadMonitor;
import com.cloud.storage.s3.S3Manager;
import com.cloud.storage.secondary.SecondaryStorageVmManager;
import com.cloud.storage.snapshot.SnapshotApiService;
import com.cloud.storage.snapshot.SnapshotManager;
@ -243,8 +235,6 @@ public class VolumeManagerImpl extends ManagerBase implements VolumeManager {
@Inject
protected VMTemplateS3Dao _vmTemplateS3Dao;
@Inject
protected S3Manager _s3Mgr;
@Inject
protected VMTemplateDao _vmTemplateDao = null;
@Inject
protected StoragePoolHostDao _poolHostDao = null;

67
server/src/com/cloud/storage/s3/S3Manager.java
View File

@ -1,67 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.storage.s3;
import java.util.List;
import java.util.Map;
import com.cloud.agent.api.to.S3TO;
import org.apache.cloudstack.api.command.admin.storage.AddS3Cmd;
import org.apache.cloudstack.api.command.admin.storage.ListS3sCmd;
import com.cloud.dc.DataCenterVO;
import com.cloud.exception.DiscoveryException;
import com.cloud.storage.S3;
import com.cloud.storage.S3VO;
import com.cloud.storage.VMTemplateS3VO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.utils.component.Manager;
public interface S3Manager extends Manager {
S3TO getS3TO();
S3TO getS3TO(Long s3Id);
S3 addS3(AddS3Cmd addS3Cmd) throws DiscoveryException;
void verifyS3Fields(Map<String, String> params) throws DiscoveryException;
Long chooseZoneForTemplateExtract(VMTemplateVO template);
boolean isS3Enabled();
boolean isTemplateInstalled(Long templateId);
//void deleteTemplate(final Long accountId, final Long templateId);
String downloadTemplateFromS3ToSecondaryStorage(final long dcId,
final long templateId, final int primaryStorageDownloadWait);
List<S3VO> listS3s(ListS3sCmd listS3sCmd);
VMTemplateS3VO findByTemplateId(Long templateId);
void propagateTemplatesToZone(DataCenterVO zone);
void propagateTemplateToAllZones(VMTemplateS3VO vmTemplateS3VO);
void uploadTemplateToS3FromSecondaryStorage(final VMTemplateVO template);
}

609
server/src/com/cloud/storage/s3/S3ManagerImpl.java
View File

@ -1,609 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.storage.s3;
import static com.cloud.storage.S3VO.ID_COLUMN_NAME;
import static com.cloud.utils.DateUtil.now;
import static com.cloud.utils.S3Utils.canConnect;
import static com.cloud.utils.S3Utils.canReadWriteBucket;
import static com.cloud.utils.S3Utils.checkBucketName;
import static com.cloud.utils.S3Utils.checkClientOptions;
import static com.cloud.utils.S3Utils.doesBucketExist;
import static com.cloud.utils.StringUtils.join;
import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock;
import static java.lang.Boolean.TRUE;
import static java.lang.String.format;
import static java.util.Collections.emptyList;
import static java.util.Collections.shuffle;
import static java.util.Collections.singletonList;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
import javax.ejb.Local;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.command.admin.storage.AddS3Cmd;
import org.apache.cloudstack.api.command.admin.storage.ListS3sCmd;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.DownloadTemplateFromS3ToSecondaryStorageCommand;
import com.cloud.agent.api.UploadTemplateToS3FromSecondaryStorageCommand;
import com.cloud.agent.api.to.S3TO;
import com.cloud.configuration.dao.ConfigurationDao;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.exception.DiscoveryException;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.storage.S3;
import com.cloud.storage.S3VO;
import com.cloud.storage.VMTemplateHostVO;
import com.cloud.storage.VMTemplateS3VO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.VMTemplateZoneVO;
import com.cloud.storage.dao.S3Dao;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplateHostDao;
import com.cloud.storage.dao.VMTemplateS3Dao;
import com.cloud.storage.dao.VMTemplateZoneDao;
import com.cloud.utils.S3Utils.ClientOptions;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.db.Filter;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.exception.CloudRuntimeException;
@Component
@Local(value = { S3Manager.class })
public class S3ManagerImpl extends ManagerBase implements S3Manager {
private static final Logger LOGGER = Logger.getLogger(S3ManagerImpl.class);
@Inject
private AgentManager agentManager;
@Inject
private S3Dao s3Dao;
@Inject
private VMTemplateZoneDao vmTemplateZoneDao;
@Inject
private VMTemplateS3Dao vmTemplateS3Dao;
@Inject
private VMTemplateHostDao vmTemplateHostDao;
@Inject
private VMTemplateDao vmTemplateDao;
@Inject
private ConfigurationDao configurationDao;
@Inject
private DataCenterDao dataCenterDao;
@Inject
private HostDao hostDao;
@Inject
private DataStoreManager dataStoreManager;
public S3ManagerImpl() {
}
private void verifyConnection(final S3TO s3) throws DiscoveryException {
if (!canConnect(s3)) {
throw new DiscoveryException(format("Unable to connect to S3 "
+ "using access key %1$s, secret key %2$s, and endpoint, "
+ "%3$S", s3.getAccessKey(), s3.getSecretKey(),
s3.getEndPoint() != null ? s3.getEndPoint() : "default"));
}
}
private void verifyBuckets(S3TO s3) throws DiscoveryException {
final List<String> errorMessages = new ArrayList<String>();
errorMessages.addAll(verifyBucket(s3, s3.getBucketName()));
throwDiscoveryExceptionFromErrorMessages(errorMessages);
}
private List<String> verifyBucket(final ClientOptions clientOptions,
final String bucketName) {
if (!doesBucketExist(clientOptions, bucketName)) {
return singletonList(format("Bucket %1$s does not exist.",
bucketName));
}
if (!canReadWriteBucket(clientOptions, bucketName)) {
return singletonList(format("Can read/write from bucket %1$s.",
bucketName));
}
return emptyList();
}
private void validateFields(final S3VO s3VO) {
final List<String> errorMessages = new ArrayList<String>();
errorMessages.addAll(checkClientOptions(s3VO.toS3TO()));
errorMessages.addAll(checkBucketName("template", s3VO.getBucketName()));
throwDiscoveryExceptionFromErrorMessages(errorMessages);
}
private void enforceS3PreConditions() throws DiscoveryException {
if (!this.isS3Enabled()) {
throw new DiscoveryException("S3 is not enabled.");
}
if (this.getS3TO() != null) {
throw new DiscoveryException("Attempt to define multiple S3 "
+ "instances. Only one instance definition is supported.");
}
}
private void throwDiscoveryExceptionFromErrorMessages(
final List<String> errorMessages) {
if (!errorMessages.isEmpty()) {
throw new CloudRuntimeException(join(errorMessages, " "));
}
}
static String determineLockId(final long accountId, final long templateId) {
// TBD The lock scope may be too coarse grained. Deletes need to lock
// the template across all zones where upload and download could
// probably safely scoped to the zone ...
return join("_", "S3_TEMPLATE", accountId, templateId);
}
@Override
public S3TO getS3TO(final Long s3Id) {
return this.s3Dao.getS3TO(s3Id);
}
@Override
public S3TO getS3TO() {
final List<S3VO> s3s = this.s3Dao.listAll();
if (s3s == null || (s3s != null && s3s.isEmpty())) {
return null;
}
if (s3s.size() == 1) {
return s3s.get(0).toS3TO();
}
throw new CloudRuntimeException("Multiple S3 instances have been "
+ "defined. Only one instance configuration is supported.");
}
@Override
public S3 addS3(final AddS3Cmd addS3Cmd) throws DiscoveryException {
this.enforceS3PreConditions();
final S3VO s3VO = new S3VO(UUID.randomUUID().toString(),
addS3Cmd.getAccessKey(), addS3Cmd.getSecretKey(),
addS3Cmd.getEndPoint(), addS3Cmd.getBucketName(),
addS3Cmd.getHttpsFlag(), addS3Cmd.getConnectionTimeout(),
addS3Cmd.getMaxErrorRetry(), addS3Cmd.getSocketTimeout(), now());
this.validateFields(s3VO);
final S3TO s3 = s3VO.toS3TO();
this.verifyConnection(s3);
this.verifyBuckets(s3);
return this.s3Dao.persist(s3VO);
}
@Override
public void verifyS3Fields(Map<String, String> params) throws DiscoveryException {
final S3VO s3VO = new S3VO(UUID.randomUUID().toString(),
params.get(ApiConstants.S3_ACCESS_KEY),
params.get(ApiConstants.S3_SECRET_KEY),
params.get(ApiConstants.S3_END_POINT),
params.get(ApiConstants.S3_BUCKET_NAME),
params.get(ApiConstants.S3_HTTPS_FLAG) == null ? false : Boolean.valueOf(params.get(ApiConstants.S3_HTTPS_FLAG)),
params.get(ApiConstants.S3_CONNECTION_TIMEOUT) == null ? null : Integer.valueOf(params.get(ApiConstants.S3_CONNECTION_TIMEOUT)),
params.get(ApiConstants.S3_MAX_ERROR_RETRY) == null ? null : Integer.valueOf(params.get(ApiConstants.S3_MAX_ERROR_RETRY)),
params.get(ApiConstants.S3_SOCKET_TIMEOUT) == null ? null : Integer.valueOf(params.get(ApiConstants.S3_SOCKET_TIMEOUT)), now());
this.validateFields(s3VO);
final S3TO s3 = s3VO.toS3TO();
this.verifyConnection(s3);
this.verifyBuckets(s3);
}
@Override
public boolean isS3Enabled() {
return false;
}
@Override
public boolean isTemplateInstalled(final Long templateId) {
throw new UnsupportedOperationException(
"S3Manager#isTemplateInstalled (DeleteIsoCmd) has not yet "
+ "been implemented");
}
@SuppressWarnings("unchecked")
@Override
public String downloadTemplateFromS3ToSecondaryStorage(
final long dataCenterId, final long templateId,
final int primaryStorageDownloadWait) {
if (!isS3Enabled()) {
return null;
}
final VMTemplateVO template = vmTemplateDao.findById(templateId);
if (template == null) {
final String errorMessage = String
.format("Failed to download template id %1$s from S3 because the template definition was not found.",
templateId);
LOGGER.error(errorMessage);
return errorMessage;
}
final VMTemplateS3VO templateS3VO = findByTemplateId(templateId);
if (templateS3VO == null) {
final String errorMessage = format(
"Failed to download template id %1$s from S3 because it does not exist in S3.",
templateId);
LOGGER.error(errorMessage);
return errorMessage;
}
final S3TO s3 = getS3TO(templateS3VO.getS3Id());
if (s3 == null) {
final String errorMessage = format(
"Failed to download template id %1$s from S3 because S3 id %2$s does not exist.",
templateId, templateS3VO);
LOGGER.error(errorMessage);
return errorMessage;
}
final DataStore secondaryStore = this.dataStoreManager.getImageStore(dataCenterId);
if (secondaryStore == null) {
final String errorMessage = format(
"Unable to find secondary storage for zone id %1$s.",
dataCenterId);
LOGGER.error(errorMessage);
throw new CloudRuntimeException(errorMessage);
}
final long accountId = template.getAccountId();
final DownloadTemplateFromS3ToSecondaryStorageCommand cmd = new DownloadTemplateFromS3ToSecondaryStorageCommand(
s3, accountId, templateId, secondaryStore.getName(),
primaryStorageDownloadWait);
try {
executeWithNoWaitLock(determineLockId(accountId, templateId),
new Callable<Void>() {
@Override
public Void call() throws Exception {
final Answer answer = agentManager.sendToSSVM(
dataCenterId, cmd);
if (answer == null || !answer.getResult()) {
final String errMsg = String
.format("Failed to download template from S3 to secondary storage due to %1$s",
(answer == null ? "answer is null"
: answer.getDetails()));
LOGGER.error(errMsg);
throw new CloudRuntimeException(errMsg);
}
final String installPath = join(File.separator, "template", "tmpl", accountId, templateId);
final VMTemplateHostVO tmpltHost = new VMTemplateHostVO(
secondaryStore.getId(), templateId,
now(), 100, Status.DOWNLOADED, null, null,
null, installPath, template.getUrl());
tmpltHost.setSize(templateS3VO.getSize());
tmpltHost.setPhysicalSize(templateS3VO
.getPhysicalSize());
vmTemplateHostDao.persist(tmpltHost);
return null;
}
});
} catch (Exception e) {
final String errMsg = "Failed to download template from S3 to secondary storage due to "
+ e.toString();
LOGGER.error(errMsg);
throw new CloudRuntimeException(errMsg);
}
return null;
}
@Override
public List<S3VO> listS3s(final ListS3sCmd cmd) {
final Filter filter = new Filter(S3VO.class, ID_COLUMN_NAME, TRUE,
cmd.getStartIndex(), cmd.getPageSizeVal());
final SearchCriteria<S3VO> criteria = this.s3Dao.createSearchCriteria();
return this.s3Dao.search(criteria, filter);
}
@Override
public VMTemplateS3VO findByTemplateId(final Long templateId) {
throw new UnsupportedOperationException(
"S3Manager#findByTemplateId(Long) has not yet "
+ "been implemented");
}
@Override
public void propagateTemplatesToZone(final DataCenterVO zone) {
if (!isS3Enabled()) {
return;
}
final List<VMTemplateS3VO> s3VMTemplateRefs = this.vmTemplateS3Dao
.listAll();
if (LOGGER.isInfoEnabled()) {
LOGGER.info(format("Propagating %1$s templates to zone %2$s.",
s3VMTemplateRefs.size(), zone.getName()));
}
for (final VMTemplateS3VO templateS3VO : s3VMTemplateRefs) {
this.vmTemplateZoneDao.persist(new VMTemplateZoneVO(zone.getId(),
templateS3VO.getTemplateId(), now()));
}
}
@Override
public boolean configure(final String name, final Map<String, Object> params)
throws ConfigurationException {
if (LOGGER.isInfoEnabled()) {
LOGGER.info(format("Configuring S3 Manager %1$s", name));
}
return true;
}
@Override
public boolean start() {
LOGGER.info("Starting S3 Manager");
return true;
}
@Override
public boolean stop() {
LOGGER.info("Stopping S3 Manager");
return true;
}
@Override
public void propagateTemplateToAllZones(final VMTemplateS3VO vmTemplateS3VO) {
final long templateId = vmTemplateS3VO.getId();
if (!isS3Enabled()) {
if (LOGGER.isTraceEnabled()) {
LOGGER.trace(format(
"Attempt to propogate template id %1$s across all zones. However, S3 is not enabled.",
templateId));
}
return;
}
final S3TO s3 = getS3TO();
if (s3 == null) {
LOGGER.warn(format(
"Unable to propagate template id %1$s across all zones because S3 is enabled, but not configured.",
templateId));
return;
}
if (vmTemplateS3VO != null) {
final List<DataCenterVO> dataCenters = dataCenterDao.listAll();
for (DataCenterVO dataCenter : dataCenters) {
final VMTemplateZoneVO tmpltZoneVO = new VMTemplateZoneVO(
dataCenter.getId(), templateId, now());
vmTemplateZoneDao.persist(tmpltZoneVO);
}
}
}
@Override
public Long chooseZoneForTemplateExtract(VMTemplateVO template) {
final S3TO s3 = getS3TO();
if (s3 == null) {
return null;
}
final List<VMTemplateHostVO> templateHosts = vmTemplateHostDao
.listByOnlyTemplateId(template.getId());
if (templateHosts != null) {
shuffle(templateHosts);
for (VMTemplateHostVO vmTemplateHostVO : templateHosts) {
final HostVO host = hostDao.findById(vmTemplateHostVO
.getHostId());
if (host != null) {
return host.getDataCenterId();
}
throw new CloudRuntimeException(
format("Unable to find secondary storage host for template id %1$s.",
template.getId()));
}
}
final List<DataCenterVO> dataCenters = dataCenterDao.listAll();
shuffle(dataCenters);
return dataCenters.get(0).getId();
}
@Override
public void uploadTemplateToS3FromSecondaryStorage(
final VMTemplateVO template) {
final Long templateId = template.getId();
final List<VMTemplateHostVO> templateHostRefs = vmTemplateHostDao
.listByTemplateId(templateId);
if (templateHostRefs == null
|| (templateHostRefs != null && templateHostRefs.isEmpty())) {
throw new CloudRuntimeException(
format("Attempt to sync template id %1$s that is not attached to a host.",
templateId));
}
final VMTemplateHostVO templateHostRef = templateHostRefs.get(0);
if (!isS3Enabled()) {
return;
}
final S3TO s3 = getS3TO();
if (s3 == null) {
LOGGER.warn("S3 Template Sync Failed: Attempt to sync templates with S3, but no S3 instance defined.");
return;
}
final HostVO secondaryHost = this.hostDao.findById(templateHostRef
.getHostId());
if (secondaryHost == null) {
throw new CloudRuntimeException(format(
"Unable to find secondary storage host id %1$s.",
templateHostRef.getHostId()));
}
final Long dataCenterId = secondaryHost.getDataCenterId();
final Long accountId = template.getAccountId();
try {
executeWithNoWaitLock(determineLockId(accountId, templateId),
new Callable<Void>() {
@Override
public Void call() throws Exception {
final UploadTemplateToS3FromSecondaryStorageCommand cmd = new UploadTemplateToS3FromSecondaryStorageCommand(
s3, secondaryHost.getStorageUrl(),
dataCenterId, accountId, templateId);
final Answer answer = agentManager.sendToSSVM(
dataCenterId, cmd);
if (answer == null || !answer.getResult()) {
final String reason = answer != null ? answer
.getDetails()
: "S3 template sync failed due to an unspecified error.";
throw new CloudRuntimeException(
format("Failed to upload template id %1$s to S3 from secondary storage due to %2$s.",
templateId, reason));
}
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format(
"Creating VMTemplateS3VO instance using template id %1s.",
templateId));
}
final VMTemplateS3VO vmTemplateS3VO = new VMTemplateS3VO(
s3.getId(), templateId, now(),
templateHostRef.getSize(), templateHostRef
.getPhysicalSize());
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Persisting %1$s",
vmTemplateS3VO));
}
vmTemplateS3Dao.persist(vmTemplateS3VO);
propagateTemplateToAllZones(vmTemplateS3VO);
return null;
}
});
} catch (Exception e) {
final String errorMessage = format(
"Failed to upload template id %1$s for zone id %2$s to S3.",
templateId, dataCenterId);
LOGGER.error(errorMessage, e);
}
}
}

6
server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java
View File

@ -99,7 +99,6 @@ import com.cloud.storage.dao.SnapshotPolicyDao;
import com.cloud.storage.dao.SnapshotScheduleDao;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.s3.S3Manager;
import com.cloud.storage.secondary.SecondaryStorageVmManager;
import com.cloud.storage.template.TemplateConstants;
import com.cloud.tags.ResourceTagVO;
@ -179,8 +178,6 @@ public class SnapshotManagerImpl extends ManagerBase implements SnapshotManager,
@Inject
private ResourceLimitService _resourceLimitMgr;
@Inject
private S3Manager _s3Mgr;
@Inject
private SecondaryStorageVmManager _ssvmMgr;
@Inject
private DomainManager _domainMgr;
@ -962,9 +959,10 @@ public class SnapshotManagerImpl extends ManagerBase implements SnapshotManager,
|| userVm.getHypervisorType() == HypervisorType.KVM) {
List<SnapshotVO> activeSnapshots = _snapshotDao.listByInstanceId(volume.getInstanceId(),
Snapshot.State.Creating, Snapshot.State.CreatedOnPrimary, Snapshot.State.BackingUp);
if (activeSnapshots.size() > 1)
if (activeSnapshots.size() > 1) {
throw new CloudRuntimeException(
"There is other active snapshot tasks on the instance to which the volume is attached, please try again later");
}
}
List<VMSnapshotVO> activeVMSnapshots = _vmSnapshotDao.listByInstanceId(userVm.getId(),

94
server/src/com/cloud/template/S3SyncTask.java
View File

@ -1,94 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.template;
import static java.lang.String.*;
import java.util.List;
import org.apache.log4j.Logger;
import com.cloud.agent.api.to.S3TO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.s3.S3Manager;
final class S3SyncTask implements Runnable {
private static final Logger LOGGER = Logger.getLogger(S3SyncTask.class);
private final VMTemplateDao vmTemplateDao;
private final S3Manager s3Mgr;
S3SyncTask(final VMTemplateDao vmTemplateDao, final S3Manager s3Mgr) {
super();
assert vmTemplateDao != null;
assert s3Mgr != null;
this.vmTemplateDao = vmTemplateDao;
this.s3Mgr = s3Mgr;
}
@Override
public void run() {
try {
final S3TO s3 = s3Mgr.getS3TO();
if (s3 == null) {
LOGGER.warn("S3 sync skipped because no S3 instance is configured.");
return;
}
final List<VMTemplateVO> candidateTemplates = vmTemplateDao
.findTemplatesToSyncToS3();
if (candidateTemplates.isEmpty()) {
LOGGER.debug("All templates are synced with S3.");
return;
}
for (VMTemplateVO candidateTemplate : candidateTemplates) {
if (LOGGER.isInfoEnabled()) {
LOGGER.info(format(
"Uploading template %1$s (id: %2$s) to S3.",
candidateTemplate.getName(),
candidateTemplate.getId()));
}
s3Mgr.uploadTemplateToS3FromSecondaryStorage(candidateTemplate);
}
LOGGER.debug("Completed S3 template sync task.");
} catch (Exception e) {
LOGGER.warn(
"S3 Sync Task ignored exception, and will continue to execute.",
e);
}
}
}

199
services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
View File

@ -71,9 +71,6 @@ import com.cloud.agent.api.CheckHealthCommand;
import com.cloud.agent.api.Command;
import com.cloud.agent.api.ComputeChecksumCommand;
import com.cloud.agent.api.DeleteSnapshotsDirCommand;
import com.cloud.agent.api.DownloadSnapshotFromS3Command;
import com.cloud.agent.api.DownloadSnapshotFromSwiftCommand;
import com.cloud.agent.api.DownloadTemplateFromSwiftToSecondaryStorageCommand;
import com.cloud.agent.api.GetStorageStatsAnswer;
import com.cloud.agent.api.GetStorageStatsCommand;
import com.cloud.agent.api.PingCommand;
@ -88,7 +85,6 @@ import com.cloud.agent.api.SecStorageSetupCommand.Certificates;
import com.cloud.agent.api.SecStorageVMSetupCommand;
import com.cloud.agent.api.StartupCommand;
import com.cloud.agent.api.StartupSecondaryStorageCommand;
import com.cloud.agent.api.UploadTemplateToSwiftFromSecondaryStorageCommand;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.DataTO;
@ -198,16 +194,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
return execute((ListTemplateCommand) cmd);
} else if (cmd instanceof ListVolumeCommand) {
return execute((ListVolumeCommand) cmd);
} else if (cmd instanceof DownloadSnapshotFromSwiftCommand) {
return execute((DownloadSnapshotFromSwiftCommand) cmd);
} else if (cmd instanceof DownloadSnapshotFromS3Command) {
return execute((DownloadSnapshotFromS3Command) cmd);
} else if (cmd instanceof DeleteSnapshotsDirCommand) {
return execute((DeleteSnapshotsDirCommand) cmd);
} else if (cmd instanceof DownloadTemplateFromSwiftToSecondaryStorageCommand) {
return execute((DownloadTemplateFromSwiftToSecondaryStorageCommand) cmd);
} else if (cmd instanceof UploadTemplateToSwiftFromSecondaryStorageCommand) {
return execute((UploadTemplateToSwiftFromSecondaryStorageCommand) cmd);
} else if (cmd instanceof CopyCommand) {
return execute((CopyCommand) cmd);
} else if (cmd instanceof DeleteCommand) {
@ -590,50 +578,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
return join(asList(getRootDir(storagePath), dataPath), File.separator);
}
private Answer execute(DownloadTemplateFromSwiftToSecondaryStorageCommand cmd) {
SwiftTO swift = cmd.getSwift();
String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
Long accountId = cmd.getAccountId();
Long templateId = cmd.getTemplateId();
String path = cmd.getPath();
String errMsg;
String lDir = null;
try {
String parent = getRootDir(secondaryStorageUrl);
lDir = parent + "/template/tmpl/" + accountId.toString() + "/" + templateId.toString();
String result = createLocalDir(lDir);
if (result != null) {
errMsg = "downloadTemplateFromSwiftToSecondaryStorageCommand failed due to Create local directory failed";
s_logger.warn(errMsg);
throw new InternalErrorException(errMsg);
}
String lPath = lDir + "/" + path;
result = swiftDownload(swift, "T-" + templateId.toString(), path, lPath);
if (result != null) {
errMsg = "failed to download template " + path + " from Swift to secondary storage " + lPath
+ " , err=" + result;
s_logger.warn(errMsg);
throw new CloudRuntimeException(errMsg);
}
path = "template.properties";
lPath = lDir + "/" + path;
result = swiftDownload(swift, "T-" + templateId.toString(), path, lPath);
if (result != null) {
errMsg = "failed to download template " + path + " from Swift to secondary storage " + lPath
+ " , err=" + result;
s_logger.warn(errMsg);
throw new CloudRuntimeException(errMsg);
}
return new Answer(cmd, true, "success");
} catch (Exception e) {
if (lDir != null) {
deleteLocalDir(lDir);
}
errMsg = cmd + " Command failed due to " + e.toString();
s_logger.warn(errMsg, e);
return new Answer(cmd, false, errMsg);
}
}
protected File downloadFromUrlToNfs(String url, NfsTO nfs, String path, String name) {
HttpClient client = new DefaultHttpClient();
@ -732,33 +676,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
private Answer execute(UploadTemplateToSwiftFromSecondaryStorageCommand cmd) {
SwiftTO swift = cmd.getSwift();
String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
Long accountId = cmd.getAccountId();
Long templateId = cmd.getTemplateId();
try {
String parent = getRootDir(secondaryStorageUrl);
String lPath = parent + "/template/tmpl/" + accountId.toString() + "/" + templateId.toString();
if (!_storage.isFile(lPath + "/template.properties")) {
String errMsg = cmd + " Command failed due to template doesn't exist ";
s_logger.debug(errMsg);
return new Answer(cmd, false, errMsg);
}
String result = swiftUpload(swift, "T-" + templateId.toString(), lPath, "*");
if (result != null) {
String errMsg = "failed to upload template from secondary storage " + lPath + " to swift , err="
+ result;
s_logger.debug(errMsg);
return new Answer(cmd, false, errMsg);
}
return new Answer(cmd, true, "success");
} catch (Exception e) {
String errMsg = cmd + " Command failed due to " + e.toString();
s_logger.warn(errMsg, e);
return new Answer(cmd, false, errMsg);
}
}
private ImageFormat getTemplateFormat(String filePath) {
String ext = null;
@ -1063,73 +981,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
}
public Answer execute(final DownloadSnapshotFromS3Command cmd) {
final S3TO s3 = cmd.getS3();
final String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
final Long accountId = cmd.getAccountId();
final Long volumeId = cmd.getVolumeId();
try {
executeWithNoWaitLock(determineSnapshotLockId(accountId, volumeId), new Callable<Void>() {
@Override
public Void call() throws Exception {
final String directoryName = determineSnapshotLocalDirectory(secondaryStorageUrl, accountId,
volumeId);
String result = createLocalDir(directoryName);
if (result != null) {
throw new InternalErrorException(format(
"Failed to create directory %1$s during S3 snapshot download.", directoryName));
}
final String snapshotFileName = determineSnapshotBackupFilename(cmd.getSnapshotUuid());
final String key = determineSnapshotS3Key(accountId, volumeId, snapshotFileName);
final File targetFile = S3Utils.getFile(s3, s3.getBucketName(), key,
_storage.getFile(directoryName), new FileNamingStrategy() {
@Override
public String determineFileName(String key) {
return snapshotFileName;
}
});
if (cmd.getParent() != null) {
final String parentPath = join(File.pathSeparator, directoryName,
determineSnapshotBackupFilename(cmd.getParent()));
result = setVhdParent(targetFile.getAbsolutePath(), parentPath);
if (result != null) {
throw new InternalErrorException(format(
"Failed to set the parent for backup %1$s to %2$s due to %3$s.",
targetFile.getAbsolutePath(), parentPath, result));
}
}
return null;
}
});
return new Answer(cmd, true, format(
"Succesfully retrieved volume id %1$s for account id %2$s to %3$s from S3.", volumeId, accountId,
secondaryStorageUrl));
} catch (Exception e) {
final String errMsg = format(
"Failed to retrieve volume id %1$s for account id %2$s to %3$s from S3 due to exception %4$s",
volumeId, accountId, secondaryStorageUrl, e.getMessage());
s_logger.error(errMsg);
return new Answer(cmd, false, errMsg);
}
}
private String determineSnapshotS3Directory(final Long accountId, final Long volumeId) {
return join(S3Utils.SEPARATOR, SNAPSHOT_ROOT_DIR, accountId, volumeId);
@ -1147,54 +998,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
return join(File.pathSeparator, getRootDir(secondaryStorageUrl), SNAPSHOT_ROOT_DIR, accountId, volumeId);
}
public Answer execute(DownloadSnapshotFromSwiftCommand cmd) {
SwiftTO swift = cmd.getSwift();
String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
Long accountId = cmd.getAccountId();
Long volumeId = cmd.getVolumeId();
String rFilename = cmd.getSnapshotUuid();
String sParent = cmd.getParent();
String errMsg = "";
try {
String parent = getRootDir(secondaryStorageUrl);
String lPath = parent + "/snapshots/" + String.valueOf(accountId) + "/" + String.valueOf(volumeId);
String result = createLocalDir(lPath);
if (result != null) {
errMsg = "downloadSnapshotFromSwiftCommand failed due to Create local path failed";
s_logger.warn(errMsg);
throw new InternalErrorException(errMsg);
}
String lFilename = rFilename;
if (rFilename.startsWith("VHD-")) {
lFilename = rFilename.replace("VHD-", "") + ".vhd";
}
String lFullPath = lPath + "/" + lFilename;
result = swiftDownload(swift, "S-" + volumeId.toString(), rFilename, lFullPath);
if (result != null) {
return new Answer(cmd, false, result);
}
if (sParent != null) {
if (sParent.startsWith("VHD-") || sParent.endsWith(".vhd")) {
String pFilename = sParent;
if (sParent.startsWith("VHD-")) {
pFilename = pFilename.replace("VHD-", "") + ".vhd";
}
String pFullPath = lPath + "/" + pFilename;
result = setVhdParent(lFullPath, pFullPath);
if (result != null) {
return new Answer(cmd, false, result);
}
}
}
return new Answer(cmd, true, "success");
} catch (Exception e) {
String msg = cmd + " Command failed due to " + e.toString();
s_logger.warn(msg, e);
throw new CloudRuntimeException(msg);
}
}
private Answer execute(ComputeChecksumCommand cmd) {