From c216990e1c2ca1eda40f367ed774f27b31980be4 Mon Sep 17 00:00:00 2001 From: Edison Su Date: Wed, 19 Dec 2012 18:43:04 -0800 Subject: [PATCH] trying to add code to download url into devcloud --- engine/storage/integration-test/pom.xml | 6 + .../storage/test/DirectAgentTest.java | 33 +++++ .../cloudstack/storage/test/TestHttp.java | 88 +++++++++++++ .../cloudstack/storage/to/TemplateTO.java | 7 +- .../resource/XenServerStorageResource.java | 116 ++++++++++++++---- scripts/vm/hypervisor/xenserver/storagePlugin | 56 +++++++++ 6 files changed, 273 insertions(+), 33 deletions(-) create mode 100644 engine/storage/integration-test/test/org/apache/cloudstack/storage/test/TestHttp.java create mode 100755 scripts/vm/hypervisor/xenserver/storagePlugin diff --git a/engine/storage/integration-test/pom.xml b/engine/storage/integration-test/pom.xml index 947614c5588..0dbc9a28832 100644 --- a/engine/storage/integration-test/pom.xml +++ b/engine/storage/integration-test/pom.xml @@ -50,6 +50,12 @@ ${project.version} test + + org.apache.httpcomponents + httpclient + 4.2.2 + compile + mysql mysql-connector-java diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentTest.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentTest.java index 852f3791139..2f5e7aebf26 100644 --- a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentTest.java +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentTest.java @@ -22,9 +22,15 @@ import java.util.UUID; import javax.inject.Inject; +import org.apache.cloudstack.storage.command.CopyTemplateToPrimaryStorageCmd; +import org.apache.cloudstack.storage.to.ImageDataStoreTO; +import org.apache.cloudstack.storage.to.ImageOnPrimayDataStoreTO; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.TemplateTO; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; @@ -122,4 +128,31 @@ public class DirectAgentTest { e.printStackTrace(); } } + + @Test + public void testDownloadTemplate() { + ImageOnPrimayDataStoreTO image = Mockito.mock(ImageOnPrimayDataStoreTO.class); + PrimaryDataStoreTO primaryStore = Mockito.mock(PrimaryDataStoreTO.class); + Mockito.when(primaryStore.getUuid()).thenReturn("cd10cac1-4772-92e5-5da6-c2bc16b1ce1b"); + Mockito.when(image.getPrimaryDataStore()).thenReturn(primaryStore); + + ImageDataStoreTO imageStore = Mockito.mock(ImageDataStoreTO.class); + Mockito.when(imageStore.getType()).thenReturn("http"); + + TemplateTO template = Mockito.mock(TemplateTO.class); + Mockito.when(template.getPath()).thenReturn("http://download.cloud.com/templates/devcloud/defaulttemplates/5/ce5b212e-215a-3461-94fb-814a635b2215.vhd"); + Mockito.when(template.getImageDataStore()).thenReturn(imageStore); + + Mockito.when(image.getTemplate()).thenReturn(template); + CopyTemplateToPrimaryStorageCmd cmd = new CopyTemplateToPrimaryStorageCmd(image); + try { + agentMgr.send(hostId, cmd); + } catch (AgentUnavailableException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (OperationTimedoutException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } } diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/TestHttp.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/TestHttp.java new file mode 100644 index 00000000000..9de1c21e997 --- /dev/null +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/TestHttp.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.test; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.FileChannel; + +import junit.framework.Assert; + +import org.apache.commons.httpclient.HttpException; +import org.apache.cxf.helpers.IOUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.impl.client.DefaultHttpClient; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(locations="classpath:/resource/storageContext.xml") +public class TestHttp { + @Test + public void testHttpclient() { + HttpHead method = new HttpHead("http://download.cloud.com/templates/devcloud/defaulttemplates/5/ce5b212e-215a-3461-94fb-814a635b2215.vhd"); + DefaultHttpClient client = new DefaultHttpClient(); + + OutputStream output = null; + long length = 0; + try { + HttpResponse response = client.execute(method); + length = Long.parseLong(response.getFirstHeader("Content-Length").getValue()); + System.out.println(response.getFirstHeader("Content-Length").getValue()); + File localFile = new File("/tmp/test"); + if (!localFile.exists()) { + localFile.createNewFile(); + } + + HttpGet getMethod = new HttpGet("http://download.cloud.com/templates/devcloud/defaulttemplates/5/ce5b212e-215a-3461-94fb-814a635b2215.vhd"); + response = client.execute(getMethod); + HttpEntity entity = response.getEntity(); + + output = new BufferedOutputStream(new FileOutputStream(localFile)); + entity.writeTo(output); + } catch (HttpException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } finally { + try { + if (output != null) + output.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + File f = new File("/tmp/test"); + Assert.assertEquals(f.length(), length); + } +} diff --git a/engine/storage/src/org/apache/cloudstack/storage/to/TemplateTO.java b/engine/storage/src/org/apache/cloudstack/storage/to/TemplateTO.java index 8d73fcc10b3..f565712011d 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/to/TemplateTO.java +++ b/engine/storage/src/org/apache/cloudstack/storage/to/TemplateTO.java @@ -8,13 +8,12 @@ public class TemplateTO { private final String uuid; private final VolumeDiskType diskType; private final ImageDataStoreTO imageDataStore; - private final long size = 0; + public TemplateTO(TemplateInfo template) { this.path = template.getPath(); this.uuid = template.getUuid(); this.diskType = template.getDiskType(); this.imageDataStore = new ImageDataStoreTO(template.getImageDataStore()); - // this.size = template.getVirtualSize(); } public String getPath() { @@ -32,8 +31,4 @@ public class TemplateTO { public ImageDataStoreTO getImageDataStore() { return this.imageDataStore; } - - public long getSize() { - return this.size; - } } diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java index 435734c6413..8789bb7f753 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java @@ -18,6 +18,9 @@ */ package com.cloud.hypervisor.xen.resource; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; import java.util.HashMap; @@ -36,6 +39,13 @@ import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.impl.client.DefaultHttpClient; import org.apache.log4j.Logger; import org.apache.xmlrpc.XmlRpcException; @@ -67,31 +77,70 @@ public class XenServerStorageResource { } private long getTemplateSize(String url) { - /* - HttpGet method = new HttpGet(url); - HttpClient client = new HttpClient(); + HttpHead method = new HttpHead(url); + DefaultHttpClient client = new DefaultHttpClient(); try { - int responseCode = client.executeMethod(method); - if (responseCode != HttpStatus.SC_OK) { - throw new CloudRuntimeException("http get returns error code:" + responseCode); + HttpResponse response = client.execute(method); + Header header = response.getFirstHeader("Content-Length"); + if (header == null) { + throw new CloudRuntimeException("Can't get content-lenght header from :" + url); } - method.get + Long length = Long.parseLong(header.getValue()); + return length; } catch (HttpException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + throw new CloudRuntimeException("Failed to get template lenght", e); } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + throw new CloudRuntimeException("Failed to get template lenght", e); + } catch (NumberFormatException e) { + throw new CloudRuntimeException("Failed to get template lenght", e); } - */ - return 0; } - protected Answer directDownloadHttpTemplate(TemplateTO template, PrimaryDataStoreTO primarDataStore) { + private void downloadHttpToLocalFile(String destFilePath, String url) { + File destFile = new File(destFilePath); + if (!destFile.exists()) { + throw new CloudRuntimeException("dest file doesn't exist: " + destFilePath); + } + + DefaultHttpClient client = new DefaultHttpClient(); + HttpGet getMethod = new HttpGet(url); + HttpResponse response; + BufferedOutputStream output = null; + long length = 0; + try { + response = client.execute(getMethod); + HttpEntity entity = response.getEntity(); + length = entity.getContentLength(); + output = new BufferedOutputStream(new FileOutputStream(destFile)); + entity.writeTo(output); + } catch (ClientProtocolException e) { + throw new CloudRuntimeException("Failed to download template", e); + } catch (IOException e) { + throw new CloudRuntimeException("Failed to download template", e); + } finally { + if (output != null) { + try { + output.close(); + } catch (IOException e) { + throw new CloudRuntimeException("Failed to download template", e); + } + } + } + + //double check the length + destFile = new File(destFilePath); + if (destFile.length() != length) { + throw new CloudRuntimeException("Download file length doesn't match: expected: " + length + ", actual: " + destFile.length()); + } + + } + + protected Answer directDownloadHttpTemplate(CopyTemplateToPrimaryStorageCmd cmd, TemplateTO template, PrimaryDataStoreTO primarDataStore) { String primaryStoreUuid = primarDataStore.getUuid(); Connection conn = hypervisorResource.getConnection(); SR poolsr = null; VDI vdi = null; + boolean result = false; try { Set srs = SR.getByNameLabel(conn, primaryStoreUuid); @@ -103,8 +152,8 @@ public class XenServerStorageResource { vdir.nameLabel = "Base-Image-" + UUID.randomUUID().toString(); vdir.SR = poolsr; vdir.type = Types.VdiType.USER; - - vdir.virtualSize = template.getSize(); + + vdir.virtualSize = getTemplateSize(template.getPath()); vdi = VDI.create(conn, vdir); vdir = vdi.getRecord(conn); @@ -114,27 +163,40 @@ public class XenServerStorageResource { throw new CloudRuntimeException("Don't how to handle multiple pbds:" + pbds.size() + " for sr: " + poolsr.getUuid(conn)); } PBD pbd = pbds.iterator().next(); - PBD.Record pbdRec = pbd.getRecord(conn); Map deviceCfg = pbd.getDeviceConfig(conn); String pbdLocation = deviceCfg.get("location"); if (pbdLocation == null) { throw new CloudRuntimeException("Can't get pbd: " + pbd.getUuid(conn) + " location"); } - String vdiPath = pbdLocation + "/" + vdiLocation; + String vdiPath = pbdLocation + "/" + vdiLocation + ".vhd"; //download a url into vdipath - + //downloadHttpToLocalFile(vdiPath, template.getPath()); + hypervisorResource.callHostPlugin(conn, "vmopsStorage", "downloadTemplateFromUrl", "destPath", vdiPath, "srcUrl", template.getPath()); + result = true; + return new CopyTemplateToPrimaryStorageAnswer(cmd, vdi.getUuid(conn)); } catch (BadServerResponse e) { - // TODO Auto-generated catch block - e.printStackTrace(); + s_logger.debug("Failed to download template", e); } catch (XenAPIException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + s_logger.debug("Failed to download template", e); } catch (XmlRpcException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + s_logger.debug("Failed to download template", e); + } catch (Exception e) { + s_logger.debug("Failed to download template", e); + } finally { + if (!result && vdi != null) { + try { + vdi.destroy(conn); + } catch (BadServerResponse e) { + s_logger.debug("Failed to cleanup newly created vdi"); + } catch (XenAPIException e) { + s_logger.debug("Failed to cleanup newly created vdi"); + } catch (XmlRpcException e) { + s_logger.debug("Failed to cleanup newly created vdi"); + } + } } - return null; + return new Answer(cmd, false, "Failed to download template"); } protected Answer execute(CopyTemplateToPrimaryStorageCmd cmd) { @@ -142,7 +204,7 @@ public class XenServerStorageResource { TemplateTO template = imageTO.getTemplate(); ImageDataStoreTO imageStore = template.getImageDataStore(); if (imageStore.getType().equalsIgnoreCase("http")) { - return directDownloadHttpTemplate(template, imageTO.getPrimaryDataStore()); + return directDownloadHttpTemplate(cmd, template, imageTO.getPrimaryDataStore()); } else { return new Answer(cmd, false, "not implemented yet"); /* diff --git a/scripts/vm/hypervisor/xenserver/storagePlugin b/scripts/vm/hypervisor/xenserver/storagePlugin new file mode 100755 index 00000000000..df1c340752e --- /dev/null +++ b/scripts/vm/hypervisor/xenserver/storagePlugin @@ -0,0 +1,56 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Version @VERSION@ +# +# A plugin for executing script needed by vmops cloud + +import os, sys, time +import XenAPIPlugin +sys.path.extend(["/opt/xensource/sm/", "/usr/local/sbin/", "/sbin/"]) +import base64 +import hostvmstats +import socket +import stat +import tempfile +import util +import subprocess +import zlib +import urllib2 +from util import CommandException + +def echo(fn): + def wrapped(*v, **k): + name = fn.__name__ + util.SMlog("#### VMOPS enter %s ####" % name ) + res = fn(*v, **k) + util.SMlog("#### VMOPS exit %s ####" % name ) + return res + return wrapped + +@echo +def downloadTemplateFromUrl(session, args): + destPath = args["destPath"] + srcUrl = args["srcUrl"] + template = urllib2.urlopen(srcUrl) + destFile = open(destPath, "wb") + destFile.write(template.read()) + destFile.close() + +if __name__ == "__main__": + XenAPIPlugin.dispatch({"downloadTemplateFromUrl": downloadTemplateFromUrl})