trying to add code to download url into devcloud

engine/storage/integration-test/pom.xml

@@ -50,6 +50,12 @@
       <version>${project.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+    <groupId>org.apache.httpcomponents</groupId>
+    <artifactId>httpclient</artifactId>
+    <version>4.2.2</version>
+    <scope>compile</scope>
+  </dependency>
     <dependency>
       <groupId>mysql</groupId>
       <artifactId>mysql-connector-java</artifactId>

engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentTest.java

@@ -22,9 +22,15 @@ import java.util.UUID;
 
 import javax.inject.Inject;
 
+import org.apache.cloudstack.storage.command.CopyTemplateToPrimaryStorageCmd;
+import org.apache.cloudstack.storage.to.ImageDataStoreTO;
+import org.apache.cloudstack.storage.to.ImageOnPrimayDataStoreTO;
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+import org.apache.cloudstack.storage.to.TemplateTO;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.mockito.Mockito;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 
@@ -122,4 +128,31 @@ public class DirectAgentTest {
             e.printStackTrace();
         }
     }
+    
+    @Test
+    public void testDownloadTemplate() {
+        ImageOnPrimayDataStoreTO image = Mockito.mock(ImageOnPrimayDataStoreTO.class);
+        PrimaryDataStoreTO primaryStore = Mockito.mock(PrimaryDataStoreTO.class);
+        Mockito.when(primaryStore.getUuid()).thenReturn("cd10cac1-4772-92e5-5da6-c2bc16b1ce1b");
+        Mockito.when(image.getPrimaryDataStore()).thenReturn(primaryStore);
+        
+        ImageDataStoreTO imageStore = Mockito.mock(ImageDataStoreTO.class);
+        Mockito.when(imageStore.getType()).thenReturn("http");
+        
+        TemplateTO template = Mockito.mock(TemplateTO.class);
+        Mockito.when(template.getPath()).thenReturn("http://download.cloud.com/templates/devcloud/defaulttemplates/5/ce5b212e-215a-3461-94fb-814a635b2215.vhd");
+        Mockito.when(template.getImageDataStore()).thenReturn(imageStore);
+        
+        Mockito.when(image.getTemplate()).thenReturn(template);
+        CopyTemplateToPrimaryStorageCmd cmd = new CopyTemplateToPrimaryStorageCmd(image);
+        try {
+            agentMgr.send(hostId, cmd);
+        } catch (AgentUnavailableException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (OperationTimedoutException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
 }

engine/storage/integration-test/test/org/apache/cloudstack/storage/test/TestHttp.java

@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.storage.test;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.channels.FileChannel;
+
+import junit.framework.Assert;
+
+import org.apache.commons.httpclient.HttpException;
+import org.apache.cxf.helpers.IOUtils;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(locations="classpath:/resource/storageContext.xml")
+public class TestHttp {
+    @Test
+    public void testHttpclient() {
+        HttpHead method = new HttpHead("http://download.cloud.com/templates/devcloud/defaulttemplates/5/ce5b212e-215a-3461-94fb-814a635b2215.vhd");
+        DefaultHttpClient client = new DefaultHttpClient();
+
+        OutputStream output = null;
+        long length = 0;
+        try {
+            HttpResponse response = client.execute(method);
+            length = Long.parseLong(response.getFirstHeader("Content-Length").getValue());
+            System.out.println(response.getFirstHeader("Content-Length").getValue());
+            File localFile = new File("/tmp/test");
+            if (!localFile.exists()) {
+               localFile.createNewFile();
+            }
+            
+            HttpGet getMethod = new HttpGet("http://download.cloud.com/templates/devcloud/defaulttemplates/5/ce5b212e-215a-3461-94fb-814a635b2215.vhd");
+            response = client.execute(getMethod);
+            HttpEntity entity = response.getEntity();
+         
+            output = new BufferedOutputStream(new FileOutputStream(localFile));
+            entity.writeTo(output);
+        } catch (HttpException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (IOException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } finally {
+            try {
+                if (output != null)
+                    output.close();
+            } catch (IOException e) {
+                // TODO Auto-generated catch block
+                e.printStackTrace();
+            }
+        }
+        
+        File f = new File("/tmp/test");
+        Assert.assertEquals(f.length(), length);
+    }
+}

engine/storage/src/org/apache/cloudstack/storage/to/TemplateTO.java

@@ -8,13 +8,12 @@ public class TemplateTO {
     private final String uuid;
     private final VolumeDiskType diskType;
     private final ImageDataStoreTO imageDataStore;
-    private final long size = 0;
+
     public TemplateTO(TemplateInfo template) {
         this.path = template.getPath();
         this.uuid = template.getUuid();
         this.diskType = template.getDiskType();
         this.imageDataStore = new ImageDataStoreTO(template.getImageDataStore());
-       // this.size = template.getVirtualSize();
     }
     
     public String getPath() {
@@ -32,8 +31,4 @@ public class TemplateTO {
     public ImageDataStoreTO getImageDataStore() {
         return this.imageDataStore;
     }
-    
-    public long getSize() {
-        return this.size;
-    }
 }

plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java

@@ -18,6 +18,9 @@
  */
 package com.cloud.hypervisor.xen.resource;
 
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URI;
 import java.util.HashMap;
@@ -36,6 +39,13 @@ import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.HttpException;
 import org.apache.commons.httpclient.HttpStatus;
 import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.log4j.Logger;
 import org.apache.xmlrpc.XmlRpcException;
 
@@ -67,31 +77,70 @@ public class XenServerStorageResource {
     }
     
     private long getTemplateSize(String url) {
-        /*
-        HttpGet method = new HttpGet(url);
-        HttpClient client = new HttpClient();
+        HttpHead method = new HttpHead(url);
+        DefaultHttpClient client = new DefaultHttpClient();
         try {
-            int responseCode = client.executeMethod(method);
-            if (responseCode != HttpStatus.SC_OK) {
-                throw new CloudRuntimeException("http get returns error code:" + responseCode);
+            HttpResponse response = client.execute(method);
+            Header header = response.getFirstHeader("Content-Length");
+            if (header == null) {
+                throw new CloudRuntimeException("Can't get content-lenght header from :" + url);
             }
-            method.get
+            Long length = Long.parseLong(header.getValue());
+            return length;
         } catch (HttpException e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
+            throw new CloudRuntimeException("Failed to get template lenght", e);
         } catch (IOException e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
+            throw new CloudRuntimeException("Failed to get template lenght", e);
+        } catch (NumberFormatException e) {
+            throw new CloudRuntimeException("Failed to get template lenght", e);
         }
-        */
-        return 0;
     }
     
-    protected Answer directDownloadHttpTemplate(TemplateTO template, PrimaryDataStoreTO primarDataStore) {
+    private void downloadHttpToLocalFile(String destFilePath, String url) {
+        File destFile = new File(destFilePath);
+        if (!destFile.exists()) {
+            throw new CloudRuntimeException("dest file doesn't exist: " + destFilePath);
+        }
+        
+        DefaultHttpClient client = new DefaultHttpClient();
+        HttpGet getMethod = new HttpGet(url);
+        HttpResponse response;
+        BufferedOutputStream output = null;
+        long length = 0;
+        try {
+            response = client.execute(getMethod);
+            HttpEntity entity = response.getEntity();
+            length = entity.getContentLength();
+            output = new BufferedOutputStream(new FileOutputStream(destFile));
+            entity.writeTo(output);
+        } catch (ClientProtocolException e) {
+           throw new CloudRuntimeException("Failed to download template", e);
+        } catch (IOException e) {
+            throw new CloudRuntimeException("Failed to download template", e);
+        } finally {
+            if (output != null) {
+                try {
+                    output.close();
+                } catch (IOException e) {
+                    throw new CloudRuntimeException("Failed to download template", e);
+                }
+            }
+        }
+        
+        //double check the length
+        destFile = new File(destFilePath);
+        if (destFile.length() != length) {
+            throw new CloudRuntimeException("Download file length doesn't match: expected: " + length + ", actual: " + destFile.length());
+        }
+       
+    }
+    
+    protected Answer directDownloadHttpTemplate(CopyTemplateToPrimaryStorageCmd cmd, TemplateTO template, PrimaryDataStoreTO primarDataStore) {
         String primaryStoreUuid = primarDataStore.getUuid();
         Connection conn = hypervisorResource.getConnection();
         SR poolsr = null;
         VDI vdi = null;
+        boolean result = false;
         try {
             
             Set<SR> srs = SR.getByNameLabel(conn, primaryStoreUuid);
@@ -103,8 +152,8 @@ public class XenServerStorageResource {
             vdir.nameLabel = "Base-Image-" + UUID.randomUUID().toString();
             vdir.SR = poolsr;
             vdir.type = Types.VdiType.USER;
-
-            vdir.virtualSize = template.getSize();
+            
+            vdir.virtualSize = getTemplateSize(template.getPath());
             vdi = VDI.create(conn, vdir);
             
             vdir = vdi.getRecord(conn);
@@ -114,27 +163,40 @@ public class XenServerStorageResource {
                 throw new CloudRuntimeException("Don't how to handle multiple pbds:" + pbds.size() + " for sr: " + poolsr.getUuid(conn));
             }
             PBD pbd = pbds.iterator().next();
-            PBD.Record pbdRec = pbd.getRecord(conn);
             Map<String, String> deviceCfg = pbd.getDeviceConfig(conn);
             String pbdLocation = deviceCfg.get("location");
             if (pbdLocation == null) {
                 throw new CloudRuntimeException("Can't get pbd: " + pbd.getUuid(conn) + " location");
             }
             
-            String vdiPath = pbdLocation + "/" + vdiLocation;
+            String vdiPath = pbdLocation + "/" + vdiLocation + ".vhd";
             //download a url into vdipath
-            
+            //downloadHttpToLocalFile(vdiPath, template.getPath());
+            hypervisorResource.callHostPlugin(conn, "vmopsStorage", "downloadTemplateFromUrl", "destPath", vdiPath, "srcUrl", template.getPath());
+            result = true;
+            return new CopyTemplateToPrimaryStorageAnswer(cmd, vdi.getUuid(conn));
         } catch (BadServerResponse e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
+            s_logger.debug("Failed to download template", e);
         } catch (XenAPIException e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
+            s_logger.debug("Failed to download template", e);
         } catch (XmlRpcException e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
+            s_logger.debug("Failed to download template", e);
+        } catch (Exception e) {
+            s_logger.debug("Failed to download template", e);
+        } finally {
+            if (!result && vdi != null) {
+                try {
+                    vdi.destroy(conn);
+                } catch (BadServerResponse e) {
+                   s_logger.debug("Failed to cleanup newly created vdi");
+                } catch (XenAPIException e) {
+                    s_logger.debug("Failed to cleanup newly created vdi");
+                } catch (XmlRpcException e) {
+                    s_logger.debug("Failed to cleanup newly created vdi");
+                }
+            }
         }
-        return null;
+        return new Answer(cmd, false, "Failed to download template");
     }
     
     protected Answer execute(CopyTemplateToPrimaryStorageCmd cmd) {
@@ -142,7 +204,7 @@ public class XenServerStorageResource {
         TemplateTO template = imageTO.getTemplate();
         ImageDataStoreTO imageStore = template.getImageDataStore();
         if (imageStore.getType().equalsIgnoreCase("http")) {
-            return directDownloadHttpTemplate(template, imageTO.getPrimaryDataStore());
+            return directDownloadHttpTemplate(cmd, template, imageTO.getPrimaryDataStore());
         } else {
             return new Answer(cmd, false, "not implemented yet");
             /*

scripts/vm/hypervisor/xenserver/storagePlugin

@@ -0,0 +1,56 @@
+#!/usr/bin/python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Version @VERSION@
+#
+# A plugin for executing script needed by vmops cloud 
+
+import os, sys, time
+import XenAPIPlugin
+sys.path.extend(["/opt/xensource/sm/", "/usr/local/sbin/", "/sbin/"])
+import base64
+import hostvmstats
+import socket
+import stat
+import tempfile
+import util
+import subprocess
+import zlib
+import urllib2
+from util import CommandException
+
+def echo(fn):
+    def wrapped(*v, **k):
+        name = fn.__name__
+        util.SMlog("#### VMOPS enter  %s ####" % name )
+        res = fn(*v, **k)
+        util.SMlog("#### VMOPS exit  %s ####" % name )
+        return res
+    return wrapped
+
+@echo
+def downloadTemplateFromUrl(session, args):
+    destPath = args["destPath"]
+    srcUrl = args["srcUrl"]
+    template = urllib2.urlopen(srcUrl)
+    destFile = open(destPath, "wb")
+    destFile.write(template.read())
+    destFile.close()
+    
+if __name__ == "__main__":
+    XenAPIPlugin.dispatch({"downloadTemplateFromUrl": downloadTemplateFromUrl})