Trigger system vm template download while adding image store. Just a

code skeleton, waiting for some code in EndPoint.

api/src/com/cloud/agent/api/storage/DownloadSystemTemplateCommand.java

@@ -0,0 +1,154 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.storage;
+
+import java.net.URI;
+
+import com.cloud.agent.api.Command;
+import com.cloud.agent.api.storage.DownloadCommand.Proxy;
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.template.VirtualMachineTemplate;
+
+
+public class DownloadSystemTemplateCommand extends Command {
+	public static class PasswordAuth {
+		String userName;
+		String password;
+		public PasswordAuth() {
+
+		}
+		public PasswordAuth(String user, String password) {
+			this.userName = user;
+			this.password = password;
+		}
+		public String getUserName() {
+			return userName;
+		}
+		public String getPassword() {
+			return password;
+		}
+	}
+
+
+	private PasswordAuth auth;
+	private Proxy _proxy;
+	private DataStoreTO _store;
+    private Long resourceId;
+    private Long accountId;
+    private String url;
+    private Long maxDownloadSizeInBytes;
+
+	protected DownloadSystemTemplateCommand() {
+	}
+
+
+
+	public DownloadSystemTemplateCommand(DataStoreTO store, String secUrl, VirtualMachineTemplate template, Long maxDownloadSizeInBytes) {
+	    this._store = store;
+	    this.accountId = template.getAccountId();
+	    this.url = secUrl;
+	    this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
+	    this.resourceId = template.getId();
+	}
+
+
+	public DownloadSystemTemplateCommand(DataStoreTO store, String secUrl, String url, VirtualMachineTemplate template, String user, String passwd, Long maxDownloadSizeInBytes) {
+        this._store = store;
+        this.accountId = template.getAccountId();
+        this.url = secUrl;
+        this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
+        this.resourceId = template.getId();
+		auth = new PasswordAuth(user, passwd);
+	}
+
+
+
+
+	public PasswordAuth getAuth() {
+		return auth;
+	}
+
+	public void setCreds(String userName, String passwd) {
+		auth = new PasswordAuth(userName, passwd);
+	}
+
+	public Proxy getProxy() {
+		return _proxy;
+	}
+
+	public void setProxy(Proxy proxy) {
+		_proxy = proxy;
+	}
+
+	public Long getMaxDownloadSizeInBytes() {
+		return maxDownloadSizeInBytes;
+	}
+
+
+    public DataStoreTO getDataStore() {
+        return _store;
+    }
+
+
+    public void setDataStore(DataStoreTO _store) {
+        this._store = _store;
+    }
+
+
+    public Long getResourceId() {
+        return resourceId;
+    }
+
+
+    public void setResourceId(Long resourceId) {
+        this.resourceId = resourceId;
+    }
+
+
+
+    public Long getAccountId() {
+        return accountId;
+    }
+
+
+
+    public void setAccountId(Long accountId) {
+        this.accountId = accountId;
+    }
+
+
+
+    public String getUrl() {
+        return url;
+    }
+
+
+
+    public void setUrl(String url) {
+        this.url = url;
+    }
+
+
+
+    @Override
+    public boolean executeInSequence() {
+        // TODO Auto-generated method stub
+        return false;
+    }
+
+
+}

core/src/com/cloud/storage/resource/LocalNfsSecondaryStorageResource.java

@@ -1,9 +1,82 @@
 package com.cloud.storage.resource;
 
+import static com.cloud.utils.StringUtils.join;
+import static java.lang.String.format;
+import static java.util.Arrays.asList;
+
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
 import org.springframework.stereotype.Component;
 
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.Command;
+import com.cloud.agent.api.storage.DownloadSystemTemplateCommand;
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.NfsTO;
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.agent.api.to.SwiftTO;
+import com.cloud.utils.S3Utils;
+import com.cloud.utils.UriUtils;
+import com.cloud.utils.exception.CloudRuntimeException;
+
 @Component
 public class LocalNfsSecondaryStorageResource extends
 		NfsSecondaryStorageResource {
 
+    @Override
+    public Answer executeRequest(Command cmd) {
+        if (cmd instanceof DownloadSystemTemplateCommand){
+            return execute((DownloadSystemTemplateCommand)cmd);
+        } else {
+            return Answer.createUnsupportedCommandAnswer(cmd);
+        }
+    }
+
+    private Answer execute(DownloadSystemTemplateCommand cmd){
+        DataStoreTO dstore = cmd.getDataStore();
+        if ( dstore instanceof S3TO ){
+            //TODO: how to handle download progress for S3
+            S3TO s3 = (S3TO)cmd.getDataStore();
+            String url = cmd.getUrl();
+            String user = null;
+            String password = null;
+            if (cmd.getAuth() != null) {
+                user = cmd.getAuth().getUserName();
+                password = new String(cmd.getAuth().getPassword());
+            }
+            // get input stream from the given url
+            InputStream in = UriUtils.getInputStreamFromUrl(url, user, password);
+            URI uri;
+            URL urlObj;
+            try {
+                uri = new URI(url);
+                urlObj = new URL(url);
+            } catch (URISyntaxException e) {
+                throw new CloudRuntimeException("URI is incorrect: " + url);
+            } catch (MalformedURLException e) {
+                throw new CloudRuntimeException("URL is incorrect: " + url);
+            }
+
+            final String bucket = s3.getBucketName();
+            String key = join(asList(determineS3TemplateDirectory(cmd.getAccountId(), cmd.getResourceId()), urlObj.getFile()), S3Utils.SEPARATOR);
+            S3Utils.putObject(s3, in, bucket, key);
+            return new Answer(cmd, true, format("Uploaded the contents of input stream from %1$s for template id %2$s to S3 bucket %3$s", url,
+                    cmd.getResourceId(), bucket));
+        }
+        else if ( dstore instanceof NfsTO ){
+            return new Answer(cmd, false, "Nfs needs to be pre-installed with system vm templates");
+        }
+        else if ( dstore instanceof SwiftTO ){
+            //TODO: need to move code from execute(uploadTemplateToSwiftFromSecondaryStorageCommand) here, but we need to handle
+            // source is url, most likely we need to modify our existing swiftUpload python script.
+            return new Answer(cmd, false, "Swift is not currently support DownloadCommand");
+        }
+        else{
+            return new Answer(cmd, false, "Unsupported image data store: " + dstore);
+        }
+    }
 }

core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java

@@ -221,7 +221,7 @@ SecondaryStorageResource {
             return Answer.createUnsupportedCommandAnswer(cmd);
         }
     }
-    
+
     protected Answer downloadFromS3ToNfs(CopyCmd cmd, DataTO srcData, S3TO s3,
     		DataTO destData, NfsTO destImageStore) {
           final String storagePath = destImageStore.getUrl();
@@ -264,27 +264,27 @@ SecondaryStorageResource {
               return new Answer(cmd, false, errMsg);
           }
     }
-    
+
     protected Answer downloadFromSwiftToNfs(CopyCmd cmd, DataTO srcData, SwiftTO srcImageStore,
     		DataTO destData, NfsTO destImageStore) {
     	return Answer.createUnsupportedCommandAnswer(cmd);
-    } 
+    }
-    
+
     protected Answer execute(CopyCmd cmd) {
     	DataTO srcData = cmd.getSrcTO();
     	DataTO destData = cmd.getDestTO();
     	DataStoreTO srcDataStore = srcData.getDataStore();
     	DataStoreTO destDataStore = destData.getDataStore();
-    	
+
-    	if (srcDataStore.getRole() == DataStoreRole.Image 
+    	if (srcDataStore.getRole() == DataStoreRole.Image
     			&& destDataStore.getRole() == DataStoreRole.ImageCache
     			) {
-    		
+
     		if (!(destDataStore instanceof NfsTO)) {
     			s_logger.debug("only support nfs as cache storage");
-    			return Answer.createUnsupportedCommandAnswer(cmd); 
+    			return Answer.createUnsupportedCommandAnswer(cmd);
     		}
-    		
+
     		if (srcDataStore instanceof S3TO) {
     			return downloadFromS3ToNfs(cmd, srcData, (S3TO)srcDataStore,
     					destData, (NfsTO)destDataStore);
@@ -292,15 +292,15 @@ SecondaryStorageResource {
     			return downloadFromSwiftToNfs(cmd, srcData, (SwiftTO)srcDataStore,
     					destData, (NfsTO)destDataStore);
     		} else {
-    			return Answer.createUnsupportedCommandAnswer(cmd); 
+    			return Answer.createUnsupportedCommandAnswer(cmd);
     		}
-    		
+
     	}
     	return Answer.createUnsupportedCommandAnswer(cmd);
     }
 
     @SuppressWarnings("unchecked")
-    private String determineS3TemplateDirectory(final Long accountId,
+    protected String determineS3TemplateDirectory(final Long accountId,
             final Long templateId) {
         return join(asList(TEMPLATE_ROOT_DIR, accountId, templateId),
                 S3Utils.SEPARATOR);

engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java

@@ -32,4 +32,5 @@ public interface TemplateService {
 
     void handleSysTemplateDownload(HypervisorType hostHyper, Long dcId);
     void handleTemplateSync(DataStore store);
+    void downloadBootstrapSysTemplate(DataStore store);
 }

engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java

@@ -185,6 +185,30 @@ public class TemplateServiceImpl implements TemplateService {
         return future;
     }
 
+    @Override
+    public void downloadBootstrapSysTemplate(DataStore store) {
+        Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
+
+        List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();
+        List<VMTemplateVO> defaultBuiltin = _templateDao.listDefaultBuiltinTemplates();
+
+        for (VMTemplateVO rtngTmplt : rtngTmplts) {
+            toBeDownloaded.add(rtngTmplt);
+        }
+
+        for (VMTemplateVO builtinTmplt : defaultBuiltin) {
+            toBeDownloaded.add(builtinTmplt);
+        }
+
+        for (VMTemplateVO template : toBeDownloaded) {
+            TemplateDataStoreVO tmpltHost = _vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId());
+            if (tmpltHost == null || tmpltHost.getState() != ObjectInDataStoreStateMachine.State.Ready) {
+                _dlMonitor.downloadBootstrapSysTemplateToStorage(template, store, null);
+            }
+        }
+    }
+
+
 
     @Override
     public void handleSysTemplateDownload(HypervisorType hostHyper, Long dcId) {

server/src/com/cloud/storage/StorageManagerImpl.java

@@ -60,6 +60,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ImageStoreProvider;
 import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator;
+import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult;
@@ -325,6 +326,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
     DataStoreManager _dataStoreMgr;
     @Inject
     DataStoreProviderManager _dataStoreProviderMgr;
+    @Inject
+    private TemplateService _imageSrv;
 
     protected List<StoragePoolAllocator> _storagePoolAllocators;
     public List<StoragePoolAllocator> getStoragePoolAllocators() {
@@ -1986,6 +1989,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
             throw new CloudRuntimeException("Failed to add data store", e);
         }
 
+        // trigger system vm template download
+        this._imageSrv.downloadBootstrapSysTemplate(store);
+
         return (ImageStore) _dataStoreMgr.getDataStore(store.getId(), DataStoreRole.Image);
     }
 

server/src/com/cloud/storage/download/DownloadMonitor.java

@@ -33,7 +33,10 @@ import com.cloud.utils.component.Manager;
  */
 public interface DownloadMonitor extends Manager{
 
-	public void downloadTemplateToStorage(VMTemplateVO template, DataStore store, AsyncCompletionCallback<CreateCmdResult> callback);
+    // when ssvm is not available yet
+    public void downloadBootstrapSysTemplateToStorage(VMTemplateVO template, DataStore store, AsyncCompletionCallback<CreateCmdResult> callback);
+
+    public void downloadTemplateToStorage(VMTemplateVO template, DataStore store, AsyncCompletionCallback<CreateCmdResult> callback);
 
 	public void cancelAllDownloads(Long templateId);
 

server/src/com/cloud/storage/download/DownloadMonitorImpl.java

@@ -30,6 +30,9 @@ import javax.inject.Inject;
 import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
+import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
+import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
 import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
 import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
@@ -48,6 +51,7 @@ import com.cloud.agent.api.storage.DownloadCommand;
 import com.cloud.agent.api.storage.DownloadCommand.Proxy;
 import com.cloud.agent.api.storage.DownloadCommand.ResourceType;
 import com.cloud.agent.api.storage.DownloadProgressCommand.RequestType;
+import com.cloud.agent.api.storage.DownloadSystemTemplateCommand;
 
 import com.cloud.agent.api.storage.DownloadProgressCommand;
 import com.cloud.agent.manager.Commands;
@@ -177,6 +181,10 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor
     protected UserVmDao _userVmDao;
     @Inject
     protected AccountManager _accountMgr;
+    @Inject
+    EndPointSelector _epSelector;
+    @Inject
+    TemplateDataFactory tmplFactory;
 
     private Boolean _sslCopy = new Boolean(false);
     private String _copyAuthPasswd;
@@ -423,6 +431,69 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor
         }
     }
 
+
+    @Override
+    public void downloadBootstrapSysTemplateToStorage(VMTemplateVO template, DataStore store, AsyncCompletionCallback<CreateCmdResult> callback) {
+        boolean downloadJobExists = false;
+        TemplateDataStoreVO vmTemplateStore = null;
+
+        vmTemplateStore = _vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId());
+        if (vmTemplateStore == null) {
+            // This method can be invoked other places, for example,
+            // handleTemplateSync, in that case, vmTemplateStore may be null
+            vmTemplateStore = new TemplateDataStoreVO(store.getId(), template.getId(), new Date(), 0,
+                    VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED, null, null, "jobid0000", null, template.getUrl());
+            _vmTemplateStoreDao.persist(vmTemplateStore);
+        } else if ((vmTemplateStore.getJobId() != null) && (vmTemplateStore.getJobId().length() > 2)) {
+            downloadJobExists = true;
+        }
+
+        Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes();
+        String secUrl = store.getUri();
+        if (vmTemplateStore != null) {
+            start();
+            DownloadSystemTemplateCommand dcmd = new DownloadSystemTemplateCommand(store.getTO(), secUrl, template, maxTemplateSizeInBytes);
+            dcmd.setProxy(getHttpProxy());
+            // TODO: handle S3 download progress
+            // if (downloadJobExists) {
+            // dcmd = new DownloadProgressCommand(dcmd,
+            // vmTemplateStore.getJobId(), RequestType.GET_OR_RESTART);
+            // }
+            if (vmTemplateStore.isCopy()) {
+                dcmd.setCreds(TemplateConstants.DEFAULT_HTTP_AUTH_USER, _copyAuthPasswd);
+            }
+            EndPoint endPoint = _epSelector.select(this.tmplFactory.getTemplate(template.getId(), store));
+            if (endPoint == null) {
+                s_logger.warn("There is no endpoint to send download template command");
+                return;
+            }
+            // TODO: wait for Edison's code to pass a listener to
+            // LocalHostEndPoint
+            /*
+            DownloadListener dl = new DownloadListener(ssAhost, store, template, _timer, _vmTemplateStoreDao, vmTemplateStore.getId(), this, dcmd,
+                    _templateDao, _resourceLimitMgr, _alertMgr, _accountMgr, callback);
+            if (downloadJobExists) {
+                // due to handling existing download job issues, we still keep
+                // downloadState in template_store_ref to avoid big change in
+                // DownloadListener to use
+                // new ObjectInDataStore.State transition. TODO: fix this later
+                // to be able to remove downloadState from template_store_ref.
+                dl.setCurrState(vmTemplateStore.getDownloadState());
+            }
+            DownloadListener old = null;
+            synchronized (_listenerTemplateMap) {
+                old = _listenerTemplateMap.put(vmTemplateStore, dl);
+            }
+            if (old != null) {
+                old.abandon();
+            }
+            */
+            // endPoint.sendMessageAsync(dcmd, callback);
+            endPoint.sendMessage(dcmd); // wait for Edison's callback code
+
+        }
+    }
+
     @Override
     public void downloadTemplateToStorage(VMTemplateVO template, DataStore store, AsyncCompletionCallback<CreateCmdResult> callback) {
         long templateId = template.getId();