apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

bug CS-10789: More changes for the imageformat, introdueced new column in db for the format, created scripts for doffernt hypervisors ...and the list goes on.

Browse Source
This commit is contained in:
Nitin Mehta 2012-04-20 21:40:05 +05:30
parent f0911817ed
commit 4434aa0d2d
10 changed files with 591 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/src/com/cloud/agent/api/storage/DownloadCommand.java
View File

@ -130,8 +130,8 @@ public class DownloadCommand extends AbstractDownloadCommand {
this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
}
public DownloadCommand(String secUrl, Volume volume, Long maxDownloadSizeInBytes, String checkSum, String url) {
super(volume.getName(), url, ImageFormat.VHD, volume.getAccountId());
public DownloadCommand(String secUrl, Volume volume, Long maxDownloadSizeInBytes, String checkSum, String url, ImageFormat format) {
super(volume.getName(), url, format, volume.getAccountId());
//this.hvm = volume.isRequiresHvm();
this.checksum = checkSum;
this.id = volume.getId();

13
core/src/com/cloud/storage/VolumeHostVO.java
View File

@ -73,6 +73,9 @@ public class VolumeHostVO {
@Column (name="url")
private String downloadUrl;
@Column(name="format")
private Storage.ImageFormat format;
@Column(name="destroyed")
boolean destroyed = false;
@ -259,7 +262,15 @@ public class VolumeHostVO {
return downloadUrl;
}
public long getVolumeSize() {
public Storage.ImageFormat getFormat() {
return format;
}
public void setFormat(Storage.ImageFormat format) {
this.format = format;
}
public long getVolumeSize() {
return -1;
}

269
scripts/installer/createvolume.sh Executable file
View File

@ -0,0 +1,269 @@
#!/usr/bin/env bash
# Copyright (C) 2011 Citrix Systems, Inc. All rights reserved
#
# This software is licensed under the GNU General Public License v3 or later.
#
# It is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# $Id: createvol.sh 9132 2010-06-04 20:17:43Z manuel $ $HeadURL: svn://svn.lab.vmops.com/repos/vmdev/java/scripts/installer/createvolume.sh $
# createvolume.sh -- install a volume
usage() {
printf "Usage: %s: -t <volume-fs> -n <volumename> -f <root disk file> -s <size in Gigabytes> -c <md5 cksum> -d <descr> -h [-u]\n" $(basename $0) >&2
}
#set -x
rollback_if_needed() {
if [ $2 -gt 0 ]
then
printf "$3\n"
#back out all changes
zfs destroy -r $1
exit 2
fi
}
verify_cksum() {
echo "$1 $2" | md5sum -c --status
#printf "$1\t$2" | md5sum -c --status
if [ $? -gt 0 ]
then
printf "Checksum failed, not proceeding with install\n"
exit 3
fi
}
untar() {
local ft=$(file $1| awk -F" " '{print $2}')
local basedir=$(dirname $1)
local tarfile=$(basename $1)
case $ft in
USTAR) local rootimg=$(tar tf $1 | grep $3)
(cp $1 $2; cd $2; tar xf $tarfile)
rm -f $1
printf "$2/$rootimg"
;;
*) printf "$1"
return 0
;;
esac
}
uncompress() {
local ft=$(file $1| awk -F" " '{print $2}')
local imgfile=${1%.*} #strip out trailing file suffix
local tmpfile=${imgfile}.tmp
case $ft in
gzip) gunzip -c $1 > $tmpfile
;;
bzip2) bunzip2 -c $1 > $tmpfile
;;
ZIP) unzip -p $1 | cat > $tmpfile
;;
*) printf "$1"
return 0
;;
esac
if [ $? -gt 0 ]
then
printf "Failed to uncompress file, exiting "
exit 1
fi
mv $tmpfile $imgfile
printf "$imgfile"
return 0
}
create_from_file() {
local volfs=$1
local volimg=$2
local tgtfile=$3
local volsize=$4
local cleanup=$5
#copy 64k of zeros for LUN metatdata
dd if=/dev/zero of=/$tgtfile bs=64k count=1
#copy the file to the disk
dd if=$volimg of=/$tgtfile bs=64k seek=1
rollback_if_needed $volfs $? "Failed to copy root disk"
if [ "$cleanup" == "true" ]
then
rm -f $volimg
fi
}
tflag=
nflag=
fflag=
sflag=
hflag=
hvm=false
cleanup=false
dflag=
cflag=
while getopts 'uht:n:f:s:c:d:' OPTION
do
case $OPTION in
t) tflag=1
volfs="$OPTARG"
;;
n) nflag=1
volname="$OPTARG"
;;
f) fflag=1
volimg="$OPTARG"
;;
s) sflag=1
volsize="$OPTARG"
;;
c) cflag=1
cksum="$OPTARG"
;;
d) dflag=1
descr="$OPTARG"
;;
h) hflag=1
hvm="true"
;;
u) cleanup="true"
;;
?) usage
exit 2
;;
esac
done
if [ "$tflag$nflag$fflag$sflag" != "1111" ]
then
usage
exit 2
fi
if [ -n "$cksum" ]
then
verify_cksum $cksum $volimg
fi
if [ ${volfs:0:1} == / ]
then
volfs=${volfs:1}
fi
if [ ! -d /$volfs ]
then
zfs create -p $volfs
if [ $? -gt 0 ]
then
printf "Failed to create user fs $volfs\n" >&2
exit 1
fi
fi
if [[ $(zfs get -H -o value -p type $volfs) != filesystem ]]
then
printf "volume fs doesn't exist\n" >&2
exit 2
fi
volimg2=$(uncompress $volimg)
volimg2=$(untar $volimg2 /$volfs vmi-root)
if [ ! -f $volimg2 ]
then
rollback_if_needed $volfs 2 "root disk file $volimg doesn't exist\n"
exit 3
fi
# need the 'G' suffix on volume size
if [ ${volsize:(-1)} != G ]
then
volsize=${volsize}G
fi
#determine source file size -- it needs to be less than or equal to volsize
imgsize=$(ls -lh $volimg2| awk -F" " '{print $5}')
if [ ${imgsize:(-1)} == G ]
then
imgsize=${imgsize%G} #strip out the G
imgsize=${imgsize%.*} #...and any decimal part
let imgsize=imgsize+1 # add 1 to compensate for decimal part
volsizetmp=${volsize%G}
if [ $volsizetmp -lt $imgsize ]
then
volsize=${imgsize}G
fi
fi
tgtfile=${volfs}/vmi-root-${volname}
create_from_file $volfs $volimg2 $tgtfile $volsize $cleanup
volswap=$(ls -lh /$volfs | grep swap)
if [ $? -eq 0 ]
then
swapsize=$(echo $volswap | awk '{print $5}')
volswap=$(echo $volswap | awk '{print $NF}')
volswap=/${volfs}/${volswap}
tgtfile=${volfs}/vmi-swap-${volname}
create_from_file $volfs $volswap $tgtfile $swapsize $cleanup
fi
if [ "$hvm" != "true" ]
then
vmlinuz=$(ls /$volfs/vmlinuz*)
if [ "$vmlinuz" == "" ]
then
touch /$volfs/pygrub
fi
fi
rollback_if_needed $volfs $? "Failed to create pygrub file"
touch /$volfs/volume.properties
rollback_if_needed $volfs $? "Failed to create volume.properties file"
echo -n "" > /$volfs/volume.properties
today=$(date '+%m_%d_%Y')
echo "snapshot.name=$today" > /$volfs/volume.properties
echo "description=$descr" >> /$volfs/volume.properties
echo "name=$volname" >> /$volfs/volume.properties
echo "checksum=$cksum" >> /$volfs/volume.properties
echo "hvm=$hvm" >> /$volfs/volume.properties
echo "volume.size=$volsize" >> /$volfs/volume.properties
zfs snapshot -r $volfs@vmops_ss
rollback_if_needed $volfs $? "Failed to snapshot filesystem"
#if [ "$cleanup" == "true" ]
#then
#rm -f $volimg
#fi
exit 0

18
scripts/storage/qcow2/createtmplt.sh
View File

@ -17,10 +17,10 @@
# $Id: createtmplt.sh 11601 2010-08-11 17:26:15Z kris $ $HeadURL: svn://svn.lab.vmops.com/repos/branches/2.1.refactor/java/scripts/storage/qcow2/createtmplt.sh $
# createtmplt.sh -- install a template
# createtmplt.sh -- install a volume
usage() {
printf "Usage: %s: -t <template-fs> -n <templatename> -f <root disk file> -s <size in Gigabytes> -c <md5 cksum> -d <descr> -h [-u]\n" $(basename $0) >&2
printf "Usage: %s: -t <volume-fs> -n <volumename> -f <root disk file> -s <size in Gigabytes> -c <md5 cksum> -d <descr> -h [-u]\n" $(basename $0) >&2
}
@ -118,7 +118,7 @@ create_from_snapshot() {
$qemu_img convert -f qcow2 -O qcow2 -s "$snapshotName" "$tmpltImg" /$tmpltfs/$tmpltname >& /dev/null
if [ $? -gt 0 ]
then
printf "Failed to create template /$tmplfs/$tmpltname from snapshot $snapshotName on disk $tmpltImg "
printf "Failed to create volume /$tmplfs/$tmpltname from snapshot $snapshotName on disk $tmpltImg "
exit 2
fi
@ -195,14 +195,14 @@ else
create_from_file $tmpltfs "$tmpltimg" $tmpltname
fi
touch /$tmpltfs/template.properties
chmod a+r /$tmpltfs/template.properties
echo -n "" > /$tmpltfs/template.properties
touch /$tmpltfs/volume.properties
chmod a+r /$tmpltfs/volume.properties
echo -n "" > /$tmpltfs/volume.properties
today=$(date '+%m_%d_%Y')
echo "filename=$tmpltname" > /$tmpltfs/template.properties
echo "snapshot.name=$today" >> /$tmpltfs/template.properties
echo "description=$descr" >> /$tmpltfs/template.properties
echo "filename=$tmpltname" > /$tmpltfs/volume.properties
echo "snapshot.name=$today" >> /$tmpltfs/volume.properties
echo "description=$descr" >> /$tmpltfs/volume.properties
if [ "$cleanup" == "true" ]
then

215
scripts/storage/qcow2/createvolume.sh Executable file
View File

@ -0,0 +1,215 @@
#!/usr/bin/env bash
# Copyright (C) 2011 Citrix Systems, Inc. All rights reserved
#
# This software is licensed under the GNU General Public License v3 or later.
#
# It is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# $Id: createvol.sh 11601 2010-08-11 17:26:15Z kris $ $HeadURL: svn://svn.lab.vmops.com/repos/branches/2.1.refactor/java/scripts/storage/qcow2/createvol.sh $
# createvol.sh -- install a volume
usage() {
printf "Usage: %s: -t <volume-fs> -n <volumename> -f <root disk file> -s <size in Gigabytes> -c <md5 cksum> -d <descr> -h [-u]\n" $(basename $0) >&2
}
#set -x
qemu_img="cloud-qemu-img"
which $qemu_img
if [ $? -gt 0 ]
then
which qemu-img
if [ $? -eq 0 ]
then
qemu_img="qemu-img"
fi
fi
verify_cksum() {
echo "$1 $2" | md5sum -c --status
#printf "$1\t$2" | md5sum -c --status
if [ $? -gt 0 ]
then
printf "Checksum failed, not proceeding with install\n"
exit 3
fi
}
untar() {
local ft=$(file $1| awk -F" " '{print $2}')
local basedir=$(dirname $1)
case $ft in
USTAR) local rootimg=$(tar tf $1 | grep $3)
(cd $2; tar xf $1)
rm -f $1
printf "$2/$rootimg"
;;
*) printf "$1"
return 0
;;
esac
}
uncompress() {
local ft=$(file $1| awk -F" " '{print $2}')
local imgfile=${1%.*} #strip out trailing file suffix
local tmpfile=${imgfile}.tmp
case $ft in
gzip) gunzip -c $1 > $tmpfile
;;
bzip2) bunzip2 -c $1 > $tmpfile
;;
ZIP) unzip -p $1 | cat > $tmpfile
;;
*) printf "$1"
return 0
;;
esac
if [ $? -gt 0 ]
then
printf "Failed to uncompress file, exiting "
exit 1
fi
mv $tmpfile $imgfile
printf "$imgfile"
return 0
}
create_from_file() {
local volfs=$1
local volimg="$2"
local volname=$3
if [ -b $volimg ]; then
$qemu-img convert -f raw -O qcow2 "$volimg" /$volfs/$volname
else
$qemu_img convert -f qcow2 -O qcow2 "$volimg" /$volfs/$volname >& /dev/null
fi
if [ "$cleanup" == "true" ]
then
rm -f "$volimg"
fi
chmod a+r /$volfs/$volname
}
create_from_snapshot() {
local volImg="$1"
local snapshotName="$2"
local volfs=$3
local volname=$4
$qemu_img convert -f qcow2 -O qcow2 -s "$snapshotName" "$volImg" /$volfs/$volname >& /dev/null
if [ $? -gt 0 ]
then
printf "Failed to create volume /$tmplfs/$volname from snapshot $snapshotName on disk $volImg "
exit 2
fi
chmod a+r /$volfs/$volname
}
tflag=
nflag=
fflag=
sflag=
hflag=
hvm=false
cleanup=false
dflag=
cflag=
snapshotName=
while getopts 'uht:n:f:sc:d:' OPTION
do
case $OPTION in
t) tflag=1
volfs="$OPTARG"
;;
n) nflag=1
volname="$OPTARG"
;;
f) fflag=1
volimg="$OPTARG"
;;
s) sflag=1
sflag=1
;;
c) cflag=1
snapshotName="$OPTARG"
;;
d) dflag=1
descr="$OPTARG"
;;
u) cleanup="true"
;;
?) usage
exit 2
;;
esac
done
if [ ! -d /$volfs ]
then
mkdir -p /$volfs
if [ $? -gt 0 ]
then
printf "Failed to create user fs $volfs\n" >&2
exit 1
fi
fi
if [ ! -f $volimg -a ! -b $volimg ]
then
printf "root disk file $volimg doesn't exist\n"
exit 3
fi
volimg=$(uncompress "$volimg")
if [ $? -ne 0 ]
then
printf "failed to uncompress $volimg\n"
fi
if [ "$sflag" == "1" ]
then
create_from_snapshot "$volimg" "$snapshotName" $volfs $volname
else
create_from_file $volfs "$volimg" $volname
fi
touch /$volfs/volume.properties
chmod a+r /$volfs/volume.properties
echo -n "" > /$volfs/volume.properties
today=$(date '+%m_%d_%Y')
echo "filename=$volname" > /$volfs/volume.properties
echo "snapshot.name=$today" >> /$volfs/volume.properties
echo "description=$descr" >> /$volfs/volume.properties
if [ "$cleanup" == "true" ]
then
rm -f "$volimg"
fi
exit 0

66
scripts/storage/qcow2/listvolume.sh Executable file
View File

@ -0,0 +1,66 @@
#!/usr/bin/env bash
# Copyright (C) 2011 Citrix Systems, Inc. All rights reserved
#
# This software is licensed under the GNU General Public License v3 or later.
#
# It is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# $Id: listvmtmplt.sh 9132 2010-06-04 20:17:43Z manuel $ $HeadURL: svn://svn.lab.vmops.com/repos/vmdev/java/scripts/storage/qcow2/listvmtmplt.sh $
# listtmplt.sh -- list volumes under a directory
usage() {
printf "Usage: %s: -r <root dir> \n" $(basename $0) >&2
}
#set -x
rflag=
rootdir=
while getopts 'r:' OPTION
do
case $OPTION in
r) rflag=1
rootdir="$OPTARG"
;;
?) usage
exit 2
;;
esac
done
if [ "$rflag" != "1" ]
then
usage
exit 2
fi
for i in $(find /$rootdir -name volume.properties );
do
d=$(dirname $i)
filename=$(grep "filename" $i | awk -F"=" '{print $NF}')
size=$(grep "virtualsize" $i | awk -F"=" '{print $NF}')
if [ -n "$filename" ] && [ -n "$size" ]
then
d=$d/$filename/$size
fi
echo ${d#/} #remove leading slash
done
exit 0

11
server/src/com/cloud/storage/StorageManagerImpl.java
View File

@ -161,6 +161,7 @@ import com.cloud.user.UserContext;
import com.cloud.user.dao.AccountDao;
import com.cloud.user.dao.UserDao;
import com.cloud.uservm.UserVm;
import com.cloud.utils.EnumUtils;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.Pair;
import com.cloud.utils.Ternary;
@ -1701,10 +1702,11 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
Long zoneId = cmd.getZoneId();
String volumeName = cmd.getVolumeName();
String url = cmd.getUrl();
String format = cmd.getFormat();
validateVolume(caller, ownerId, zoneId, volumeName, url, cmd.getFormat());
validateVolume(caller, ownerId, zoneId, volumeName, url, format);
VolumeVO volume = persistVolume(caller, ownerId, zoneId, volumeName, url, cmd.getFormat());
_downloadMonitor.downloadVolumeToStorage(volume, zoneId, url, cmd.getChecksum());
_downloadMonitor.downloadVolumeToStorage(volume, zoneId, url, cmd.getChecksum(), ImageFormat.valueOf(format.toUpperCase()));
return volume;
}
@ -1733,6 +1735,11 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
throw new InvalidParameterValueException("File:// type urls are currently unsupported");
}
ImageFormat imgfmt = ImageFormat.valueOf(format.toUpperCase());
if (imgfmt == null) {
throw new IllegalArgumentException("Image format is incorrect " + format + ". Supported formats are " + EnumUtils.listValues(ImageFormat.values()));
}
String userSpecifiedName = volumeName;
if (userSpecifiedName == null) {
userSpecifiedName = getRandomVolumeName();

3
server/src/com/cloud/storage/download/DownloadMonitor.java
View File

@ -18,6 +18,7 @@ import com.cloud.exception.StorageUnavailableException;
import com.cloud.host.HostVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.template.TemplateInfo;
import com.cloud.utils.component.Manager;
@ -43,7 +44,7 @@ public interface DownloadMonitor extends Manager{
void addSystemVMTemplatesToHost(HostVO host, Map<String, TemplateInfo> templateInfos);
boolean downloadVolumeToStorage(VolumeVO volume, Long zoneId, String url, String checkSum);
boolean downloadVolumeToStorage(VolumeVO volume, Long zoneId, String url, String checkSum, ImageFormat format);
void handleVolumeSync(HostVO ssHost);

10
server/src/com/cloud/storage/download/DownloadMonitorImpl.java
View File

@ -417,16 +417,16 @@ public class DownloadMonitorImpl implements DownloadMonitor {
}
@Override
public boolean downloadVolumeToStorage(VolumeVO volume, Long zoneId, String url, String checkSum) {
public boolean downloadVolumeToStorage(VolumeVO volume, Long zoneId, String url, String checkSum, ImageFormat format) {
List<HostVO> ssHosts = _ssvmMgr.listAllTypesSecondaryStorageHostsInOneZone(zoneId);
Collections.shuffle(ssHosts);
HostVO ssHost = ssHosts.get(0);
downloadVolumeToStorage(volume, ssHost, url, checkSum);
downloadVolumeToStorage(volume, ssHost, url, checkSum, format);
return true;
}
private void downloadVolumeToStorage(VolumeVO volume, HostVO sserver, String url, String checkSum) {
private void downloadVolumeToStorage(VolumeVO volume, HostVO sserver, String url, String checkSum, ImageFormat format) {
boolean downloadJobExists = false;
VolumeHostVO volumeHost = null;
@ -444,7 +444,7 @@ public class DownloadMonitorImpl implements DownloadMonitor {
String secUrl = sserver.getStorageUrl();
if(volumeHost != null) {
start();
DownloadCommand dcmd = new DownloadCommand(secUrl, volume, maxVolumeSizeInBytes, checkSum, url);
DownloadCommand dcmd = new DownloadCommand(secUrl, volume, maxVolumeSizeInBytes, checkSum, url, format);
dcmd.setProxy(getHttpProxy());
if (downloadJobExists) {
dcmd = new DownloadProgressCommand(dcmd, volumeHost.getJobId(), RequestType.GET_OR_RESTART);
@ -735,7 +735,7 @@ public class DownloadMonitorImpl implements DownloadMonitor {
continue;
}
s_logger.debug("Volume " + volumeHost.getVolumeId() + " needs to be downloaded to " + ssHost.getName());
downloadVolumeToStorage(_volumeDao.findById(volumeHost.getVolumeId()), ssHost, volumeHost.getDownloadUrl(), volumeHost.getChecksum());
downloadVolumeToStorage(_volumeDao.findById(volumeHost.getVolumeId()), ssHost, volumeHost.getDownloadUrl(), volumeHost.getChecksum(), volumeHost.getFormat());
}
}

1
setup/db/create-schema.sql
View File

@ -1142,6 +1142,7 @@ CREATE TABLE `cloud`.`volume_host_ref` (
`local_path` varchar(255),
`install_path` varchar(255),
`url` varchar(255),
`format` varchar(32) NOT NULL COMMENT 'format for the volume',
`destroyed` tinyint(1) COMMENT 'indicates whether the volume_host entry was destroyed by the user or not',
PRIMARY KEY (`id`),
CONSTRAINT `fk_volume_host_ref__host_id` FOREIGN KEY `fk_volume_host_ref__host_id` (`host_id`) REFERENCES `host` (`id`) ON DELETE CASCADE,