cloudstack/scripts/vm/hypervisor/xenserver/vmopsSnapshot

#!/usr/bin/python
#
# A plugin for executing script needed by vmops cloud 

import os, sys, time
import XenAPIPlugin
sys.path.append("/opt/xensource/sm/")
import SR, VDI, SRCommand, util, lvutil
from util import CommandException
import vhdutil
import shutil
import lvhdutil
import errno
import subprocess
import xs_errors
import cleanup
import stat
import random

VHD_UTIL = '/usr/sbin/vhd-util'
VHD_PREFIX = 'VHD-'

def echo(fn):
    def wrapped(*v, **k):
        name = fn.__name__
        util.SMlog("#### VMOPS enter  %s ####" % name )
        res = fn(*v, **k)
        util.SMlog("#### VMOPS exit  %s ####" % name )
        return res
    return wrapped


@echo
def create_secondary_storage_folder(session, args):
    local_mount_path = None

    util.SMlog("create_secondary_storage_folder, args: " + str(args))

    try:
        try:
            # Mount the remote resource folder locally
            remote_mount_path = args["remoteMountPath"]
            local_mount_path = os.path.join(SR.MOUNT_BASE, "mount" + str(int(random.random() * 1000000)))
            mount(remote_mount_path, local_mount_path)

            # Create the new folder
            new_folder = local_mount_path + "/" + args["newFolder"]
            if not os.path.isdir(new_folder):
                current_umask = os.umask(0)
                os.makedirs(new_folder)
                os.umask(current_umask)
        except OSError, (errno, strerror):
            errMsg = "create_secondary_storage_folder failed: errno: " + str(errno) + ", strerr: " + strerror
            util.SMlog(errMsg)
            raise xs_errors.XenError(errMsg)
        except:
            errMsg = "create_secondary_storage_folder failed."
            util.SMlog(errMsg)
            raise xs_errors.XenError(errMsg)
    finally:
        if local_mount_path != None:
            # Unmount the local folder
            umount(local_mount_path)
            # Remove the local folder
            os.system("rm -rf " + local_mount_path)
        
    return "1"

@echo
def delete_secondary_storage_folder(session, args):
    local_mount_path = None

    util.SMlog("delete_secondary_storage_folder, args: " + str(args))

    try:
        try:
            # Mount the remote resource folder locally
            remote_mount_path = args["remoteMountPath"]
            local_mount_path = os.path.join(SR.MOUNT_BASE, "mount" + str(int(random.random() * 1000000)))
            mount(remote_mount_path, local_mount_path)

            # Delete the specified folder
            folder = local_mount_path + "/" + args["folder"]
            if os.path.isdir(folder):
                os.system("rm -rf " + folder)
        except OSError, (errno, strerror):
            errMsg = "delete_secondary_storage_folder failed: errno: " + str(errno) + ", strerr: " + strerror
            util.SMlog(errMsg)
            raise xs_errors.XenError(errMsg)
        except:
            errMsg = "delete_secondary_storage_folder failed."
            util.SMlog(errMsg)
            raise xs_errors.XenError(errMsg)
    finally:
        if local_mount_path != None:
            # Unmount the local folder
            umount(local_mount_path)
            # Remove the local folder
            os.system("rm -rf " + local_mount_path)
        
    return "1"
     
@echo
def post_create_private_template(session, args):
    local_mount_path = None
    try:
        # get local template folder 
        sruuid = args["tmpltSrUUID"]
        local_mount_path = os.path.join(SR.MOUNT_BASE, sruuid)

        # Retrieve args
        filename = args["templateFilename"]
        name = args["templateName"]
        description = args["templateDescription"]
        checksum = args["checksum"]
        size = args["size"]
        virtual_size = args["virtualSize"]
        template_id = args["templateId"]

        # Determine the template size
        template_install_path = local_mount_path + "/" + filename
        file_size = os.path.getsize(template_install_path)
        util.SMlog("Got template file_size: " + str(file_size))
            
        # Create the template.properties file
        template_properties_install_path = local_mount_path + "/template.properties"
        f = open(template_properties_install_path, "w")
        f.write("filename=" + filename + "\n")
        f.write("name=" + filename + "\n")
        f.write("vhd=true\n")
        f.write("id=" + template_id + "\n")
        f.write("vhd.filename=" + filename + "\n")
        f.write("uniquename=" + name + "\n")
        f.write("vhd.virtualsize=" + virtual_size + "\n")
        f.write("vhd.size=" + str(file_size) + "\n")
        f.write("virtualsize=" + virtual_size + "\n")
        f.write("checksum=" + checksum + "\n")
        f.write("hvm=true\n")
        f.write("description=" + name + "\n")
        f.close()
        util.SMlog("Created template.properties file")
           
        # Set permissions
        permissions = stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH
        os.chmod(template_properties_install_path, permissions)
        util.SMlog("Set permissions on template and template.properties")

    except:
        errMsg = "post_create_private_template failed."
        util.SMlog(errMsg)
        raise xs_errors.XenError(errMsg)

    return "1" 
  
def isfile(path, isISCSI):
    errMsg = ''
    exists = True
    if isISCSI:
        exists = checkVolumeAvailablility(path)
    else:
        exists = os.path.isfile(path)
        
    if not exists:
        errMsg = "File " + path + " does not exist."
        util.SMlog(errMsg)
        raise xs_errors.XenError(errMsg)
    return errMsg

def copyfile(fromFile, toFile, isISCSI):
    util.SMlog("Starting to copy " + fromFile + " to " + toFile)
    errMsg = ''
    if isISCSI:
        try:
            cmd = ['dd', 'if=' + fromFile, 'of=' + toFile, 'bs=1M']
            txt = util.pread2(cmd)
        except:
            txt = ''
            errMsg = "Error while copying " + fromFile + " to " + toFile + " in ISCSI mode"
            util.SMlog(errMsg)
            raise xs_errors.XenError(errMsg)

    else:
        try:
            shutil.copy2(fromFile, toFile)
        except OSError, (errno, strerror):
            errMsg = "Error while copying " + fromFile + " to " + toFile + " with errno: " + str(errno) + " and strerr: " + strerror
            util.SMlog(errMsg)
            raise xs_errors.XenError(errMsg)

    util.SMlog("Successfully copied " + fromFile + " to " + toFile)
    return errMsg

def chdir(path):
    try:
        os.chdir(path)
    except OSError, (errno, strerror):
        errMsg = "Unable to chdir to " + path + " because of OSError with errno: " + str(errno) + " and strerr: " + strerror
        util.SMlog(errMsg)
        raise xs_errors.XenError(errMsg)
    util.SMlog("Chdired to " + path)
    return

def scanParent(path):
    # Do a scan for the parent for ISCSI volumes
    # Note that the parent need not be visible on the XenServer
    parentUUID = ''
    try:
        lvName = os.path.basename(path)
        dirname = os.path.dirname(path)
        vgName = os.path.basename(dirname) 
        vhdInfo = vhdutil.getVHDInfoLVM(lvName, lvhdutil.extractUuid, vgName)
        parentUUID = vhdInfo.parentUuid
    except:
        errMsg = "Could not get vhd parent of " + path
        util.SMlog(errMsg)
        raise xs_errors.XenError(errMsg)
    return parentUUID

def getParent(path, isISCSI):
    parentUUID = ''
    try :
        if isISCSI:
            parentUUID = vhdutil.getParent(path, lvhdutil.extractUuid)
        else:
            parentUUID = vhdutil.getParent(path, cleanup.FileVDI.extractUuid)
    except:
        errMsg = "Could not get vhd parent of " + path
        util.SMlog(errMsg)
        raise xs_errors.XenError(errMsg)
    return parentUUID

def getParentOfSnapshot(snapshotUuid, primarySRPath, isISCSI):
    snapshotVHD    = getVHD(snapshotUuid, isISCSI)
    snapshotPath   = os.path.join(primarySRPath, snapshotVHD)

    baseCopyUuid = ''
    if isISCSI:
        checkVolumeAvailablility(snapshotPath)
        baseCopyUuid = scanParent(snapshotPath)
    else:
        baseCopyUuid = getParent(snapshotPath, isISCSI)
    
    util.SMlog("Base copy of snapshotUuid: " + snapshotUuid + " is " + baseCopyUuid)
    return baseCopyUuid

def setParent(parent, child):
    try:
        cmd = [VHD_UTIL, "modify", "-p", parent, "-n", child]
        txt = util.pread2(cmd)
    except:
        errMsg = "Unexpected error while trying to set parent of " + child + " to " + parent 
        util.SMlog(errMsg)
        raise xs_errors.XenError(errMsg)
    util.SMlog("Successfully set parent of " + child + " to " + parent)
    return

def rename(originalVHD, newVHD):
    try:
        os.rename(originalVHD, newVHD)
    except OSError, (errno, strerror):
        errMsg = "OSError while renaming " + origiinalVHD + " to " + newVHD + "with errno: " + str(errno) + " and strerr: " + strerror
        util.SMlog(errMsg)
        raise xs_errors.XenError(errMsg)
    return

def makedirs(path):
    if not os.path.isdir(path):
        try:
            os.makedirs(path)
        except OSError, (errno, strerror):
            umount(path)
            if os.path.isdir(path):
                return
            errMsg = "OSError while creating " + path + " with errno: " + str(errno) + " and strerr: " + strerror
            util.SMlog(errMsg)
            raise xs_errors.XenError(errMsg)
    return

def mount(remoteDir, localDir):
    makedirs(localDir)
    
    try: 
        cmd = ['mount', remoteDir, localDir]
        txt = util.pread2(cmd)
    except:
        txt = ''
        errMsg = "Unexpected error while trying to mount " + remoteDir + " to " + localDir 
        util.SMlog(errMsg)
        raise xs_errors.XenError(errMsg)
    util.SMlog("Successfully mounted " + remoteDir + " to " + localDir)

    return

def umount(localDir):
    try: 
        cmd = ['umount', localDir]
        util.pread2(cmd)
    except CommandException:
        errMsg = "CommandException raised while trying to umount " + localDir 
        util.SMlog(errMsg)
        return

    util.SMlog("Successfully unmounted " + localDir)
    return

def mountSnapshotsDir(secondaryStorageMountPath, relativeDir, dcId, accountId, instanceId):
    # The aim is to mount secondaryStorageMountPath on 
    # SR.MOUNT_BASE/<dcId>/<relativeDir>
    # And create <accountId>/<instanceId> dir on it, if it doesn't exist already.
    # Assuming that secondaryStorageMountPath  exists remotely

    # Alex's suggestion and currently implemented:
    # Just mount secondaryStorageMountPath/<relativeDir> everytime
    # Never unmount.
    snapshotsDir = os.path.join(secondaryStorageMountPath, relativeDir)

    # Mkdir local mount point dir, if it doesn't exist.
    localMountPointPath = os.path.join(SR.MOUNT_BASE, dcId)
    localMountPointPath = os.path.join(localMountPointPath, relativeDir)

    makedirs(localMountPointPath)
    # if something is not mounted already on localMountPointPath, 
    # mount secondaryStorageMountPath on localMountPath
    if os.path.ismount(localMountPointPath):
        # There is only one secondary storage per zone. 
        # And we are mounting each sec storage under a zone-specific directory
        # So two secondary storage snapshot dirs will never get mounted on the same point on the same XenServer.
        util.SMlog("The remote snapshots directory has already been mounted on " + localMountPointPath)
    else:
        mount(snapshotsDir, localMountPointPath)

    # Create accountId/instanceId dir on localMountPointPath, if it doesn't exist
    backupsDir = os.path.join(localMountPointPath, accountId)
    backupsDir = os.path.join(backupsDir, instanceId)
    makedirs(backupsDir)
    return backupsDir

@echo
def unmountSnapshotsDir(session, args):
    dcId = args['dcId']
    localMountPointPath = os.path.join(SR.MOUNT_BASE, dcId)
    localMountPointPath = os.path.join(localMountPointPath, "snapshots")
    try:
        umount(localMountPointPath)
    except:
        util.SMlog("Ignoring the error while trying to unmount the snapshots dir.")

    return "1"

def getPrimarySRPath(primaryStorageSRUuid, isISCSI):
    if isISCSI:
        primarySRDir = lvhdutil.VG_PREFIX + primaryStorageSRUuid
        return os.path.join(lvhdutil.VG_LOCATION, primarySRDir)
    else:
        return os.path.join(SR.MOUNT_BASE, primaryStorageSRUuid)

def getBackupVHD(UUID):
    return UUID + '.' + SR.DEFAULT_TAP

def getVHD(UUID, isISCSI):
    if isISCSI:
        return VHD_PREFIX + UUID
    else:
        return UUID + '.' + SR.DEFAULT_TAP

def getIsTrueString(stringValue):
    booleanValue = False
    if (stringValue and stringValue == 'true'):
        booleanValue = True
    return booleanValue 

def makeUnavailable(uuid, primarySRPath, isISCSI):
    if not isISCSI:
        return
    VHD = getVHD(uuid, isISCSI)
    path = os.path.join(primarySRPath, VHD)
    manageAvailability(path, '-an')
    return

def manageAvailability(path, value):
    if path.__contains__("/var/run/sr-mount"):
        return
    util.SMlog("Setting availability of " + path + " to " + value)
    try:
        cmd = ['/usr/sbin/lvchange', value, path]
        util.pread2(cmd)
    except: #CommandException, (rc, cmdListStr, stderr):
        #errMsg = "CommandException thrown while executing: " + cmdListStr + " with return code: " + str(rc) + " and stderr: " + stderr
        errMsg = "Unexpected exception thrown by lvchange"
        util.SMlog(errMsg)
        if value == "-ay":
            # Raise an error only if we are trying to make it available.
            # Just warn if we are trying to make it unavailable after the 
            # snapshot operation is done.
            raise xs_errors.XenError(errMsg)
    return


def checkVolumeAvailablility(path):
    try:
        if not isVolumeAvailable(path):
            # The VHD file is not available on XenSever. The volume is probably
            # inactive or detached.
            # Do lvchange -ay to make it available on XenServer
            manageAvailability(path, '-ay')
    except:
        errMsg = "Could not determine status of ISCSI path: " + path
        util.SMlog(errMsg)
        raise xs_errors.XenError(errMsg)
    
    success = False
    i = 0
    while i < 6:
        i = i + 1
        # Check if the vhd is actually visible by checking for the link
        # set isISCSI to true
        success = isVolumeAvailable(path)
        if success:
            util.SMlog("Made vhd: " + path + " available and confirmed that it is visible")
            break

        # Sleep for 10 seconds before checking again.
        time.sleep(10)

    # If not visible within 1 min fail
    if not success:
        util.SMlog("Could not make vhd: " +  path + " available despite waiting for 1 minute. Does it exist?")

    return success

def isVolumeAvailable(path):
    # Check if iscsi volume is available on this XenServer.
    status = "0"
    try:
        p = subprocess.Popen(["/bin/bash", "-c", "if [ -L " + path + " ]; then echo 1; else echo 0;fi"], stdout=subprocess.PIPE)
        status = p.communicate()[0].strip("\n")
    except:
        errMsg = "Could not determine status of ISCSI path: " + path
        util.SMlog(errMsg)
        raise xs_errors.XenError(errMsg)

    return (status == "1")  

def getVhdParent(session, args):
    util.SMlog("getParent with " + str(args))
    primaryStorageSRUuid      = args['primaryStorageSRUuid']
    snapshotUuid              = args['snapshotUuid']
    isISCSI                   = getIsTrueString(args['isISCSI']) 

    primarySRPath = getPrimarySRPath(primaryStorageSRUuid, isISCSI)
    util.SMlog("primarySRPath: " + primarySRPath)

    baseCopyUuid = getParentOfSnapshot(snapshotUuid, primarySRPath, isISCSI)

    return  baseCopyUuid


def backupSnapshot(session, args):
    util.SMlog("Called backupSnapshot with " + str(args))
    primaryStorageSRUuid      = args['primaryStorageSRUuid']
    dcId                      = args['dcId']
    accountId                 = args['accountId']
    volumeId                  = args['volumeId']
    secondaryStorageMountPath = args['secondaryStorageMountPath']
    snapshotUuid              = args['snapshotUuid']
    prevSnapshotUuid          = args['prevSnapshotUuid']
    prevBackupUuid            = args['prevBackupUuid']
    isISCSI                   = getIsTrueString(args['isISCSI']) 

    primarySRPath = getPrimarySRPath(primaryStorageSRUuid, isISCSI)
    util.SMlog("primarySRPath: " + primarySRPath)

    baseCopyUuid = getParentOfSnapshot(snapshotUuid, primarySRPath, isISCSI)
    baseCopyVHD  = getVHD(baseCopyUuid, isISCSI)
    baseCopyPath = os.path.join(primarySRPath, baseCopyVHD)
    util.SMlog("Base copy path: " + baseCopyPath)

    prevBaseCopyUuid = ''
    if prevSnapshotUuid:
        prevBaseCopyUuid = getParentOfSnapshot(prevSnapshotUuid, primarySRPath, isISCSI)

    # Mount secondary storage mount path on XenServer along the path
    # /var/run/sr-mount/<dcId>/snapshots/ and create <accountId>/<volumeId> dir
    # on it.
    backupsDir = mountSnapshotsDir(secondaryStorageMountPath, "snapshots", dcId, accountId, volumeId)
    util.SMlog("Backups dir " + backupsDir)

    if baseCopyUuid == prevBaseCopyUuid:
        # There has been no change since the last snapshot so no need to backup 
        util.SMlog("There has been no change since the last snapshot with backup: " + prevBaseCopyUuid) 
        # Set the uuid of the current backup to that of last backup
        txt = "1#" + prevBaseCopyUuid
        return txt

    # Check existence of snapshot on primary storage 
    isfile(baseCopyPath, isISCSI)
    # copy baseCopyPath to backupsDir with new uuid
    backupUuid = util.gen_uuid()
    backupVHD = getBackupVHD(backupUuid)  
    backupFile = os.path.join(backupsDir, backupVHD)
    util.SMlog("Back up " + baseCopyUuid + " to Secondary Storage as " + backupUuid)
    copyfile(baseCopyPath, backupFile, isISCSI)
    vhdutil.setHidden(backupFile, False)
    
    # Now set the availability of the snapshotPath and the baseCopyPath to false
    makeUnavailable(snapshotUuid, primarySRPath, isISCSI)
    manageAvailability(baseCopyPath, '-an')
    if prevSnapshotUuid:
        makeUnavailable(prevSnapshotUuid, primarySRPath, isISCSI)
        makeUnavailable(prevBaseCopyUuid, primarySRPath, isISCSI)

    # Because the primary storage is always scanned, the parent of this base copy is always the first base copy.
    # We don't want that, we want a chain of VHDs each of which is a delta from the previous.
    # So set the parent of the current baseCopyVHD to prevBackupVHD 
    if prevBackupUuid:
        # If there was a previous snapshot
        prevBackupVHD = getBackupVHD(prevBackupUuid)
        prevBackupFile = os.path.join(backupsDir, prevBackupVHD)
        setParent(prevBackupFile, backupFile)

    txt = "1#" + backupUuid
    return txt

@echo
def deleteSnapshotBackup(session, args):
    util.SMlog("Calling deleteSnapshotBackup with " + str(args))
    dcId                      = args['dcId']
    accountId                 = args['accountId']
    volumeId                  = args['volumeId']
    secondaryStorageMountPath = args['secondaryStorageMountPath']
    backupUUID                = args['backupUUID']

    backupsDir = mountSnapshotsDir(secondaryStorageMountPath, "snapshots", dcId, accountId, volumeId)
    # chdir to the backupsDir for convenience
    chdir(backupsDir)

    backupVHD = getBackupVHD(backupUUID)
    util.SMlog("checking existence of " + backupVHD)

    # The backupVHD is on secondary which is NFS and not ISCSI.
    if not os.path.isfile(backupVHD):
        util.SMlog("backupVHD " + backupVHD + "does not exist. Not trying to delete it")
        return "1"
    util.SMlog("backupVHD " + backupVHD + " exists.")
        
    # Just delete the backupVHD
    try:
        os.remove(backupVHD)
    except OSError, (errno, strerror):
        errMsg = "OSError while removing " + backupVHD + " with errno: " + str(errno) + " and strerr: " + strerror
        util.SMlog(errMsg)
        raise xs_errors.XenError(errMsg)

    return "1"
   
def rmtree(path):
    if os.path.isdir(path):
        try:
            shutil.rmtree(path)
        except OSError, (errno, strerror):
            errMsg = "Error while deleting " + path + " on secondary storage with errno: " + str(errno) + " and strerr: " + strerror + ". Please delete it manually"
            util.SMlog(errMsg)
        util.SMlog("Successfully deleted " + path)
    else:
        util.SMlog("Could not find directory with path " + path)
    return

@echo
def deleteSnapshotsDir(session, args):
    util.SMlog("Calling deleteSnapshotsDir with " + str(args))
    dcId                      = args['dcId']
    accountId                 = args['accountId']
    volumeId                  = args['volumeId']
    secondaryStorageMountPath = args['secondaryStorageMountPath']

    backupsDir = mountSnapshotsDir(secondaryStorageMountPath, "snapshots", dcId, accountId, volumeId)
    accountDir = os.path.dirname(backupsDir)
    util.SMlog("accountDir is " + accountDir)
    rmtree(accountDir)

    return "1"
   
if __name__ == "__main__":
    XenAPIPlugin.dispatch({"getVhdParent":getVhdParent,  "create_secondary_storage_folder":create_secondary_storage_folder, "delete_secondary_storage_folder":delete_secondary_storage_folder, "post_create_private_template":post_create_private_template, "backupSnapshot": backupSnapshot, "deleteSnapshotBackup": deleteSnapshotBackup, "unmountSnapshotsDir": unmountSnapshotsDir, "deleteSnapshotsDir": deleteSnapshotsDir})