cloudsysvmadm script (used to restart system vms during the upgrade) - added jobId, timestamp to the logged statements about router/systemvm stop/start failures

This commit is contained in:
Alena Prokharchyk 2014-04-14 16:16:10 -07:00
parent f5ba020633
commit 498cf6183d

View File

@ -86,20 +86,20 @@ length_console=(${#console[@]})
echo -e "\nStopping and starting $length_secondary secondary storage vm(s)$inzone..."
echo -e "Stopping and starting $length_secondary secondary storage vm(s)$inzone..." >>$LOGFILE
echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Stopping and starting $length_secondary secondary storage vm(s)$inzone..." >>$LOGFILE
for d in "${secondary[@]}"; do
echo "INFO: Stopping secondary storage vm with id $d" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] INFO: Stopping secondary storage vm with id $d" >>$LOGFILE
jobresult=$(send_request stopSystemVm $d)
if [ "$jobresult" != "1" ]; then
echo -e "ERROR: Failed to stop secondary storage vm with id $d \n"
echo "ERROR: Failed to stop secondary storage vm with id $d" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] ERROR: Failed to stop secondary storage vm with id $d" >>$LOGFILE
else
echo "INFO: Starting secondary storage vm with id $d" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] INFO: Starting secondary storage vm with id $d" >>$LOGFILE
jobresult=$(send_request startSystemVm $d SSVM)
if [ "$jobresult" != "1" ]; then
echo "ERROR: Failed to start secondary storage vm with id $d" >>$LOGFILE
echo "ERROR: Failed to start secondary storage vm with id $d" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] ERROR: Failed to start secondary storage vm with id $d" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] ERROR: Failed to start secondary storage vm with id $d" >>$LOGFILE
fi
fi
done
@ -108,24 +108,24 @@ if [ "$length_secondary" == "0" ];then
echo -e "No running secondary storage vms found \n"
else
echo -e "Done stopping and starting secondary storage vm(s)$inzone"
echo -e "Done stopping and starting secondary storage vm(s)$inzone." >>$LOGFILE
echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Done stopping and starting secondary storage vm(s)$inzone." >>$LOGFILE
fi
echo -e "\nStopping and starting $length_console console proxy vm(s)$inzone..."
echo -e "Stopping and starting $length_console console proxy vm(s)$inzone..." >>$LOGFILE
echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Stopping and starting $length_console console proxy vm(s)$inzone..." >>$LOGFILE
for d in "${console[@]}"; do
echo "INFO: Stopping console proxy with id $d" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] INFO: Stopping console proxy with id $d" >>$LOGFILE
jobresult=$(send_request stopSystemVm $d)
if [ "$jobresult" != "1" ]; then
echo -e "ERROR: Failed to stop console proxy vm with id $d \n"
echo "ERROR: Failed to stop console proxy vm with id $d" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] ERROR: Failed to stop console proxy vm with id $d" >>$LOGFILE
else
echo "INFO: Starting console proxy vm with id $d" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] INFO: Starting console proxy vm with id $d" >>$LOGFILE
jobresult=$(send_request startSystemVm $d consoleProxy)
if [ "$jobresult" != "1" ]; then
echo -e "ERROR: Failed to start console proxy vm with id $d \n"
echo "ERROR: Failed to start console proxy vm with id $d" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] ERROR: Failed to start console proxy vm with id $d" >>$LOGFILE
fi
fi
done
@ -134,7 +134,7 @@ if [ "$length_console" == "0" ];then
echo -e "No running console proxy vms found \n"
else
echo "Done stopping and starting console proxy vm(s) $inzone."
echo "Done stopping and starting console proxy vm(s) $inzone." >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] Done stopping and starting console proxy vm(s) $inzone." >>$LOGFILE
fi
}
@ -143,7 +143,7 @@ stop_start_router() {
length_router=(${#router[@]})
echo -e "\nStopping and starting $length_router running routing vm(s)$inzone... "
echo -e "Stopping and starting $length_router running routing vm(s)$inzone... " >>$LOGFILE
echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Stopping and starting $length_router running routing vm(s)$inzone... " >>$LOGFILE
#Spawn reboot router in parallel - run commands in <n> chunks - number of threads is configurable
@ -188,14 +188,14 @@ stop_start_router() {
done
if [ "$length_router" == "0" ];then
echo -e "No running router vms found \n" >>$LOGFILE
echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] No running router vms found \n" >>$LOGFILE
else
while [ $unfinishedPids -gt 0 ]; do
sleep 10
done
echo -e "Done restarting router(s)$inzone. \n"
echo -e "Done restarting router(s)$inzone. \n" >>$LOGFILE
echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Done restarting router(s)$inzone. \n" >>$LOGFILE
fi
}
@ -212,15 +212,19 @@ send_request(){
return
fi
jobresult=$(query_async_job_result $jobid)
if [ "$jobresult" != "1" ]; then
echo -e "ERROR: Failed to $1 id=$2; jobId is $jobid \n"
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] ERROR: Failed to $1 id=$2; jobId is $jobid" >>$LOGFILE
fi
echo $jobresult
}
reboot_router(){
echo "INFO: Restarting router with id $1" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] INFO: Restarting router with id $1" >>$LOGFILE
jobid=`curl -sS "http://$ms:8096/?command=rebootRouter&id=$1&response=json" | sed 's/\"//g' | sed 's/ //g' | sed 's/{//g' | sed 's/}//g' | awk -F: {'print $3'}`
if [ "$jobid" == "" ]; then
echo "ERROR: Failed to restart domainRouter with id $1" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] ERROR: Failed to restart domainRouter with id $1; unable to submit the job" >>$LOGFILE
echo 2
return
fi
@ -229,10 +233,10 @@ reboot_router(){
if [ "$jobresult" != "1" ]; then
echo -e "ERROR: Failed to restart domainRouter with id $1 \n"
echo "ERROR: Failed to restart domainRouter with id $1" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] ERROR: Failed to restart domainRouter with id $1; jobId $jobid" >>$LOGFILE
exit 0
else
echo "INFO: Successfully restarted domainRouter with id $1" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] INFO: Successfully restarted domainRouter with id $1; jobId $jobid" >>$LOGFILE
exit 0
fi
@ -244,7 +248,7 @@ restart_networks(){
length_networks=(${#networks[@]})
echo -e "\nRestarting $length_networks networks$inzone... "
echo -e "Restarting $length_networks networks$inzone... " >>$LOGFILE
echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Restarting $length_networks networks$inzone... " >>$LOGFILE
#Spawn restart network in parallel - run commands in <n> chunks - number of threads is configurable
@ -290,14 +294,14 @@ restart_networks(){
if [ "$length_networks" == "0" ];then
echo -e "No networks found \n" >>$LOGFILE
echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] No networks found \n" >>$LOGFILE
else
while [ $unfinishedPids -gt 0 ]; do
sleep 10
done
echo -e "Done restarting networks$inzone. \n"
echo -e "Done restarting networks$inzone. \n" >>$LOGFILE
echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Done restarting networks$inzone. \n" >>$LOGFILE
fi
}
@ -305,7 +309,7 @@ restart_networks(){
restart_network(){
jobid=`curl -sS "http://$ms:8096/?command=restartNetwork&id=$1&response=json" | sed 's/\"//g' | sed 's/ //g' | sed 's/{//g' | sed 's/}//g' | awk -F: {'print $3'}`
if [ "$jobid" == "" ]; then
echo "ERROR: Failed to restart network with id $1" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] ERROR: Failed to restart network with id $1; unable to submit the job" >>$LOGFILE
echo 2
return
fi
@ -313,9 +317,9 @@ restart_network(){
jobresult=$(query_async_job_result $jobid)
if [ "$jobresult" != "1" ]; then
echo "ERROR: Failed to restart network with id $1" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] ERROR: Failed to restart network with id $1; jobId $jobid" >>$LOGFILE
else
echo "INFO: Successfully restarted network with id $1" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] INFO: Successfully restarted network with id $1; jobId $jobid" >>$LOGFILE
fi
}
@ -323,10 +327,10 @@ restart_network(){
restart_vpc(){
echo -e "INFO: Restarting vpc with id $1"
echo "INFO: Restarting vpc with id $1" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] INFO: Restarting vpc with id $1" >>$LOGFILE
jobid=`curl -sS "http://$ms:8096/?command=restartVPC&id=$1&response=json" | sed 's/\"//g' | sed 's/ //g' | sed 's/{//g' | sed 's/}//g' | awk -F: {'print $3'}`
if [ "$jobid" == "" ]; then
echo "ERROR: Failed to restart vpc with id $1" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] ERROR: Failed to restart vpc with id $1; unable to submit the job" >>$LOGFILE
echo 2
return
fi
@ -335,10 +339,10 @@ restart_vpc(){
if [ "$jobresult" != "1" ]; then
echo -e "ERROR: Failed to restart vpc with id $1 \n"
echo "ERROR: Failed to restart vpc with id $1" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] ERROR: Failed to restart vpc with id $1; jobId $jobid" >>$LOGFILE
else
echo -e "INFO: Successfully restarted vpc with id $1 \n"
echo "INFO: Successfully restarted vpc with id $1" >>$LOGFILE
echo "[$(date "+%Y.%m.%d-%H.%M.%S")] INFO: Successfully restarted vpc with id $1; jobId $jobid" >>$LOGFILE
fi
}
@ -348,7 +352,7 @@ restart_vpcs(){
length_vpcs=(${#vpcs[@]})
echo -e "\nRestarting $length_vpcs vpcs... "
echo -e "Restarting $length_vpcs vpcs... " >>$LOGFILE
echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Restarting $length_vpcs vpcs... " >>$LOGFILE
#Spawn restart vpcs in parallel - run commands in <n> chunks - number of threads is configurable
@ -394,14 +398,14 @@ restart_vpcs(){
if [ "$length_vpcs" == "0" ];then
echo -e "No vpcs found \n" >>$LOGFILE
echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] No vpcs found \n" >>$LOGFILE
else
while [ $unfinishedPids -gt 0 ]; do
sleep 10
done
echo -e "Done restarting vpcs$inzone. \n"
echo -e "Done restarting vpcs$inzone. \n" >>$LOGFILE
echo -e "[$(date "+%Y.%m.%d-%H.%M.%S")] Done restarting vpcs$inzone. \n" >>$LOGFILE
fi
}