mirror of
https://github.com/apache/cloudstack.git
synced 2025-10-26 08:42:29 +01:00
tool to aid db comaprision for upgrade testing
This commit is contained in:
shweta agarwal
sanjeev
parent
79a46fe0b4
commit
00b6a54dcf
45
tools/utils/database_comparision_during_upgrade/README
Normal file
45
tools/utils/database_comparision_during_upgrade/README
Normal file
@ -0,0 +1,45 @@
|
||||
What this tool is capable to do is
|
||||
|
||||
1. Compare all the tables schema between upgraded setup and fresh install setup and find if there is any schema difference between any tables
|
||||
2. Compare global configuration between upgraded and fresh install setup and find out if there is any difference between the two on following fields
|
||||
a. Value
|
||||
b. Scope
|
||||
c. Description
|
||||
d. Component
|
||||
e. Category
|
||||
|
||||
3. It will also find out if there is some global configuration present only in fresh setup and missing in upgraded environment and vice versa
|
||||
4. It will also find out global configuration value difference between before upgraded and after upgrade setup
|
||||
|
||||
|
||||
|
||||
The usage is as follows
|
||||
1. First run fresh_install_data_collection.sh file to generate data from fresh install setup .
|
||||
This will be used for comparing between fresh install and upgrade setup.
|
||||
This is a onetime activity and need to be repeated only when there is some DB changes for that release .
|
||||
Output of this script will come in a base_data folder
|
||||
|
||||
2. Just before upgrade you need to run before_upgrade_data_collection.sh file to collect required data needed to compare before upgrade and after upgrade setup data
|
||||
The output of this script will come in folder data_before_upgrade
|
||||
|
||||
3. After upgrade run cloud_schema_comparision.sh to compare cloud database all tables schema between fresh and upgraded setup.
|
||||
NOTE: this script requires step 1 output in current working directory
|
||||
|
||||
4. After upgrade run usage_schema_comparision.sh to compare cloud usage all tables schema between fresh and upgraded setup
|
||||
NOTE: this script requires step 1 output in current working directory
|
||||
|
||||
5. Run test_config_between_fresh_and_upgraded_setup.sh to comapre table global configuration values between fresh and upgraded setup
|
||||
NOTE: this script requires step 1 output in current working directory
|
||||
|
||||
|
||||
6. Run test_config_before_and_after_upgrade.sh to comapre table global configuration values between before upgraded and after upgraded setup
|
||||
NOTE: this script requires step 2 output in current working directory
|
||||
|
||||
|
||||
7. In order to run any *.sh file you need to provide 3 command line argument
|
||||
• Database host ip/localhost
|
||||
• Database user
|
||||
• Database user password
|
||||
|
||||
8. Result will be shown in the form of files .
|
||||
|
||||
@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
dbhost=$1
|
||||
dbuser=$2
|
||||
dbpwd=$3
|
||||
rm -rf data_before_upgrade
|
||||
mkdir data_before_upgrade
|
||||
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value from cloud.configuration" > ./data_before_upgrade/configuration_before_upgrade
|
||||
|
||||
|
||||
@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
dbhost=$1
|
||||
dbuser=$2
|
||||
dbpwd=$3
|
||||
path=./base_data
|
||||
path1=./data_after_upgrade
|
||||
rm -rf $path1
|
||||
mkdir $path1
|
||||
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost -e "show tables from cloud" > $path1/tables_upgrade
|
||||
|
||||
# to check if number of tables and table name differs
|
||||
|
||||
diff $path/tables $path1/tables_upgrade > tables_diff_file
|
||||
if [ -s tables_diff_file ]
|
||||
then
|
||||
echo "cloud table differs between upgraded and fresh install "
|
||||
cat tables_diff_file
|
||||
# do something as file has data
|
||||
else
|
||||
echo "cloud tables are identical in upgraded and fresh install"
|
||||
rm -rf tables_diff_file
|
||||
# do something as file is empty
|
||||
|
||||
fi
|
||||
|
||||
|
||||
|
||||
for tablename in `cat $path1/tables_upgrade`
|
||||
do
|
||||
if [ $tablename != 'Tables_in_cloud' ]
|
||||
then
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost -e "describe cloud.$tablename" > $path1/upgradedschema
|
||||
cat $path/$tablename > $tablename.diff
|
||||
cat $path1/upgradedschema >> $tablename.diff
|
||||
sort $tablename.diff > $tablename.sort
|
||||
uniq -u $tablename.sort > $tablename.uniq
|
||||
|
||||
if [ -s $tablename.uniq ]
|
||||
then
|
||||
echo $tablename "table schema is different."
|
||||
cat $path1/upgradedschema > $tablename
|
||||
rm -rf $tablename.diff $tablename.sort
|
||||
|
||||
# do something as file has data
|
||||
else
|
||||
|
||||
rm -rf $tablename.diff $tablename.sort $tablename.uniq
|
||||
fi
|
||||
|
||||
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
|
||||
|
||||
rm -rf $path1
|
||||
@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
dbhost=$1
|
||||
dbuser=$2
|
||||
dbpwd=$3
|
||||
rm -rf base_data
|
||||
mkdir base_data
|
||||
mkdir ./base_data/usage_data
|
||||
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost -e "show tables from cloud" > ./base_data/tables
|
||||
for tablename in `cat ./base_data/tables`
|
||||
do
|
||||
if [ $tablename != 'Tables_in_cloud' ]
|
||||
then
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost -e "describe cloud.$tablename" > ./base_data/$tablename
|
||||
fi
|
||||
done
|
||||
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value, default_value from cloud.configuration" > ./base_data/configuration_fresh
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, component from cloud.configuration" > ./base_data/component_configuration_fresh
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, category from cloud.configuration" > ./base_data/category_configuration_fresh
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, description from cloud.configuration" > ./base_data/description_configuration_fresh
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, scope from cloud.configuration" > ./base_data/scope_configuration_fresh
|
||||
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost -e "show tables from cloud_usage" > ./base_data/usage_data/usage_tables
|
||||
for tablename in `cat ./base_data/usage_data/usage_tables`
|
||||
do
|
||||
if [ $tablename != 'Tables_in_cloud_usage' ]
|
||||
then
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost -e "describe cloud_usage.$tablename" > ./base_data/usage_data/$tablename
|
||||
fi
|
||||
done
|
||||
|
||||
@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
dbhost=$1
|
||||
dbuser=$2
|
||||
dbpwd=$3
|
||||
a=0
|
||||
rm -rf *.uniq mismatch_config_between_before_and_after_upgrade difference_in_config_between_fresh_and_upgraded configurations_only_before_upgrade only_in_upgrade only_in_fresh config_difference_before_and_after_upgrade new_in_upgrade
|
||||
path=./base_data
|
||||
path1=./data_before_upgrade
|
||||
path2=./data_after_upgrade
|
||||
|
||||
rm -rf $path2
|
||||
mkdir $path2
|
||||
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value from cloud.configuration" > $path2/configuration_upgrade
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, component from cloud.configuration" > $path2/component_configuration_upgrade
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, category from cloud.configuration" > $path2/category_configuration_upgrade
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, description from cloud.configuration" > $path2/description_configuration_upgrade
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, scope from cloud.configuration" > $path2/scope_configuration_upgrade
|
||||
|
||||
|
||||
IFS=$'\n'
|
||||
|
||||
# to find any missing or mismatch configuration value before upgrade and after upgrade setup
|
||||
for row in `cat $path1/configuration_before_upgrade`
|
||||
do
|
||||
|
||||
grep $row $path2/configuration_upgrade > $a
|
||||
if [ ! -s $a ]
|
||||
then
|
||||
|
||||
echo $row >> ./mismatch_config_between_before_and_after_upgrade
|
||||
count=`wc -l <./mismatch_config_between_before_and_after_upgrade`
|
||||
echo $row > temp
|
||||
awk '{print $1}' temp > temp1
|
||||
for word in `cat ./temp1`
|
||||
do
|
||||
#echo $word
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value from cloud.configuration where name= '$word'" >> ./mismatch_config_between_before_and_after_upgrade
|
||||
count1=`wc -l <./mismatch_config_between_before_and_after_upgrade`
|
||||
if [ $count == $count1 ]
|
||||
then
|
||||
echo $row >> ./configurations_only_before_upgrade
|
||||
sed -i '$ d' ./mismatch_config_between_before_and_after_upgrade
|
||||
fi
|
||||
|
||||
|
||||
|
||||
done
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
|
||||
#to find configuration present only in the upgraded setup
|
||||
|
||||
|
||||
for row in `cat $path2/configuration_upgrade`
|
||||
do
|
||||
|
||||
grep $row $path1/configuration_before_upgrade > $a
|
||||
if [ ! -s $a ]
|
||||
then
|
||||
echo $row >> ./config_difference_before_and_after_upgrade
|
||||
count=`wc -l <./config_difference_before_and_after_upgrade`
|
||||
echo $row > temp
|
||||
awk '{print $1}' temp > temp1
|
||||
for word in `cat ./temp1`
|
||||
do
|
||||
grep '^'$word'[^\.]\w*' $path1/configuration_before_upgrade >> ./config_difference_before_and_after_upgrade
|
||||
count1=`wc -l <./config_difference_before_and_after_upgrade`
|
||||
if [ $count == $count1 ]
|
||||
then
|
||||
echo $row >> ./new_in_upgrade
|
||||
sed -i '$ d' ./config_difference_before_and_after_upgrade
|
||||
|
||||
fi
|
||||
|
||||
|
||||
|
||||
done
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
#to find all the difference between before and after upgrade
|
||||
cat ./mismatch_config_between_before_and_after_upgrade >> ./config_difference_before_and_after_upgrade
|
||||
sort ./config_difference_before_and_after_upgrade > ./config_difference_before_and_after_upgrade.sort
|
||||
uniq ./config_difference_before_and_after_upgrade.sort > ./config_difference_before_and_after_upgrade
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
rm -rf $path2 *.sort category description scope component temp temp1 $a
|
||||
rm -rf mismatch_config_between_before_and_after_upgrade config_difference_before_and_after_upgrade.sort t
|
||||
|
||||
@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env bash
|
||||
dbhost=$1
|
||||
dbuser=$2
|
||||
dbpwd=$3
|
||||
a=0
|
||||
rm -rf *.uniq only_in_upgraded only_in_fresh new_in_upgrade difference_in_config_between_fresh_and_upgraded
|
||||
path=./base_data
|
||||
path2=./data_after_upgrade
|
||||
|
||||
rm -rf $path2
|
||||
mkdir $path2
|
||||
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value from cloud.configuration" > $path2/configuration_upgrade
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, component from cloud.configuration" > $path2/component_configuration_upgrade
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, category from cloud.configuration" > $path2/category_configuration_upgrade
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, description from cloud.configuration" > $path2/description_configuration_upgrade
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, scope from cloud.configuration" > $path2/scope_configuration_upgrade
|
||||
|
||||
|
||||
IFS=$'\n'
|
||||
|
||||
|
||||
#to find difference between upgraded configuration and fresh install configuration
|
||||
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value, default_value from cloud.configuration" > $path2/configuration_upgrade
|
||||
|
||||
|
||||
|
||||
for row in `cat $path2/configuration_upgrade`
|
||||
do
|
||||
|
||||
grep $row $path/configuration_fresh > $a
|
||||
if [ ! -s $a ]
|
||||
then
|
||||
|
||||
echo $row > temp
|
||||
name=`awk '{print $1}' temp`
|
||||
value=`awk '{print $2}' temp`
|
||||
default=`awk '{print $3}' temp`
|
||||
grep '^'$name'[^\.]\w*' $path/configuration_fresh > t
|
||||
if [ ! -s t ]
|
||||
then
|
||||
echo $row >> ./only_in_upgraded
|
||||
else
|
||||
fname=`awk '{print $1}' t`
|
||||
fvalue=`awk '{print $2}' t`
|
||||
fdefault=`awk '{print $3}' t`
|
||||
#echo $fname $fvalue $fdefault
|
||||
if [ $default != $value ]
|
||||
then
|
||||
if [ $default == $fdefault ] && [ $value == $fvalue ]
|
||||
then
|
||||
echo
|
||||
else
|
||||
first="$name $value"
|
||||
second="$fname $fvalue"
|
||||
echo "in upgrade" >>difference_in_config_between_fresh_and_upgraded
|
||||
echo $first >> ./difference_in_config_between_fresh_and_upgraded
|
||||
echo "in fresh" >>difference_in_config_between_fresh_and_upgraded
|
||||
echo $second >> ./difference_in_config_between_fresh_and_upgraded
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
fi
|
||||
done
|
||||
|
||||
# to find configuration only available in fresh install but missing in upgraded setup
|
||||
|
||||
for row in `cat $path/configuration_fresh`
|
||||
do
|
||||
|
||||
grep $row $path2/configuration_upgrade > $a
|
||||
if [ ! -s $a ]
|
||||
then
|
||||
|
||||
echo "in fresh install" >>final_diff4
|
||||
echo $row >> ./final_diff4
|
||||
count=`wc -l <./final_diff4`
|
||||
echo $row > temp
|
||||
awk '{print $1}' temp > temp1
|
||||
for word in `cat ./temp1`
|
||||
do
|
||||
#echo $word
|
||||
echo "in upgrde\n" >> final_diff4
|
||||
grep '^'$word'[^\.]\w*' $path2/configuration_upgrade >> ./final_diff4
|
||||
count1=`wc -l <./final_diff4`
|
||||
count1=`expr $count1 - 1`
|
||||
|
||||
if [ $count == $count1 ]
|
||||
then
|
||||
echo $row >> ./only_in_fresh
|
||||
fi
|
||||
|
||||
done
|
||||
fi
|
||||
|
||||
done
|
||||
rm -rf final_diff4
|
||||
|
||||
|
||||
# to find difference between upgraded and fresh install on component field
|
||||
|
||||
cat $path2/component_configuration_upgrade > ./component
|
||||
cat $path/component_configuration_fresh >> ./component
|
||||
sort ./component > ./component.sort
|
||||
uniq -u ./component.sort > component.uniq
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# to find different between upgraded and fresh install on category field
|
||||
|
||||
cat $path2/category_configuration_upgrade > ./category
|
||||
cat $path/category_configuration_fresh >> ./category
|
||||
sort ./category > ./category.sort
|
||||
uniq -u ./category.sort > category.uniq
|
||||
|
||||
# to find different between upgraded and fresh install on scope field
|
||||
|
||||
cat $path2/scope_configuration_upgrade > ./scope
|
||||
cat $path/scope_configuration_fresh >> ./scope
|
||||
sort ./scope > ./scope.sort
|
||||
uniq -u ./scope.sort > scope.uniq
|
||||
|
||||
# to find different between upgraded and fresh install on description field
|
||||
|
||||
cat $path2/description_configuration_upgrade > ./description
|
||||
cat $path/description_configuration_fresh >> ./description
|
||||
sort ./description > ./description.sort
|
||||
uniq -u ./description.sort > description.uniq
|
||||
|
||||
|
||||
|
||||
|
||||
rm -rf $path2 *.sort category description scope component temp temp1 $a
|
||||
rm -rf mismatch_config_between_before_and_after_upgrade config_difference_before_and_after_upgrade.sort t
|
||||
#rm -rf $path2
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
dbhost=$1
|
||||
dbuser=$2
|
||||
dbpwd=$3
|
||||
path=./base_data/usage_data
|
||||
path1=./data_after_upgrade
|
||||
rm -rf $path1
|
||||
mkdir $path1
|
||||
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost -e "show tables from cloud_usage" > $path1/usage_tables_upgrade
|
||||
|
||||
# to check if number of tables and table name differs
|
||||
|
||||
diff $path/usage_tables $path1/usage_tables_upgrade > usage_table_difference
|
||||
if [ -s usage_table_difference ]
|
||||
then
|
||||
echo "usage table differs between fresh and upgraded install "
|
||||
cat usage_table_difference
|
||||
# do something as file has data
|
||||
else
|
||||
echo "usage tables are identicals between fresh and upgraded install "
|
||||
rm -rf usage_tables_difference
|
||||
# do something as file is empty
|
||||
|
||||
fi
|
||||
|
||||
|
||||
|
||||
for tablename in `cat $path1/usage_tables_upgrade`
|
||||
do
|
||||
if [ $tablename != 'Tables_in_cloud_usage' ]
|
||||
then
|
||||
mysql -u $dbuser -p$dbpwd -h $dbhost -e "describe cloud_usage.$tablename" > $path1/upgradedschema
|
||||
cat $path/$tablename > $tablename.diff
|
||||
cat $path1/upgradedschema >> $tablename.diff
|
||||
sort $tablename.diff > $tablename.sort
|
||||
uniq -u $tablename.sort > $tablename.uniq
|
||||
|
||||
if [ -s $tablename.uniq ]
|
||||
then
|
||||
echo $tablename "table schema is different."
|
||||
cat $path1/upgradedschema > usage_$tablename
|
||||
rm -rf $tablename.diff $tablename.sort
|
||||
|
||||
# do something as file has data
|
||||
else
|
||||
|
||||
rm -rf $tablename.diff $tablename.sort $tablename.uniq
|
||||
fi
|
||||
|
||||
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
|
||||
|
||||
rm -rf $path1
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user