diff --git a/tools/utils/database_comparision_during_upgrade/README b/tools/utils/database_comparision_during_upgrade/README new file mode 100644 index 00000000000..abe44d0a4f9 --- /dev/null +++ b/tools/utils/database_comparision_during_upgrade/README @@ -0,0 +1,45 @@ +What this tool is capable to do is + +1. Compare all the tables schema between upgraded setup and fresh install setup and find if there is any schema difference between any tables +2. Compare global configuration between upgraded and fresh install setup and find out if there is any difference between the two on following fields + a. Value + b. Scope + c. Description + d. Component + e. Category + +3. It will also find out if there is some global configuration present only in fresh setup and missing in upgraded environment and vice versa +4. It will also find out global configuration value difference between before upgraded and after upgrade setup + + + +The usage is as follows +1. First run fresh_install_data_collection.sh file to generate data from fresh install setup . + This will be used for comparing between fresh install and upgrade setup. + This is a onetime activity and need to be repeated only when there is some DB changes for that release . + Output of this script will come in a base_data folder + +2. Just before upgrade you need to run before_upgrade_data_collection.sh file to collect required data needed to compare before upgrade and after upgrade setup data + The output of this script will come in folder data_before_upgrade + +3. After upgrade run cloud_schema_comparision.sh to compare cloud database all tables schema between fresh and upgraded setup. + NOTE: this script requires step 1 output in current working directory + +4. After upgrade run usage_schema_comparision.sh to compare cloud usage all tables schema between fresh and upgraded setup + NOTE: this script requires step 1 output in current working directory + +5. Run test_config_between_fresh_and_upgraded_setup.sh to comapre table global configuration values between fresh and upgraded setup + NOTE: this script requires step 1 output in current working directory + + +6. Run test_config_before_and_after_upgrade.sh to comapre table global configuration values between before upgraded and after upgraded setup + NOTE: this script requires step 2 output in current working directory + + +7. In order to run any *.sh file you need to provide 3 command line argument + • Database host ip/localhost + • Database user + • Database user password + +8. Result will be shown in the form of files . + diff --git a/tools/utils/database_comparision_during_upgrade/before_upgrade_data_collection.sh b/tools/utils/database_comparision_during_upgrade/before_upgrade_data_collection.sh new file mode 100644 index 00000000000..a726848c1cc --- /dev/null +++ b/tools/utils/database_comparision_during_upgrade/before_upgrade_data_collection.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +dbhost=$1 +dbuser=$2 +dbpwd=$3 +rm -rf data_before_upgrade +mkdir data_before_upgrade + +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value from cloud.configuration" > ./data_before_upgrade/configuration_before_upgrade + + diff --git a/tools/utils/database_comparision_during_upgrade/cloud_schema_comparision.sh b/tools/utils/database_comparision_during_upgrade/cloud_schema_comparision.sh new file mode 100644 index 00000000000..56fb66ac445 --- /dev/null +++ b/tools/utils/database_comparision_during_upgrade/cloud_schema_comparision.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +dbhost=$1 +dbuser=$2 +dbpwd=$3 +path=./base_data +path1=./data_after_upgrade +rm -rf $path1 +mkdir $path1 + +mysql -u $dbuser -p$dbpwd -h $dbhost -e "show tables from cloud" > $path1/tables_upgrade + +# to check if number of tables and table name differs + +diff $path/tables $path1/tables_upgrade > tables_diff_file +if [ -s tables_diff_file ] +then + echo "cloud table differs between upgraded and fresh install " + cat tables_diff_file + # do something as file has data +else + echo "cloud tables are identical in upgraded and fresh install" + rm -rf tables_diff_file + # do something as file is empty + +fi + + + +for tablename in `cat $path1/tables_upgrade` +do + if [ $tablename != 'Tables_in_cloud' ] + then + mysql -u $dbuser -p$dbpwd -h $dbhost -e "describe cloud.$tablename" > $path1/upgradedschema + cat $path/$tablename > $tablename.diff + cat $path1/upgradedschema >> $tablename.diff + sort $tablename.diff > $tablename.sort + uniq -u $tablename.sort > $tablename.uniq + + if [ -s $tablename.uniq ] + then + echo $tablename "table schema is different." + cat $path1/upgradedschema > $tablename + rm -rf $tablename.diff $tablename.sort + + # do something as file has data + else + + rm -rf $tablename.diff $tablename.sort $tablename.uniq + fi + + + fi +done + + + + +rm -rf $path1 diff --git a/tools/utils/database_comparision_during_upgrade/fresh_install_data_collection.sh b/tools/utils/database_comparision_during_upgrade/fresh_install_data_collection.sh new file mode 100644 index 00000000000..45af8d329d8 --- /dev/null +++ b/tools/utils/database_comparision_during_upgrade/fresh_install_data_collection.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +dbhost=$1 +dbuser=$2 +dbpwd=$3 +rm -rf base_data +mkdir base_data +mkdir ./base_data/usage_data + +mysql -u $dbuser -p$dbpwd -h $dbhost -e "show tables from cloud" > ./base_data/tables +for tablename in `cat ./base_data/tables` +do + if [ $tablename != 'Tables_in_cloud' ] + then + mysql -u $dbuser -p$dbpwd -h $dbhost -e "describe cloud.$tablename" > ./base_data/$tablename + fi +done + +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value, default_value from cloud.configuration" > ./base_data/configuration_fresh +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, component from cloud.configuration" > ./base_data/component_configuration_fresh +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, category from cloud.configuration" > ./base_data/category_configuration_fresh +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, description from cloud.configuration" > ./base_data/description_configuration_fresh +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, scope from cloud.configuration" > ./base_data/scope_configuration_fresh + +mysql -u $dbuser -p$dbpwd -h $dbhost -e "show tables from cloud_usage" > ./base_data/usage_data/usage_tables +for tablename in `cat ./base_data/usage_data/usage_tables` +do + if [ $tablename != 'Tables_in_cloud_usage' ] + then + mysql -u $dbuser -p$dbpwd -h $dbhost -e "describe cloud_usage.$tablename" > ./base_data/usage_data/$tablename + fi +done + diff --git a/tools/utils/database_comparision_during_upgrade/test_config_before_and_after_upgrade.sh b/tools/utils/database_comparision_during_upgrade/test_config_before_and_after_upgrade.sh new file mode 100644 index 00000000000..f181821d2a6 --- /dev/null +++ b/tools/utils/database_comparision_during_upgrade/test_config_before_and_after_upgrade.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +dbhost=$1 +dbuser=$2 +dbpwd=$3 +a=0 +rm -rf *.uniq mismatch_config_between_before_and_after_upgrade difference_in_config_between_fresh_and_upgraded configurations_only_before_upgrade only_in_upgrade only_in_fresh config_difference_before_and_after_upgrade new_in_upgrade +path=./base_data +path1=./data_before_upgrade +path2=./data_after_upgrade + +rm -rf $path2 +mkdir $path2 + +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value from cloud.configuration" > $path2/configuration_upgrade +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, component from cloud.configuration" > $path2/component_configuration_upgrade +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, category from cloud.configuration" > $path2/category_configuration_upgrade +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, description from cloud.configuration" > $path2/description_configuration_upgrade +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, scope from cloud.configuration" > $path2/scope_configuration_upgrade + + +IFS=$'\n' + +# to find any missing or mismatch configuration value before upgrade and after upgrade setup +for row in `cat $path1/configuration_before_upgrade` +do + + grep $row $path2/configuration_upgrade > $a + if [ ! -s $a ] + then + + echo $row >> ./mismatch_config_between_before_and_after_upgrade + count=`wc -l <./mismatch_config_between_before_and_after_upgrade` + echo $row > temp + awk '{print $1}' temp > temp1 + for word in `cat ./temp1` + do + #echo $word + mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value from cloud.configuration where name= '$word'" >> ./mismatch_config_between_before_and_after_upgrade + count1=`wc -l <./mismatch_config_between_before_and_after_upgrade` + if [ $count == $count1 ] + then + echo $row >> ./configurations_only_before_upgrade + sed -i '$ d' ./mismatch_config_between_before_and_after_upgrade + fi + + + + done + fi + +done + + +#to find configuration present only in the upgraded setup + + +for row in `cat $path2/configuration_upgrade` +do + + grep $row $path1/configuration_before_upgrade > $a + if [ ! -s $a ] + then + echo $row >> ./config_difference_before_and_after_upgrade + count=`wc -l <./config_difference_before_and_after_upgrade` + echo $row > temp + awk '{print $1}' temp > temp1 + for word in `cat ./temp1` + do + grep '^'$word'[^\.]\w*' $path1/configuration_before_upgrade >> ./config_difference_before_and_after_upgrade + count1=`wc -l <./config_difference_before_and_after_upgrade` + if [ $count == $count1 ] + then + echo $row >> ./new_in_upgrade + sed -i '$ d' ./config_difference_before_and_after_upgrade + + fi + + + + done + fi + +done + +#to find all the difference between before and after upgrade +cat ./mismatch_config_between_before_and_after_upgrade >> ./config_difference_before_and_after_upgrade +sort ./config_difference_before_and_after_upgrade > ./config_difference_before_and_after_upgrade.sort +uniq ./config_difference_before_and_after_upgrade.sort > ./config_difference_before_and_after_upgrade + + + + + +rm -rf $path2 *.sort category description scope component temp temp1 $a +rm -rf mismatch_config_between_before_and_after_upgrade config_difference_before_and_after_upgrade.sort t + diff --git a/tools/utils/database_comparision_during_upgrade/test_config_between_fresh_and_upgraded_setup.sh b/tools/utils/database_comparision_during_upgrade/test_config_between_fresh_and_upgraded_setup.sh new file mode 100644 index 00000000000..e621a7948ae --- /dev/null +++ b/tools/utils/database_comparision_during_upgrade/test_config_between_fresh_and_upgraded_setup.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +dbhost=$1 +dbuser=$2 +dbpwd=$3 +a=0 +rm -rf *.uniq only_in_upgraded only_in_fresh new_in_upgrade difference_in_config_between_fresh_and_upgraded +path=./base_data +path2=./data_after_upgrade + +rm -rf $path2 +mkdir $path2 + +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value from cloud.configuration" > $path2/configuration_upgrade +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, component from cloud.configuration" > $path2/component_configuration_upgrade +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, category from cloud.configuration" > $path2/category_configuration_upgrade +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, description from cloud.configuration" > $path2/description_configuration_upgrade +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, scope from cloud.configuration" > $path2/scope_configuration_upgrade + + +IFS=$'\n' + + +#to find difference between upgraded configuration and fresh install configuration + +mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value, default_value from cloud.configuration" > $path2/configuration_upgrade + + + +for row in `cat $path2/configuration_upgrade` +do + + grep $row $path/configuration_fresh > $a + if [ ! -s $a ] + then + + echo $row > temp + name=`awk '{print $1}' temp` + value=`awk '{print $2}' temp` + default=`awk '{print $3}' temp` + grep '^'$name'[^\.]\w*' $path/configuration_fresh > t + if [ ! -s t ] + then + echo $row >> ./only_in_upgraded + else + fname=`awk '{print $1}' t` + fvalue=`awk '{print $2}' t` + fdefault=`awk '{print $3}' t` + #echo $fname $fvalue $fdefault + if [ $default != $value ] + then + if [ $default == $fdefault ] && [ $value == $fvalue ] + then + echo + else + first="$name $value" + second="$fname $fvalue" + echo "in upgrade" >>difference_in_config_between_fresh_and_upgraded + echo $first >> ./difference_in_config_between_fresh_and_upgraded + echo "in fresh" >>difference_in_config_between_fresh_and_upgraded + echo $second >> ./difference_in_config_between_fresh_and_upgraded + fi + fi + fi + + fi +done + +# to find configuration only available in fresh install but missing in upgraded setup + +for row in `cat $path/configuration_fresh` +do + + grep $row $path2/configuration_upgrade > $a + if [ ! -s $a ] + then + + echo "in fresh install" >>final_diff4 + echo $row >> ./final_diff4 + count=`wc -l <./final_diff4` + echo $row > temp + awk '{print $1}' temp > temp1 + for word in `cat ./temp1` + do + #echo $word + echo "in upgrde\n" >> final_diff4 + grep '^'$word'[^\.]\w*' $path2/configuration_upgrade >> ./final_diff4 + count1=`wc -l <./final_diff4` + count1=`expr $count1 - 1` + + if [ $count == $count1 ] + then + echo $row >> ./only_in_fresh + fi + + done + fi + +done +rm -rf final_diff4 + + +# to find difference between upgraded and fresh install on component field + +cat $path2/component_configuration_upgrade > ./component +cat $path/component_configuration_fresh >> ./component +sort ./component > ./component.sort +uniq -u ./component.sort > component.uniq + + + + + +# to find different between upgraded and fresh install on category field + +cat $path2/category_configuration_upgrade > ./category +cat $path/category_configuration_fresh >> ./category +sort ./category > ./category.sort +uniq -u ./category.sort > category.uniq + +# to find different between upgraded and fresh install on scope field + +cat $path2/scope_configuration_upgrade > ./scope +cat $path/scope_configuration_fresh >> ./scope +sort ./scope > ./scope.sort +uniq -u ./scope.sort > scope.uniq + +# to find different between upgraded and fresh install on description field + +cat $path2/description_configuration_upgrade > ./description +cat $path/description_configuration_fresh >> ./description +sort ./description > ./description.sort +uniq -u ./description.sort > description.uniq + + + + +rm -rf $path2 *.sort category description scope component temp temp1 $a +rm -rf mismatch_config_between_before_and_after_upgrade config_difference_before_and_after_upgrade.sort t +#rm -rf $path2 + + + + + + + + + + + + + diff --git a/tools/utils/database_comparision_during_upgrade/usage_schema_comparison.sh b/tools/utils/database_comparision_during_upgrade/usage_schema_comparison.sh new file mode 100644 index 00000000000..bd7d3fc5be2 --- /dev/null +++ b/tools/utils/database_comparision_during_upgrade/usage_schema_comparison.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +dbhost=$1 +dbuser=$2 +dbpwd=$3 +path=./base_data/usage_data +path1=./data_after_upgrade +rm -rf $path1 +mkdir $path1 + +mysql -u $dbuser -p$dbpwd -h $dbhost -e "show tables from cloud_usage" > $path1/usage_tables_upgrade + +# to check if number of tables and table name differs + +diff $path/usage_tables $path1/usage_tables_upgrade > usage_table_difference +if [ -s usage_table_difference ] +then + echo "usage table differs between fresh and upgraded install " + cat usage_table_difference + # do something as file has data +else + echo "usage tables are identicals between fresh and upgraded install " + rm -rf usage_tables_difference + # do something as file is empty + +fi + + + +for tablename in `cat $path1/usage_tables_upgrade` +do + if [ $tablename != 'Tables_in_cloud_usage' ] + then + mysql -u $dbuser -p$dbpwd -h $dbhost -e "describe cloud_usage.$tablename" > $path1/upgradedschema + cat $path/$tablename > $tablename.diff + cat $path1/upgradedschema >> $tablename.diff + sort $tablename.diff > $tablename.sort + uniq -u $tablename.sort > $tablename.uniq + + if [ -s $tablename.uniq ] + then + echo $tablename "table schema is different." + cat $path1/upgradedschema > usage_$tablename + rm -rf $tablename.diff $tablename.sort + + # do something as file has data + else + + rm -rf $tablename.diff $tablename.sort $tablename.uniq + fi + + + fi +done + + + + +rm -rf $path1 +