CLOUDSTACK-8429: optimizing travis to run configurable set of tests only after configurable runs

.travis.yml

@@ -24,25 +24,26 @@ jdk:
 notifications:
   email: false
 env:
-- RUNUNITTESTS=true TESTS=""
+  global:
-- TESTS="smoke/test_affinity_groups smoke/test_deploy_vms_with_varied_deploymentplanners smoke/test_disk_offerings smoke/test_global_settings"
+     - REGRESSION_CYCLE=4
-- TESTS="smoke/test_portable_publicip smoke/test_primary_storage smoke/test_privategw_acl smoke/test_public_ip_range smoke/test_pvlan smoke/test_regions"
+     - REGRESSION_INDEX=6
-- TESTS="smoke/test_reset_vm_on_reboot smoke/test_resource_detail smoke/test_routers smoke/test_guest_vlan_range smoke/test_iso"
+  matrix:
-- TESTS="smoke/test_secondary_storage smoke/test_service_offerings smoke/test_ssvm smoke/test_templates"
+    - TESTS="smoke/test_affinity_groups smoke/test_primary_storage"
-- TESTS="smoke/test_multipleips_per_nic smoke/test_network smoke/test_non_contigiousvlan smoke/test_over_provisioning"
+    - TESTS="smoke/test_deploy_vms_with_varied_deploymentplanners smoke/test_disk_offerings smoke/test_global_settings smoke/test_multipleips_per_nic"
-- TESTS="smoke/test_volumes smoke/test_vpc_vpn smoke/misc/test_deploy_vm smoke/test_vm_life_cycle component/test_mm_max_limits"
+    - TESTS="smoke/test_portable_publicip smoke/test_privategw_acl smoke/test_public_ip_range smoke/test_pvlan smoke/test_regions smoke/test_network"
-- TESTS="component/test_acl_isolatednetwork_delete component/test_mm_domain_limits component/test_acl_listsnapshot"
+    - TESTS="smoke/test_reset_vm_on_reboot smoke/test_resource_detail smoke/test_routers smoke/test_guest_vlan_range smoke/test_iso smoke/test_non_contigiousvlan"
-- TESTS="component/test_acl_listvm component/test_acl_listvolume component/test_acl_sharednetwork_deployVM-impersonation component/test_acl_sharednetwork"
+    - TESTS="smoke/test_secondary_storage smoke/test_service_offerings smoke/test_ssvm smoke/test_templates smoke/test_over_provisioning"
-- TESTS="component/test_snapshots"
+
+    - TESTS="smoke/test_volumes smoke/test_vpc_vpn smoke/misc/test_deploy_vm smoke/test_vm_life_cycle component/test_mm_max_limits"
+    - TESTS="component/test_acl_isolatednetwork_delete component/test_mm_domain_limits component/test_acl_listsnapshot"
+    - TESTS="component/test_acl_listvm component/test_acl_sharednetwork_deployVM-impersonation component/test_acl_sharednetwork"
+    - TESTS="component/test_snapshots component/test_acl_listvolume"
 before_install: travis_wait 30 ./tools/travis/before_install.sh
 install: ./tools/travis/install.sh
 before_script: travis_wait 30 ./tools/travis/before_script.sh
 script:
-- mkdir -p integration-test-results/smoke/misc
+  - travis_wait 30 sleep 30
-- mkdir -p integration-test-results/component
+  - ./tools/travis/script.sh $TESTS
-- travis_wait 30 sleep 30
-- for suite in $TESTS; do travis_wait 30 nosetests --with-xunit --xunit-file=integration-test-results/$suite.xml --with-marvin --marvin-config=setup/dev/advanced.cfg test/integration/$suite.py -s -a tags=advanced,required_hardware=false --zone=Sandbox-simulator --hypervisor=simulator || true ; done
-- python ./tools/travis/xunit-reader.py integration-test-results/
 after_success: ./tools/travis/after_success.sh
 after_failure: ./tools/travis/after_failure.sh
 after_script: ./tools/travis/after_script.sh

tools/travis/after_script.sh

@@ -19,4 +19,25 @@
 # This script should run any tear down commands required.
 #
 
+export TEST_JOB_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f1`
+export TEST_SEQUENCE_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f2`
+
+echo "REGRESSION_CYCLE=$REGRESSION_CYCLE"
+echo "TEST_JOB_NUMBER=$TEST_JOB_NUMBER"
+echo "TEST_SEQUENCE_NUMBER=$TEST_SEQUENCE_NUMBER"
+
+#run regression test only on $REGRESSION_CYCLE
+MOD=$(( $TEST_JOB_NUMBER % $REGRESSION_CYCLE ))
+
+echo "MOD=$MOD"
+
+if [ $MOD -ne 0 ]; then
+ if [ $TEST_SEQUENCE_NUMBER -ge $REGRESSION_INDEX ]; then
+   #skip test
+   echo "Skipping tests ... SUCCESS !"
+   exit 0
+ fi
+fi
+
+
 mvn -Dsimulator -pl client jetty:stop 2>&1

tools/travis/before_install.sh

@@ -21,6 +21,26 @@
 # or internet downloads.
 #
 
+export TEST_JOB_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f1`
+export TEST_SEQUENCE_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f2`
+
+echo "REGRESSION_CYCLE=$REGRESSION_CYCLE"
+echo "TEST_JOB_NUMBER=$TEST_JOB_NUMBER"
+echo "TEST_SEQUENCE_NUMBER=$TEST_SEQUENCE_NUMBER"
+
+#run regression test only on $REGRESSION_CYCLE
+MOD=$(( $TEST_JOB_NUMBER % $REGRESSION_CYCLE ))
+
+echo "MOD=$MOD"
+
+if [ $MOD -ne 0 ]; then
+ if [ $TEST_SEQUENCE_NUMBER -ge $REGRESSION_INDEX ]; then
+   #skip test
+   echo "Skipping tests ... SUCCESS !"
+   exit 0
+ fi
+fi
+
 echo -e "#### System Information ####"
 
 echo -e "\nJava Version: "

tools/travis/before_script.sh

@@ -19,6 +19,22 @@
 # This script should be used to bring up the environment.
 #
 
+
+export TEST_JOB_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f1`
+export TEST_SEQUENCE_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f2`
+
+#run regression test only on $REGRESSION_CYCLE
+MOD=$(( $TEST_JOB_NUMBER % $REGRESSION_CYCLE ))
+
+if [ $MOD -ne 0 ]; then
+ if [ $TEST_SEQUENCE_NUMBER -ge $REGRESSION_INDEX ]; then
+   #skip test
+   echo "Skipping tests ... SUCCESS !"
+   exit 0
+ fi
+fi
+
+
 export CATALINA_BASE=/opt/tomcat
 export CATALINA_HOME=/opt/tomcat
 export M2_HOME="/usr/local/maven-3.2.1/"
@@ -27,4 +43,4 @@ export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=500m"
 mvn -Dsimulator -pl :cloud-client-ui jetty:run 2>&1 > /dev/null &
 
 while ! nc -vz localhost 8096 2>&1 > /dev/null; do sleep 10; done
-python -m marvin.deployDataCenter -i setup/dev/advanced.cfg 2>&1 || true
+python -m marvin.deployDataCenter -i setup/dev/advanced.cfg 2>&1 || true

tools/travis/install.sh

@@ -19,19 +19,32 @@
 # This should be used to create the build.
 #
 
+
+export TEST_JOB_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f1`
+export TEST_SEQUENCE_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f2`
+
+#run regression test only on $REGRESSION_CYCLE
+MOD=$(( $TEST_JOB_NUMBER % $REGRESSION_CYCLE ))
+
+if [ $MOD -ne 0 ]; then
+ if [ $TEST_SEQUENCE_NUMBER -ge $REGRESSION_INDEX ]; then
+   #skip test
+   echo "Skipping tests ... SUCCESS !"
+   exit 0
+ fi
+fi
+
 export CATALINA_BASE=/opt/tomcat
 export CATALINA_HOME=/opt/tomcat
 export M2_HOME="/usr/local/maven-3.2.1/"
 export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=500m"
 
-# Compile Cloudstack
+if [ $TEST_SEQUENCE_NUMBER -eq 1 ]; then
-if [[ $RUNUNITTESTS == true ]]; then
+   mvn -q -Pimpatient -Dsimulator clean install
-    mvn -q -Pimpatient -Dsimulator clean install
 else
-    mvn -q -Pimpatient -Dsimulator clean install -DskipTests=true
+   mvn -q -Pimpatient -Dsimulator clean install -DskipTests=true
 fi
 
-
 # Compile API Docs
 cd tools/apidoc
 mvn -q clean install
@@ -45,4 +58,4 @@ cd ../../
 
 # Deploy the database
 mvn -q -Pdeveloper -pl developer -Ddeploydb
-mvn -q -Pdeveloper -pl developer -Ddeploydb-simulator
+mvn -q -Pdeveloper -pl developer -Ddeploydb-simulator

tools/travis/script.sh

@@ -18,4 +18,30 @@
 #
 # This script should be responsible for bring up the
 # test environment and executing the tests.
-#
+#
+
+export TEST_JOB_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f1`
+export TEST_SEQUENCE_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f2`
+
+#run regression test only on $REGRESSION_CYCLE
+MOD=$(( $TEST_JOB_NUMBER % $REGRESSION_CYCLE ))
+
+if [ $MOD -ne 0 ]; then
+ if [ $TEST_SEQUENCE_NUMBER -ge $REGRESSION_INDEX ]; then
+   #skip test
+   echo "Skipping tests ... SUCCESS !"
+   exit 0
+ fi
+fi
+
+mkdir -p integration-test-results/smoke/misc
+mkdir -p integration-test-results/component
+
+
+for suite in $1; do
+  travis_wait 30
+  nosetests --with-xunit --xunit-file=integration-test-results/$suite.xml --with-marvin --marvin-config=setup/dev/advanced.cfg test/integration/$suite.py -s -a tags=advanced,required_hardware=false --zone=Sandbox-simulator --hypervisor=simulator || true ;
+done
+
+
+python ./tools/travis/xunit-reader.py integration-test-results/