diff --git a/.travis.yml b/.travis.yml index 34220fe05b0..da81633bb17 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,9 +17,6 @@ sudo: required dist: trusty language: java -cache: - directories: - - $HOME/.cache jdk: - oraclejdk7 python: @@ -27,31 +24,134 @@ python: cache: directories: - $HOME/.m2 - timeout: 1000 + timeout: 500 notifications: email: false env: global: - - REGRESSION_CYCLE=4 - - REGRESSION_INDEX=6 - PATH=$HOME/.local/bin:$PATH matrix: - - TESTS="smoke/test_affinity_groups smoke/test_affinity_groups_projects smoke/test_dynamicroles smoke/test_deploy_vgpu_enabled_vm smoke/test_deploy_vm_iso smoke/test_deploy_vm_root_resize smoke/test_deploy_vm_with_userdata smoke/test_deploy_vms_with_varied_deploymentplanners smoke/test_disk_offerings smoke/test_global_settings smoke/test_guest_vlan_range" - - TESTS="smoke/test_hosts smoke/test_internal_lb smoke/test_iso smoke/test_list_ids_parameter smoke/test_loadbalance smoke/test_login smoke/test_multipleips_per_nic smoke/test_network smoke/test_network_acl smoke/test_nic smoke/test_nic_adapter_type smoke/test_non_contigiousvlan" - - TESTS="smoke/test_outofbandmanagement smoke/test_over_provisioning smoke/test_password_server smoke/test_portable_publicip smoke/test_primary_storage smoke/test_privategw_acl smoke/test_public_ip_range smoke/test_pvlan smoke/test_regions smoke/test_reset_vm_on_reboot smoke/test_resource_detail" - - TESTS="smoke/test_router_dhcphosts smoke/test_routers smoke/test_routers_iptables_default_policy smoke/test_routers_network_ops smoke/test_staticroles smoke/test_scale_vm smoke/test_secondary_storage smoke/test_service_offerings smoke/test_snapshots smoke/test_ssvm smoke/test_templates" - - TESTS="smoke/test_usage_events smoke/test_vm_life_cycle smoke/test_vm_snapshots smoke/test_volumes smoke/test_vpc_redundant smoke/test_vpc_router_nics smoke/test_vpc_vpn smoke/misc/test_deploy_vm smoke/misc/test_vm_ha smoke/misc/test_escalations_templates smoke/misc/test_vm_sync" + - TESTS="smoke/test_affinity_groups + smoke/test_affinity_groups_projects + smoke/test_deploy_vgpu_enabled_vm + smoke/test_deploy_vm_iso + smoke/test_deploy_vm_root_resize + smoke/test_deploy_vm_with_userdata + smoke/test_deploy_vms_with_varied_deploymentplanners + smoke/test_disk_offerings + smoke/test_dynamicroles + smoke/test_global_settings + smoke/test_guest_vlan_range + smoke/test_hosts + smoke/test_internal_lb + smoke/test_iso + smoke/test_list_ids_parameter + smoke/test_loadbalance + smoke/test_login + smoke/test_multipleips_per_nic + smoke/test_network + smoke/test_network_acl + smoke/test_nic + smoke/test_nic_adapter_type + smoke/test_non_contigiousvlan" + + - TESTS="smoke/test_outofbandmanagement + smoke/test_over_provisioning + smoke/test_password_server + smoke/test_portable_publicip + smoke/test_primary_storage + smoke/test_privategw_acl + smoke/test_public_ip_range + smoke/test_pvlan + smoke/test_regions + smoke/test_reset_vm_on_reboot + smoke/test_resource_detail + smoke/test_router_dhcphosts + smoke/test_router_dns + smoke/test_routers + smoke/test_routers_iptables_default_policy + smoke/test_routers_network_ops + smoke/test_scale_vm + smoke/test_secondary_storage + smoke/test_service_offerings + smoke/test_snapshots + smoke/test_ssvm + smoke/test_staticroles + smoke/test_templates + smoke/test_usage_events + smoke/test_vm_life_cycle + smoke/test_vm_snapshots + smoke/test_volumes + smoke/test_vpc_redundant + smoke/test_vpc_router_nics + smoke/test_vpc_vpn + smoke/misc/test_deploy_vm + smoke/misc/test_escalations_templates + smoke/misc/test_vm_ha + smoke/misc/test_vm_sync" + + - TESTS="component/find_hosts_for_migration + component/test_acl_isolatednetwork_delete + component/test_acl_listsnapshot + component/test_acl_listvm + component/test_acl_listvolume + component/test_acl_sharednetwork + component/test_acl_sharednetwork_deployVM-impersonation" + component/test_cpu_limits" + + - TESTS="component/test_cpu_max_limits + component/test_cpu_project_limits + component/test_deploy_vm_userdata_multi_nic + component/test_egress_fw_rules + component/test_invalid_gw_nm + component/test_ip_reservation + component/test_lb_secondary_ip" + + - TESTS="component/test_memory_limits + component/test_mm_domain_limits + component/test_mm_max_limits + component/test_mm_project_limits + component/test_network_offering + component/test_non_contiguous_vlan + component/test_persistent_networks" + + - TESTS="component/test_projects + component/test_project_configs + component/test_project_limits + component/test_project_usage + component/test_regions + component/test_regions_accounts + component/test_routers + component/test_snapshots" + + - TESTS="component/test_resource_limits" + + - TESTS="component/test_stopped_vm + component/test_tags + component/test_templates + component/test_update_vm + component/test_usage" + + - TESTS="component/test_volumes + component/test_vpc_network + component/test_vpc_offerings + component/test_vpn_users" + +# FIXME: fix following tests and include them in Travis +# - TESTS="component/test_affinity_groups_projects" +# - TESTS="component/test_allocation_states" +# - TESTS="component/test_vpc" +# - TESTS="component/test_project_resources" +# - TESTS="component/test_cpu_domain_limits" +# - TESTS="component/test_acl_isolatednetwork" +# - TESTS="component/test_accounts" +# - TESTS="component/test_organization_states" - - TESTS="component/test_mm_max_limits component/test_acl_isolatednetwork_delete" - - TESTS="component/test_mm_domain_limits component/test_acl_listsnapshot" - - TESTS="component/test_acl_listvm component/test_acl_sharednetwork_deployVM-impersonation component/test_acl_sharednetwork" - - TESTS="component/test_snapshots component/test_acl_listvolume" before_install: travis_wait 30 ./tools/travis/before_install.sh install: ./tools/travis/install.sh before_script: travis_wait 30 ./tools/travis/before_script.sh script: - - travis_wait 30 sleep 30 - - ./tools/travis/script.sh ${TESTS} + - travis_wait 40 ./tools/travis/script.sh ${TESTS} after_success: ./tools/travis/after_success.sh after_failure: ./tools/travis/after_failure.sh after_script: ./tools/travis/after_script.sh diff --git a/developer/developer-prefill.sql b/developer/developer-prefill.sql index 132659e3d94..3fd3a1bb93d 100644 --- a/developer/developer-prefill.sql +++ b/developer/developer-prefill.sql @@ -52,15 +52,15 @@ INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) VALUES ('Advanced', 'DEFAULT', 'management-server', - 'account.cleanup.interval', '60'); + 'account.cleanup.interval', '20'); INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) VALUES ('Advanced', 'DEFAULT', 'management-server', - 'expunge.delay', '60'); + 'expunge.delay', '20'); INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) VALUES ('Advanced', 'DEFAULT', 'management-server', - 'expunge.interval', '60'); + 'expunge.interval', '20'); INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) VALUES ('Advanced', 'DEFAULT', 'management-server', @@ -84,15 +84,15 @@ INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) VALUES ('Advanced', 'DEFAULT', 'management-server', - 'network.gc.wait', '60'); + 'network.gc.wait', '20'); INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) VALUES ('Advanced', 'DEFAULT', 'management-server', - 'network.gc.interval', '60'); + 'network.gc.interval', '20'); INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) VALUES ('Advanced', 'DEFAULT', 'management-server', - 'storage.cleanup.interval', '150'); + 'storage.cleanup.interval', '40'); INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) VALUES ('Advanced', 'DEFAULT', 'management-server', diff --git a/setup/dev/advanced.cfg b/setup/dev/advanced.cfg index 48241fa6645..0742f2468ee 100644 --- a/setup/dev/advanced.cfg +++ b/setup/dev/advanced.cfg @@ -150,11 +150,11 @@ "globalConfig": [ { "name": "network.gc.wait", - "value": "60" + "value": "20" }, { "name": "storage.cleanup.interval", - "value": "60" + "value": "40" }, { "name": "vm.op.wait.interval", @@ -166,7 +166,7 @@ }, { "name": "network.gc.interval", - "value": "60" + "value": "20" }, { "name": "instance.name", @@ -178,7 +178,7 @@ }, { "name": "account.cleanup.interval", - "value": "60" + "value": "20" }, { "name": "guest.domain.suffix", @@ -186,7 +186,7 @@ }, { "name": "expunge.delay", - "value": "60" + "value": "20" }, { "name": "vm.allocation.algorithm", @@ -194,7 +194,7 @@ }, { "name": "expunge.interval", - "value": "60" + "value": "20" }, { "name": "expunge.workers", diff --git a/test/integration/smoke/test_outofbandmanagement.py b/test/integration/smoke/test_outofbandmanagement.py index 122fcfc75e7..e91d49a91b0 100644 --- a/test/integration/smoke/test_outofbandmanagement.py +++ b/test/integration/smoke/test_outofbandmanagement.py @@ -38,24 +38,48 @@ class TestOutOfBandManagement(cloudstackTestCase): """ Test cases for out of band management """ - def setUp(self): - self.apiclient = self.testClient.getApiClient() - self.hypervisor = self.testClient.getHypervisorInfo() - self.dbclient = self.testClient.getDbConnection() - self.services = self.testClient.getParsedTestDataConfig() - self.mgtSvrDetails = self.config.__dict__["mgtSvr"][0].__dict__ - self.fakeMsId = random.randint(10000, 99999) * random.randint(10, 20) - - self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests()) - self.host = None - self.server = None + @classmethod + def setUpClass(cls): + testClient = super(TestOutOfBandManagement, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) + cls.host = None + cls.cleanup = [] # use random port for ipmisim s = socket.socket() s.bind(('', 0)) - self.serverPort = s.getsockname()[1] + cls.serverPort = s.getsockname()[1] s.close() + def startIpmiServer(tname, server): + try: + server.serve_forever() + except Exception: pass + IpmiServerContext('reset') + ThreadedIpmiServer.allow_reuse_address = True + server = ThreadedIpmiServer(('0.0.0.0', cls.serverPort), IpmiServer) + thread.start_new_thread(startIpmiServer, ("ipmi-server", server,)) + cls.server = server + + + @classmethod + def tearDownClass(cls): + try: + cleanup_resources(cls.apiclient, cls.cleanup) + cls.server.shutdown() + cls.server.server_close() + IpmiServerContext('reset') + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.mgtSvrDetails = self.config.__dict__["mgtSvr"][0].__dict__ + self.fakeMsId = random.randint(10000, 99999) * random.randint(10, 20) self.cleanup = [] @@ -67,10 +91,6 @@ class TestOutOfBandManagement(cloudstackTestCase): self.dbclient.execute("delete from cluster_details where name='outOfBandManagementEnabled'") self.dbclient.execute("delete from data_center_details where name='outOfBandManagementEnabled'") cleanup_resources(self.apiclient, self.cleanup) - if self.server: - self.server.shutdown() - self.server.server_close() - IpmiServerContext('reset') except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) @@ -145,31 +165,28 @@ class TestOutOfBandManagement(cloudstackTestCase): if timeout: cmd.timeout = timeout - try: - return self.apiclient.issueOutOfBandManagementPowerAction(cmd) - except Exception as e: - if "packet session id 0x0 does not match active session" in str(e): - raise self.skipTest("Known ipmitool issue hit, skipping test") - raise e + tries = 3 + while tries > 0: + tries -= 1 + try: + return self.apiclient.issueOutOfBandManagementPowerAction(cmd) + except Exception as e: + if tries <= 0: + if "packet session id 0x0 does not match active session" in str(e): + raise self.skipTest("Known ipmitool issue hit, skipping test") + raise e - def configureAndEnableOobm(self): + def configureAndEnableOobm(self, power_state=None): + """ + Setup ipmisim and enable out-of-band management for host + """ self.apiclient.configureOutOfBandManagement(self.getOobmConfigCmd()) response = self.apiclient.enableOutOfBandManagementForHost(self.getOobmEnableCmd()) self.assertEqual(response.enabled, True) - - - def startIpmiServer(self): - def startIpmiServer(tname, server): - self.debug("Starting ipmisim server") - try: - server.serve_forever() - except Exception: pass - IpmiServerContext('reset') - ThreadedIpmiServer.allow_reuse_address = False - server = ThreadedIpmiServer(('0.0.0.0', self.getIpmiServerPort()), IpmiServer) - thread.start_new_thread(startIpmiServer, ("ipmi-server", server,)) - self.server = server + if power_state: + bmc = IpmiServerContext().bmc + bmc.powerstate = power_state def checkSyncToState(self, state, interval): @@ -178,7 +195,7 @@ class TestOutOfBandManagement(cloudstackTestCase): return response.powerstate == expectedState, None sync_interval = 1 + int(interval)/1000 - res, _ = wait_until(sync_interval, 10, checkForStateSync, state) + res, _ = wait_until(sync_interval, 20, checkForStateSync, state) if not res: self.fail("Failed to get host.powerstate synced to expected state:" + state) response = self.getHost(hostId=self.getHost().id).outofbandmanagement @@ -301,7 +318,6 @@ class TestOutOfBandManagement(cloudstackTestCase): and zone level sequentially Zone > Cluster > Host """ self.configureAndEnableOobm() - self.startIpmiServer() bmc = IpmiServerContext().bmc bmc.powerstate = 'off' @@ -366,17 +382,6 @@ class TestOutOfBandManagement(cloudstackTestCase): self.assertEqual(response.powerstate, 'Off') - def configureAndStartIpmiServer(self, power_state=None): - """ - Setup ipmisim and enable out-of-band management for host - """ - self.configureAndEnableOobm() - self.startIpmiServer() - if power_state: - bmc = IpmiServerContext().bmc - bmc.powerstate = power_state - - def assertIssueCommandState(self, command, expected): """ Asserts power action result for a given power command @@ -392,7 +397,7 @@ class TestOutOfBandManagement(cloudstackTestCase): """ Tests out-of-band management issue power action """ - self.configureAndStartIpmiServer(power_state='on') + self.configureAndEnableOobm(power_state='on') self.assertIssueCommandState('STATUS', 'On') @@ -401,7 +406,7 @@ class TestOutOfBandManagement(cloudstackTestCase): """ Tests out-of-band management issue power on action """ - self.configureAndStartIpmiServer() + self.configureAndEnableOobm() self.assertIssueCommandState('ON', 'On') @@ -410,7 +415,7 @@ class TestOutOfBandManagement(cloudstackTestCase): """ Tests out-of-band management issue power off action """ - self.configureAndStartIpmiServer() + self.configureAndEnableOobm() self.assertIssueCommandState('OFF', 'Off') @@ -419,7 +424,7 @@ class TestOutOfBandManagement(cloudstackTestCase): """ Tests out-of-band management issue power cycle action """ - self.configureAndStartIpmiServer() + self.configureAndEnableOobm() self.assertIssueCommandState('CYCLE', 'On') @@ -428,7 +433,7 @@ class TestOutOfBandManagement(cloudstackTestCase): """ Tests out-of-band management issue power reset action """ - self.configureAndStartIpmiServer() + self.configureAndEnableOobm() self.assertIssueCommandState('RESET', 'On') @@ -437,7 +442,7 @@ class TestOutOfBandManagement(cloudstackTestCase): """ Tests out-of-band management issue power soft action """ - self.configureAndStartIpmiServer() + self.configureAndEnableOobm() self.assertIssueCommandState('SOFT', 'Off') @@ -453,7 +458,6 @@ class TestOutOfBandManagement(cloudstackTestCase): )[0].value self.configureAndEnableOobm() - self.startIpmiServer() bmc = IpmiServerContext().bmc bmc.powerstate = 'on' @@ -462,10 +466,10 @@ class TestOutOfBandManagement(cloudstackTestCase): bmc.powerstate = 'off' self.checkSyncToState('Off', interval) - self.server.shutdown() - self.server.server_close() - # Check for unknown state (ipmi server not reachable) + cmd = self.getOobmConfigCmd() + cmd.port = 1 + response = self.apiclient.configureOutOfBandManagement(cmd) self.checkSyncToState('Unknown', interval) @@ -533,7 +537,6 @@ class TestOutOfBandManagement(cloudstackTestCase): name='outofbandmanagement.sync.interval' )[0].value - self.startIpmiServer() bmc = IpmiServerContext().bmc bmc.powerstate = 'on' @@ -550,7 +553,6 @@ class TestOutOfBandManagement(cloudstackTestCase): Tests out-of-band management change password feature """ self.configureAndEnableOobm() - self.startIpmiServer() self.debug("Testing oobm change password") diff --git a/test/integration/smoke/test_regions.py b/test/integration/smoke/test_regions.py index f8e792d00ff..7b0dec3c6ee 100644 --- a/test/integration/smoke/test_regions.py +++ b/test/integration/smoke/test_regions.py @@ -35,7 +35,7 @@ class TestRegions(cloudstackTestCase): cls.domain = get_domain(cls.apiclient) cls.cleanup = [] - @attr(tags=["basic", "advanced"], required_hardware="true") + @attr(tags=["basic", "advanced"], required_hardware="false") def test_createRegion(self): """ Test for create region """ diff --git a/tools/travis/after_failure.sh b/tools/travis/after_failure.sh index df40f733671..e365567ddf2 100755 --- a/tools/travis/after_failure.sh +++ b/tools/travis/after_failure.sh @@ -19,4 +19,5 @@ # As the filename suggests, this is executed on build failure # failure. # +find rat.txt -type f -exec echo -e "Printing RAT license-failures {} :\n" \; -exec cat {} \; find /tmp//MarvinLogs -type f -exec echo -e "Printing logfile {} :\n" \; -exec cat {} \; diff --git a/tools/travis/after_script.sh b/tools/travis/after_script.sh index 828592d6091..17a9d5356bd 100755 --- a/tools/travis/after_script.sh +++ b/tools/travis/after_script.sh @@ -19,25 +19,4 @@ # This script should run any tear down commands required. # -export TEST_JOB_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f1` -export TEST_SEQUENCE_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f2` - -echo "REGRESSION_CYCLE=$REGRESSION_CYCLE" -echo "TEST_JOB_NUMBER=$TEST_JOB_NUMBER" -echo "TEST_SEQUENCE_NUMBER=$TEST_SEQUENCE_NUMBER" - -#run regression test only on $REGRESSION_CYCLE -MOD=$(( $TEST_JOB_NUMBER % $REGRESSION_CYCLE )) - -echo "MOD=$MOD" - -if [ $MOD -ne 0 ]; then - if [ $TEST_SEQUENCE_NUMBER -ge $REGRESSION_INDEX ]; then - #skip test - echo "Skipping tests ... SUCCESS !" - exit 0 - fi -fi - - mvn -Dsimulator -pl client jetty:stop 2>&1 diff --git a/tools/travis/before_install.sh b/tools/travis/before_install.sh index 99195c265a0..07eadd6c2b4 100755 --- a/tools/travis/before_install.sh +++ b/tools/travis/before_install.sh @@ -21,26 +21,6 @@ # or internet downloads. # -export TEST_JOB_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f1` -export TEST_SEQUENCE_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f2` - -echo "REGRESSION_CYCLE=$REGRESSION_CYCLE" -echo "TEST_JOB_NUMBER=$TEST_JOB_NUMBER" -echo "TEST_SEQUENCE_NUMBER=$TEST_SEQUENCE_NUMBER" - -#run regression test only on $REGRESSION_CYCLE -MOD=$(( $TEST_JOB_NUMBER % $REGRESSION_CYCLE )) - -echo "MOD=$MOD" - -if [ $MOD -ne 0 ]; then - if [ $TEST_SEQUENCE_NUMBER -ge $REGRESSION_INDEX ]; then - #skip test - echo "Skipping tests ... SUCCESS !" - exit 0 - fi -fi - echo -e "#### System Information ####" echo -e "\nJava Version: " @@ -125,41 +105,3 @@ do cat /tmp/piplog done -#Download project dependencies in a way we can retry if there's a failure, without failing the whole build - -#Resolve plugins first -echo -e "\nDownloading Plugin dependencies" -for ((i=0;i<$RETRY_COUNT;i++)) -do - #The output file is used on the next phase by the downloadDeps.sh script - mvn org.apache.maven.plugins:maven-dependency-plugin:resolve-plugins | grep "Plugin Resolved:" | sort -u | awk '{print $4}' | tee /tmp/resolvedPlugins - if [[ $? -eq 0 ]]; then - echo -e "\nPlugin dependencies downloaded successfully" - break; - fi - echo -e "\nDependency download failed" - #Test DNS record - getent hosts repo1.maven.org - while ! nc -vzw 5 repo1.maven.org 80; do echo -e "\nFailed to connect to repo1.maven.org:80 will retry in 10 seconds"; sleep 10; done -done - -#Resolve remaining deps -cd tools/travis -echo -e "\nDownloading Project dependencies" - -for ((i=0;i<$RETRY_COUNT;i++)) -do - ./downloadDeps.sh > /tmp/phase2 - if [[ $? -eq 0 ]]; then - echo -e "\n$(cat cleandeps.out |wc -l) project dependencies downloaded successfully" - break; - fi - echo -e "\nDependency download failed" - #Print out errors from failed run - cat /tmp/phase2 | grep -i -e "fail" -e "error" -e "exception" - #Test DNS record - getent hosts repo1.maven.org - while ! nc -vzw 5 repo1.maven.org 80; do echo -e "\nFailed to connect to repo1.maven.org:80 will retry in 10 seconds"; sleep 10; done - echo -e "\nRetrying download" -done -cd ../.. diff --git a/tools/travis/before_script.sh b/tools/travis/before_script.sh index beb8acc89ae..bbfc4472f5a 100755 --- a/tools/travis/before_script.sh +++ b/tools/travis/before_script.sh @@ -18,24 +18,7 @@ # # This script should be used to bring up the environment. # - - -export TEST_JOB_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f1` -export TEST_SEQUENCE_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f2` - -#run regression test only on $REGRESSION_CYCLE -MOD=$(( $TEST_JOB_NUMBER % $REGRESSION_CYCLE )) - -if [ $MOD -ne 0 ]; then - if [ $TEST_SEQUENCE_NUMBER -ge $REGRESSION_INDEX ]; then - #skip test - echo "Skipping tests ... SUCCESS !" - exit 0 - fi -fi - - -export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=500m -Djava.security.egd=file:/dev/./urandom" +export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/urandom" echo -e "\nStarting simulator" mvn -Dsimulator -pl :cloud-client-ui jetty:run 2>&1 > /tmp/jetty-log & diff --git a/tools/travis/downloadDeps.sh b/tools/travis/downloadDeps.sh deleted file mode 100755 index c05e5f5bb03..00000000000 --- a/tools/travis/downloadDeps.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# This script should be used to install additional dependencies -# This includes: installing ubuntu packages, custom services -# or internet downloads. - -# Authored by Rafael da Fonseca - -#Get all dependency blocks from all pom.xml files in the project -for line in $(find ../../ -name pom.xml -exec sed -n '//{:a;n;/<\/dependencies>/b;p;ba}' {} \; | grep -e "artifactId" -e "groupId" -e "version" -e "dependency\>" -e "exclusion\>" -e "exclusions\>"| sed -e 's/\^M//'); do - - #Tokenize values - set -- $(echo $line | awk -v FS="(>|<)" '{print $2, $3}') - - #Start processing data - if [[ $1 == "dependency" ]]; then - #Create new artifact dep - unset ARTIFACT - unset VERSION - unset GROUP - elif [[ $1 == "/dependency" ]]; then - #Filter out project modules interdependencies - if [[ $GROUP != *org.apache.cloudstack* ]] && [[ $GROUP != *com.cloud* ]] && [[ $ARTIFACT != cloudstack-service-console-proxy-rdpclient ]]; then - if [[ -z $VERSION ]] ; then - VERSION=LATEST - #These dependencies don't support the LATEST keywork for some reason, and would cause mvn runs to file on dummy poms - if [[ $GROUP == jstl ]] || [[ $ARTIFACT == mysql-connector-java ]] || [[ $GROUP == org.apache.axis ]]; then - continue - fi - fi - #Output resolved dependency to a file, to be picked up later - echo "$GROUP $ARTIFACT $VERSION" >> deps.out - fi - elif [[ $1 == "version" ]]; then - #If version is a maven var, get the value from parent pom - if [[ $2 == \$\{* ]]; then - VERSION=$(grep \<$(echo $2 | awk -v FS="(}|{)" '{print $2 }') ../../pom.xml | awk -v FS="(>|<)" '{print $3}') - #If version tag is empty, add LATEST to avoid maven errors - elif [[ "$2" == "" ]]; then - VERSION="LATEST" - else - VERSION=$2 - fi - elif [[ $1 == "artifactId" ]]; then - #This avoids exclusions inside dependency block to overwrite original dependency - if [[ -z $ARTIFACT ]]; then - ARTIFACT=$2 - fi - elif [[ $1 == "groupId" ]]; then - #This avoids exclusions inside dependency block to overwrite original dependency - if [[ -z $GROUP ]]; then - GROUP=$2 - fi - fi -done - -#Add the resolved plugins to properly download their dependencies -while read line ; do - NAME=$(echo $line | sed -e 's/.jar$//') - VERSION=${NAME##*-} - ARTIFACT=${NAME%-*} - GROUP=$(find ~/.m2/repository -name ${NAME}.pom -exec sed -n "1,/${ARTIFACT}/p" {} \; | tac | grep -m 1 -e "" | sed -e 's/^[[:space:]]*//' -e 's/\^M//' | tr -d '\r' | awk -v FS="(>|<)" '{print $3}') - DATA="${GROUP} ${ARTIFACT} ${VERSION}" - echo $DATA >> deps.out -done < /tmp/resolvedPlugins - -#Remove duplicates and sort them, LANG export is needed to fix some sorting issue, sorting is needed for later function that relies on sorted input -cat deps.out | LANG=C sort -u > cleandeps.out - -#Define index of pomfiles, to avoid duplicate deps with different versions in pom.xml, several poms are created in case of more than one version of same artifact -LASTPOM=0 -#Create first pom -echo '4.0.0org.apache.cloudstacktravis-build-depsDownload Deps for Travis CI1' > pom${LASTPOM}.xml -#Create pom for dependencies not on central repo, this is done separately to not adversely impact performance on downloading the majority of deps -echo '4.0.0org.apache.cloudstacktravis-build-depsDownload Deps for Travis CI1mido-maven-public-releasesmido-maven-public-releaseshttp://cs-maven.midokura.com/releasesjuniper-contrailhttp://juniper.github.io/contrail-maven/snapshots' > pomX.xml -while read line ; do - set -- $line - #This relies on correct sorting, and distributes different versions of same dependency througout different pom files - if [[ $2 == $LASTARTIFACT ]]; then - POMID=$(($POMID+1)) - #If value is greater than current number of poms, create a new one - if [[ $POMID -gt $LASTPOM ]]; then - LASTPOM=$POMID - #This outputs the necessary structure to start a pom and also defines the extra repositories - echo '4.0.0org.apache.cloudstacktravis-build-depsDownload Deps for Travis CI1' > pom${LASTPOM}.xml - fi - else - POMID=0 - fi - LASTARTIFACT=$2 - if [[ $1 == org.midonet ]] || [[ $1 == net.juniper* ]]; then - echo "$1$2$3" >> pomX.xml - else - echo "$1$2$3" >> pom${POMID}.xml - fi -done < cleandeps.out - -RETURN_CODE=0 -#Close and resolve all pom files -for ((i=0;i<=$LASTPOM;i++)) -do - echo "" >> pom${i}.xml - mvn org.apache.maven.plugins:maven-dependency-plugin:resolve -f pom${i}.xml - if [[ $? -ne 0 ]]; then - RETURN_CODE=1 - fi -done -#Close and resolve external deps pom file -echo "" >> pomX.xml -mvn org.apache.maven.plugins:maven-dependency-plugin:resolve -f pomX.xml - -#Run a few plugin goals to download some more deps - -#Hack to run maven-jaxb2-plugin generate, can be removed when we stop using such an old version... -mkdir -p src/main/resources -echo '' > src/main/resources/test.xsd - -#Declare plugin tests to run -declare -a arr=("maven-surefire-plugin test" "maven-pmd-plugin pmd" "maven-compiler-plugin compile" "maven-resources-plugin resources" "maven-checkstyle-plugin check" "maven-site-plugin attach-descriptor" "maven-surefire-plugin test" "maven-jar-plugin jar" "license-maven-plugin check" "maven-jgit-buildnumber-plugin extract-buildnumber" "maven-jaxb2-plugin generate" "maven-war-plugin war -DfailOnMissingWebXml=false" "gmaven-plugin compile") -for i in "${arr[@]}" -do - set -- $i - PLUGIN=$1 - MOJO=$2 - OPTION=$3 - #Get every listed version of the plugin and make a run for each version - while read line ; do - set -- $line - JOBS="${JOBS} ${1}:${2}:${3}:${MOJO} $OPTION" - done < <(grep $PLUGIN cleandeps.out) -done -echo "Running $JOBS" -#Call all the constructed plugin goals -mvn $JOBS -f pom0.xml -if [[ $? -ne 0 ]]; then - RETURN_CODE=1 -fi - -#Cleanup some files created in the run -rm -rf deps.out src target - -exit $RETURN_CODE diff --git a/tools/travis/install.sh b/tools/travis/install.sh index bbea5503df5..61cdec00849 100755 --- a/tools/travis/install.sh +++ b/tools/travis/install.sh @@ -19,27 +19,19 @@ # This should be used to create the build. # - -export TEST_JOB_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f1` export TEST_SEQUENCE_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f2` +export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/./urandom" -#run regression test only on $REGRESSION_CYCLE -MOD=$(( $TEST_JOB_NUMBER % $REGRESSION_CYCLE )) - -if [ $MOD -ne 0 ]; then - if [ $TEST_SEQUENCE_NUMBER -ge $REGRESSION_INDEX ]; then - #skip test - echo "Skipping tests ... SUCCESS !" - exit 0 - fi -fi - -export MAVEN_OPTS="-Xmx2048m -XX:MaxPermSize=500m -Djava.security.egd=file:/dev/./urandom" +set -e if [ $TEST_SEQUENCE_NUMBER -eq 1 ]; then - mvn -Pdeveloper,systemvm -Dsimulator clean install -T4 | egrep "Building|Tests|SUCCESS|FAILURE" + git clone https://github.com/rhtyd/cloudstack-nonoss.git nonoss + cd nonoss && bash -x install-non-oss.sh && cd .. + git clean -fdx . + mvn -P developer,systemvm -Dsimulator -Dnoredist --projects='org.apache.cloudstack:cloudstack' org.apache.rat:apache-rat-plugin:0.12:check + mvn -P developer,systemvm -Dsimulator -Dnoredist clean install else - mvn -Pdeveloper -Dsimulator clean install -DskipTests=true -T4 | egrep "Building|Tests|SUCCESS|FAILURE" + mvn -Pdeveloper -Dsimulator clean install -DskipTests -T4 | egrep "Building|Tests|SUCCESS|FAILURE" fi # Install mysql-connector-python diff --git a/tools/travis/script.sh b/tools/travis/script.sh index dc955b004c7..48eb3f6b688 100755 --- a/tools/travis/script.sh +++ b/tools/travis/script.sh @@ -20,20 +20,6 @@ # test environment and executing the tests. # -export TEST_JOB_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f1` -export TEST_SEQUENCE_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f2` - -#run regression test only on $REGRESSION_CYCLE -MOD=$(( $TEST_JOB_NUMBER % $REGRESSION_CYCLE )) - -if [ $MOD -ne 0 ]; then - if [ $TEST_SEQUENCE_NUMBER -ge $REGRESSION_INDEX ]; then - #skip test - echo "Skipping tests ... SUCCESS !" - exit 0 - fi -fi - mkdir -p integration-test-results/smoke/misc mkdir -p integration-test-results/component diff --git a/tools/travis/xunit-reader.py b/tools/travis/xunit-reader.py index 80e228b3822..b23fb467dfc 100755 --- a/tools/travis/xunit-reader.py +++ b/tools/travis/xunit-reader.py @@ -73,7 +73,7 @@ def _generate_file_list(args): def parse_reports(file_path_list): table = texttable.Texttable() - table.header(['Test', 'Result', 'Time']) + table.header(['Test', 'Result', 'Time', 'Test file']) exit_code = 0 @@ -99,7 +99,7 @@ def parse_reports(file_path_list): if 'type' in children.attrib: status = children.attrib['type'] - table.add_row([name, status, time]) + table.add_row([name, status, time, file_path.replace(".xml", "").split("/")[-1]]) print table.draw()