diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index cc0fabdc0d2..dfab81d0f79 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -137,8 +137,8 @@ labelPRBasedOnFilePath: - systemvm/debian/etc/systemd/* "component:templates": - tools/appliance/* - "component:travis": - - ".travis.yml" + "component:integration": + - ".github/workflows/*" "component:UI": - ui/* "component:unit-test": diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000..af0b965fae2 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Maven Build (noredist) + +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'adopt' + architecture: x64 + cache: maven + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + architecture: 'x64' + + - name: Install Build Dependencies + run: | + sudo apt-get update + sudo apt-get install -y git uuid-runtime genisoimage netcat ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 ipmitool libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools + + - name: Env details + run: | + uname -a + whoami + javac -version + mvn -v + python3 --version + free -m + nproc + git status + + - name: Noredist Build + run: | + git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss && cd nonoss && bash -x install-non-oss.sh && cd .. + rm -fr nonoss + mvn -B -P developer,systemvm -Dsimulator -Dnoredist clean install diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..5503ba8df16 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,280 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Simulator CI + +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-20.04 + + strategy: + fail-fast: false + matrix: + tests: [ "smoke/test_accounts + smoke/test_affinity_groups + smoke/test_affinity_groups_projects + smoke/test_annotations + smoke/test_async_job + smoke/test_attach_multiple_volumes + smoke/test_backup_recovery_dummy + smoke/test_certauthority_root + smoke/test_console_endpoint + smoke/test_create_list_domain_account_project + smoke/test_create_network + smoke/test_deploy_vgpu_enabled_vm + smoke/test_deploy_virtio_scsi_vm + smoke/test_deploy_vm_extra_config_data + smoke/test_deploy_vm_iso + smoke/test_deploy_vm_iso_uefi + smoke/test_deploy_vm_root_resize + smoke/test_deploy_vm_with_userdata + smoke/test_deploy_vms_in_parallel + smoke/test_deploy_vms_with_varied_deploymentplanners + smoke/test_diagnostics + smoke/test_direct_download + smoke/test_disk_offerings + smoke/test_disk_provisioning_types + smoke/test_domain_disk_offerings + smoke/test_domain_network_offerings + smoke/test_domain_service_offerings + smoke/test_domain_vpc_offerings", + "smoke/test_dynamicroles + smoke/test_enable_account_settings_for_domain + smoke/test_enable_role_based_users_in_projects + smoke/test_events_resource + smoke/test_global_settings + smoke/test_guest_vlan_range + smoke/test_host_maintenance + smoke/test_hostha_kvm + smoke/test_hostha_simulator + smoke/test_internal_lb + smoke/test_ipv6_infra + smoke/test_iso + smoke/test_kubernetes_clusters + smoke/test_kubernetes_supported_versions + smoke/test_list_ids_parameter + smoke/test_loadbalance + smoke/test_login + smoke/test_metrics_api + smoke/test_migration + smoke/test_multipleips_per_nic + smoke/test_nested_virtualization", + "smoke/test_network + smoke/test_network_acl + smoke/test_network_ipv6 + smoke/test_network_permissions + smoke/test_nic + smoke/test_nic_adapter_type + smoke/test_non_contigiousvlan + smoke/test_outofbandmanagement + smoke/test_outofbandmanagement_nestedplugin + smoke/test_over_provisioning + smoke/test_password_server + smoke/test_persistent_network + smoke/test_portable_publicip + smoke/test_portforwardingrules + smoke/test_primary_storage + smoke/test_privategw_acl + smoke/test_privategw_acl_ovs_gre + smoke/test_projects + smoke/test_public_ip_range + smoke/test_pvlan + smoke/test_regions + smoke/test_register_userdata + smoke/test_reset_configuration_settings + smoke/test_reset_vm_on_reboot + smoke/test_resource_accounting + smoke/test_resource_detail", + "smoke/test_router_dhcphosts + smoke/test_router_dns + smoke/test_router_dnsservice + smoke/test_routers + smoke/test_routers_iptables_default_policy + smoke/test_routers_network_ops + smoke/test_scale_vm + smoke/test_secondary_storage + smoke/test_service_offerings + smoke/test_snapshots + smoke/test_ssvm + smoke/test_staticroles + smoke/test_templates + smoke/test_update_security_group + smoke/test_usage + smoke/test_usage_events + smoke/test_vm_deployment_planner + smoke/test_vm_life_cycle + smoke/test_vm_lifecycle_unmanage_import + smoke/test_vm_snapshot_kvm + smoke/test_vm_snapshots + smoke/test_volumes + smoke/test_vpc_ipv6 + smoke/test_vpc_redundant + smoke/test_vpc_router_nics + smoke/test_vpc_vpn", + "component/find_hosts_for_migration + component/test_acl_isolatednetwork + component/test_acl_isolatednetwork_delete + component/test_acl_listsnapshot", + "component/test_acl_listvm + component/test_acl_listvolume", + "component/test_acl_sharednetwork + component/test_acl_sharednetwork_deployVM-impersonation + component/test_user_private_gateway + component/test_user_shared_network", + "component/test_affinity_groups_projects + component/test_allocation_states + component/test_assign_vm", + "component/test_concurrent_snapshots_limit + component/test_cpu_domain_limits + component/test_cpu_limits + component/test_cpu_max_limits + component/test_cpu_project_limits + component/test_deploy_vm_userdata_multi_nic", + "component/test_egress_fw_rules + component/test_invalid_gw_nm + component/test_ip_reservation", + "component/test_lb_secondary_ip + component/test_list_nics + component/test_list_pod + component/test_memory_limits", + "component/test_mm_domain_limits + component/test_mm_max_limits + component/test_mm_project_limits + component/test_network_offering + component/test_non_contiguous_vlan", + "component/test_persistent_networks + component/test_project_configs + component/test_project_limits + component/test_project_resources", + "component/test_project_usage + component/test_protocol_number_security_group + component/test_public_ip + component/test_resource_limits", + "component/test_regions_accounts + component/test_routers + component/test_snapshots + component/test_stopped_vm + component/test_tags + component/test_templates + component/test_updateResourceCount + component/test_update_vm", + "component/test_volumes + component/test_vpc + component/test_vpc_distributed_routing_offering + component/test_vpc_network + component/test_vpc_offerings + component/test_vpc_routers + component/test_vpn_users", + "component/test_vpc_network_lbrules" ] + + steps: + - uses: actions/checkout@v3 + + - name: Set up JDK + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'adopt' + architecture: x64 + cache: maven + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + architecture: 'x64' + + - name: Install Build Dependencies + run: | + sudo apt-get update + sudo apt-get install -y git uuid-runtime genisoimage netcat ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 ipmitool libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools + + - name: Install Python dependencies + run: | + python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid + + - name: Env details + run: | + uname -a + whoami + javac -version + mvn -v + python3 --version + free -m + nproc + git status + ipmitool -V + + - name: Setup MySQL Server + run: | + # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md#mysql + sudo apt-get install -y mysql-server + sudo systemctl start mysql + sudo mysql -uroot -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY ''; FLUSH PRIVILEGES;" + sudo systemctl restart mysql + sudo mysql -uroot -e "SELECT VERSION();" + + - name: Build with Maven + run: | + mvn -B -P developer,systemvm -Dsimulator clean install -DskipTests=true + + - name: Setup Simulator Prerequisites + run: | + sudo python3 -m pip install --upgrade netaddr mysql-connector-python + python3 -m pip install --user --upgrade tools/marvin/dist/Marvin-*.tar.gz + mvn -q -Pdeveloper -pl developer -Ddeploydb + mvn -q -Pdeveloper -pl developer -Ddeploydb-simulator + + - name: Start CloudStack Management Server with Simulator + run: | + export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/urandom" + echo -e "\nStarting simulator" + set +e + mvn -Dsimulator -Dorg.eclipse.jetty.annotations.maxWait=120 -pl :cloud-client-ui jetty:run 2>&1 > /tmp/jetty-log || true & + while ! nc -vzw 5 localhost 8096 2>&1 > /dev/null; do grep Exception /tmp/jetty-log; sleep 10; done + set -e + echo -e "\nStarting Advanced Zone DataCenter deployment" + python3 tools/marvin/marvin/deployDataCenter.py -i setup/dev/advanced.cfg 2>&1 || true + + - name: Run Integration Tests with Simulator + run: | + mkdir -p integration-test-results/smoke/misc + mkdir -p integration-test-results/component + set -e + TESTS="${{ matrix.tests }}" + echo "Tests in this job: $TESTS" + + TESTS=($(echo $TESTS | tr -d '\n' | tr -s ' ')) + for suite in "${TESTS[@]}" ; do + echo -e "Currently running test: $suite\n" + time nosetests-3.4 --with-xunit --xunit-file=integration-test-results/$suite.xml --with-marvin --marvin-config=setup/dev/advanced.cfg test/integration/$suite.py -s -a tags=advanced,required_hardware=false --zone=Sandbox-simulator --hypervisor=simulator || true ; + done + + echo -e "Stopping Simulator, integration tests run completed\n" + mvn -Dsimulator -pl client jetty:stop 2>&1 + find /tmp//MarvinLogs -type f -exec echo -e "Printing marvin logs {} :\n" \; -exec cat {} \; + + - name: Integration Tests Result + run: | + echo -e "Simulator CI Test Results: (only failures listed)\n" + python3 ./tools/marvin/xunit-reader.py integration-test-results/ diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index b21061fb681..af0dd92c808 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -22,6 +22,10 @@ on: [pull_request, push] permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: build: name: codecov diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index cafe777a90a..7d5a35464df 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -22,6 +22,10 @@ on: [pull_request] permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: build: permissions: diff --git a/.github/workflows/main-coverage.yml b/.github/workflows/main-sonar-check.yml similarity index 98% rename from .github/workflows/main-coverage.yml rename to .github/workflows/main-sonar-check.yml index cd54d7d82a5..897954bb9c0 100644 --- a/.github/workflows/main-coverage.yml +++ b/.github/workflows/main-sonar-check.yml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -name: Main Branch Quality Check +name: Main Branch Sonar Quality Check on: push: diff --git a/.github/workflows/merge-conflict-checker.yml b/.github/workflows/merge-conflict-checker.yml index a0fdb1b7a08..1bf3c6c7d07 100644 --- a/.github/workflows/merge-conflict-checker.yml +++ b/.github/workflows/merge-conflict-checker.yml @@ -24,6 +24,10 @@ on: permissions: # added using https://github.com/step-security/secure-workflows contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: triage: permissions: diff --git a/.github/workflows/rat.yml b/.github/workflows/rat.yml new file mode 100644 index 00000000000..4206480eacb --- /dev/null +++ b/.github/workflows/rat.yml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: RAT License Check + +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'adopt' + architecture: x64 + cache: maven + - name: RAT licence checks + run: | + git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss && cd nonoss && bash -x install-non-oss.sh && cd .. + rm -fr nonoss + mvn -P developer,systemvm -Dsimulator -Dnoredist -pl . org.apache.rat:apache-rat-plugin:0.12:check + - name: Rat Report + if: always() + run: | + echo -e "Printing RAT report\n" + cat target/rat.txt || true diff --git a/.github/workflows/coverage-check.yml b/.github/workflows/sonar-check.yml similarity index 68% rename from .github/workflows/coverage-check.yml rename to .github/workflows/sonar-check.yml index 0a51702f11f..99da2b52f8a 100644 --- a/.github/workflows/coverage-check.yml +++ b/.github/workflows/sonar-check.yml @@ -15,10 +15,14 @@ # specific language governing permissions and limitations # under the License. -name: PR Quality Check +name: Sonar Quality Check on: [pull_request_target] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: build: name: JaCoCo Coverage @@ -29,38 +33,8 @@ jobs: ref: "refs/pull/${{ github.event.number }}/merge" fetch-depth: 0 - - name: Changed UI Files - id: ui-changes - uses: tj-actions/changed-files@a59f800cbb60ed483623848e31be67659a2940f8 - with: - sha: "${{ github.pull_request.sha }}" - base_sha: "${{ github.sha }}" - files: | - ui/* - - - name: Kick UI Build - uses: peter-evans/create-or-update-comment@c9fcb64660bc90ec1cc535646af190c992007c32 - if: steps.ui-changes.outputs.any_changed == 'true' - with: - token: ${{ secrets.ACS_ROBOT_TOKEN }} - issue-number: ${{ github.event.pull_request.number }} - body: | - Found UI changes, kicking a new UI QA build - @blueorangutan ui - - - name: Changed Java Files - id: java-changes - uses: tj-actions/changed-files@a59f800cbb60ed483623848e31be67659a2940f8 - with: - sha: "${{ github.pull_request.sha }}" - base_sha: "${{ github.sha }}" - files: | - **/*.java - **/*.xml - - name: Set up JDK11 uses: actions/setup-java@v3 - if: steps.java-changes.outputs.any_changed == 'true' with: distribution: 'temurin' java-version: '11' @@ -83,7 +57,6 @@ jobs: - name: Run Build and Tests with Coverage id: coverage - if: steps.java-changes.outputs.any_changed == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/tools/travis/script.sh b/.github/workflows/ui.yml old mode 100755 new mode 100644 similarity index 51% rename from tools/travis/script.sh rename to .github/workflows/ui.yml index b900426c8b5..6f1062bb11b --- a/tools/travis/script.sh +++ b/.github/workflows/ui.yml @@ -1,4 +1,3 @@ -#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,22 +14,41 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -# This script should be responsible for bring up the -# test environment and executing the tests. -# -mkdir -p integration-test-results/smoke/misc -mkdir -p integration-test-results/component +name: UI Build -TESTS=($@) -echo "Running tests: " ${TESTS[@]} +on: [push, pull_request] -set -e +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true -for suite in "${TESTS[@]}" ; do - echo "Currently running test: $suite" - time nosetests-3.4 --with-xunit --xunit-file=integration-test-results/$suite.xml --with-marvin --marvin-config=setup/dev/advanced.cfg test/integration/$suite.py -s -a tags=advanced,required_hardware=false --zone=Sandbox-simulator --hypervisor=simulator || true ; -done +jobs: + build: + runs-on: ubuntu-20.04 -python3 ./tools/travis/xunit-reader.py integration-test-results/ + steps: + - uses: actions/checkout@v3 + + - name: Set up Node + uses: actions/setup-node@v3 + with: + node-version: 14 + + - name: Env details + run: | + uname -a + whoami + free -m + nproc + npm version + + - name: Build UI + run: | + cd ui + sudo npm install -g @vue/cli npm-check-updates + rm -f package-lock.json + npm install + npm run build + npm run lint + npm run test:unit diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index eb51ab9ec8b..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,228 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -sudo: required -dist: bionic -group: stable - -language: java - -jdk: - - openjdk11 - -python: - - "3.9" - -node_js: - - 12 - -cache: - directories: - - $HOME/.m2 - timeout: 500 - npm: false - -notifications: - email: false - -env: - global: - - PATH=$HOME/.local/bin:$PATH - matrix: - # Keep the TESTS sorted by name and grouped by type - - TESTS="smoke/test_certauthority_root" - - - TESTS="smoke/test_accounts - smoke/test_affinity_groups - smoke/test_affinity_groups_projects - smoke/test_async_job - smoke/test_backup_recovery_dummy - smoke/test_create_list_domain_account_project - smoke/test_create_network - smoke/test_deploy_vgpu_enabled_vm - smoke/test_deploy_vm_extra_config_data - smoke/test_deploy_vm_iso - smoke/test_deploy_vm_root_resize - smoke/test_deploy_vm_with_userdata - smoke/test_deploy_vms_with_varied_deploymentplanners - smoke/test_diagnostics - smoke/test_disk_offerings - smoke/test_dynamicroles - smoke/test_global_settings - smoke/test_guest_vlan_range - smoke/test_annotations - smoke/test_host_maintenance - smoke/test_hostha_simulator" - - - TESTS="smoke/test_internal_lb - smoke/test_iso - smoke/test_list_ids_parameter - smoke/test_loadbalance - smoke/test_login - smoke/test_metrics_api - smoke/test_multipleips_per_nic - smoke/test_nested_virtualization - smoke/test_network - smoke/test_network_acl - smoke/test_network_permissions - smoke/test_nic - smoke/test_nic_adapter_type - smoke/test_non_contigiousvlan - smoke/test_outofbandmanagement - smoke/test_outofbandmanagement_nestedplugin - smoke/test_over_provisioning - smoke/test_password_server - smoke/test_portable_publicip - smoke/test_portforwardingrules - smoke/test_privategw_acl - smoke/test_privategw_acl_ovs_gre - smoke/test_projects - smoke/test_public_ip_range" - - - TESTS="smoke/test_pvlan - smoke/test_regions - smoke/test_reset_vm_on_reboot - smoke/test_resource_accounting - smoke/test_resource_detail - smoke/test_router_dhcphosts - smoke/test_router_dns - smoke/test_routers - smoke/test_routers_iptables_default_policy - smoke/test_routers_network_ops - smoke/test_scale_vm - smoke/test_secondary_storage - smoke/test_service_offerings - smoke/test_snapshots - smoke/test_ssvm - smoke/test_staticroles - smoke/test_templates - smoke/test_update_security_group - smoke/test_usage - smoke/test_usage_events" - - - TESTS="smoke/test_vm_life_cycle - smoke/test_vm_snapshots - smoke/test_volumes - smoke/test_vpc_redundant - smoke/test_vpc_router_nics - smoke/test_vpc_vpn - smoke/test_primary_storage" - - - TESTS="component/find_hosts_for_migration - component/test_acl_isolatednetwork - component/test_acl_isolatednetwork_delete - component/test_acl_listsnapshot" - - - TESTS="component/test_acl_listvm - component/test_acl_listvolume" - - - TESTS="component/test_acl_sharednetwork - component/test_acl_sharednetwork_deployVM-impersonation - component/test_user_private_gateway - component/test_user_shared_network" - - - TESTS="component/test_affinity_groups_projects - component/test_allocation_states - component/test_assign_vm" - - - TESTS="component/test_concurrent_snapshots_limit - component/test_cpu_domain_limits - component/test_cpu_limits - component/test_cpu_max_limits - component/test_cpu_project_limits - component/test_deploy_vm_userdata_multi_nic" - - - TESTS="component/test_egress_fw_rules - component/test_invalid_gw_nm - component/test_ip_reservation" - - - TESTS="component/test_lb_secondary_ip - component/test_list_nics - component/test_list_pod - component/test_memory_limits" - - - TESTS="component/test_mm_domain_limits - component/test_mm_max_limits - component/test_mm_project_limits - component/test_network_offering - component/test_non_contiguous_vlan" - - - TESTS="component/test_persistent_networks - component/test_project_configs - component/test_project_limits - component/test_project_resources" - - - TESTS="component/test_project_usage - component/test_protocol_number_security_group - component/test_public_ip - component/test_resource_limits" - - - TESTS="component/test_regions_accounts - component/test_routers - component/test_snapshots - component/test_stopped_vm - component/test_tags - component/test_templates - component/test_updateResourceCount - component/test_update_vm" - - - TESTS="component/test_volumes - component/test_vpc - component/test_vpc_distributed_routing_offering - component/test_vpc_network" - - - TESTS="component/test_vpc_network_lbrules - component/test_vpc_offerings - component/test_vpc_routers - component/test_vpn_users" - -before_install: travis_wait 30 ./tools/travis/before_install.sh -install: ./tools/travis/install.sh -before_script: travis_wait 30 ./tools/travis/before_script.sh -script: - - travis_wait 40 ./tools/travis/script.sh ${TESTS} -after_success: ./tools/travis/after_success.sh -after_failure: ./tools/travis/after_failure.sh -after_script: ./tools/travis/after_script.sh - -# Packaging job definition, will be reused -.package_job: &package_job - before_install: docker pull ${IMAGE} - install: true - before_script: true - script: | - docker run \ - --volume ${TRAVIS_BUILD_DIR}:/mnt/build/cloudstack \ - --volume $HOME/.m2:/root/.m2 \ - --rm \ - ${IMAGE} ${PARAMS} - after_script: true - after_success: true - after_failure: true - -#Disable due to intermittent failures -#jobs: -# include: -# - stage: package -# services: docker -# env: IMAGE=khos2ow/cloudstack-rpm-builder:centos7 PARAMS="--distribution centos7 --pack oss" -# <<: *package_job -# -# - stage: package -# services: docker -# env: IMAGE=khos2ow/cloudstack-deb-builder:ubuntu1804 PARAMS="" -# <<: *package_job diff --git a/pom.xml b/pom.xml index 601e660129a..51533139914 100644 --- a/pom.xml +++ b/pom.xml @@ -1025,6 +1025,7 @@ debian/rules debian/source/format dist/console-proxy/js/jquery.js + engine/schema/dist/** plugins/hypervisors/hyperv/conf/agent.properties plugins/hypervisors/hyperv/DotNet/ServerResource/** scripts/installer/windows/acs_license.rtf diff --git a/tools/cli/README b/tools/cli/README deleted file mode 100644 index 0508d0be87d..00000000000 --- a/tools/cli/README +++ /dev/null @@ -1 +0,0 @@ -Moved to https://gitbox.apache.org/repos/asf?p=cloudstack-cloudmonkey.git diff --git a/tools/travis/xunit-reader.py b/tools/marvin/xunit-reader.py similarity index 100% rename from tools/travis/xunit-reader.py rename to tools/marvin/xunit-reader.py diff --git a/tools/travis/after_failure.sh b/tools/travis/after_failure.sh deleted file mode 100755 index 901d06c6e43..00000000000 --- a/tools/travis/after_failure.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# As the filename suggests, this is executed on build failure -# failure. -# -find rat.txt -type f -exec echo -e "Printing RAT license-failures {} :\n" \; -exec cat {} \; -find /tmp//MarvinLogs -type f -exec echo -e "Printing marvin logs {} :\n" \; -exec cat {} \; -find vmops.log -type f -exec echo -e "Printing management server logs {} :\n" \; -exec cat {} \; diff --git a/tools/travis/after_script.sh b/tools/travis/after_script.sh deleted file mode 100755 index 17a9d5356bd..00000000000 --- a/tools/travis/after_script.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# This script should run any tear down commands required. -# - -mvn -Dsimulator -pl client jetty:stop 2>&1 diff --git a/tools/travis/after_success.sh b/tools/travis/after_success.sh deleted file mode 100755 index 67b87454d08..00000000000 --- a/tools/travis/after_success.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# As the filename suggests this is executed on build success. -# diff --git a/tools/travis/before_install.sh b/tools/travis/before_install.sh deleted file mode 100755 index 887206c6f82..00000000000 --- a/tools/travis/before_install.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# This script should be used to install additional dependencies -# This includes: installing ubuntu packages, custom services -# or internet downloads. -# - -echo -e "#### System Information ####" -echo -e "\nO.S. information:" -echo $(uname -a) - -echo -e "\nWho am I:" -whoami - -echo -e "\nJava Version: " -javac -version - -echo -e "\nMaven Version: " -mvn -v - -echo -e "\nPython Version: " -python --version - -echo -e "\nPip Version: " -pip --version - -echo -e "\nDisk Status: " -df - -echo -e "\nMemory Status: " -free - -echo -e "\nTotal CPUs: " -nproc - -echo -e "\nCheck Git status" -git status - -echo -e "\nCleaning up stale files in /tmp: " -sudo find /tmp -type f -mtime +2 | grep -v "`sudo lsof | grep /tmp |awk '{print $9}'|sed -e '1 d' |sort |uniq | tr \\n \|`" | xargs sudo rm -vf - -echo -e "\nUpdating the system: " -sudo apt-get -y clean -sudo apt-get -y update > /dev/null - -echo -e "\nInstalling MySQL: " - -export DEBIAN_FRONTEND=noninteractive -sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password password password' -sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password' -sudo apt-get -q -y install mysql-server > /dev/null - -mysql -uroot -ppassword -e "SET PASSWORD = PASSWORD(''); FLUSH PRIVILEGES;" -sudo service mysql restart - -echo -e "\nInstalling Development tools: " -RETRY_COUNT=3 - -sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1397BC53640DB551 -sudo sh -c 'echo "deb http://mirrors.kernel.org/ubuntu bionic-updates main" >> /etc/apt/sources.list' -sudo apt-get update -q -y > /dev/null -sudo apt-get -q -y -t bionic-updates install openjdk-11-jdk -sudo apt-get -q -y install uuid-runtime genisoimage netcat > /dev/null -if [[ $? -ne 0 ]]; then - echo -e "\napt-get packages failed to install" -fi - -sudo apt-get -q -y -V install freeipmi-common libfreeipmi16 libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 ipmitool libpython-dev libssl-dev libffi-dev python-openssl build-essential --no-install-recommends > /dev/null - -sudo apt-get -y install python3 python3-pip python3-setuptools -sudo apt-get -y install python3-dev # in order to be able to pip3 install pycrypto -# for now we need both: -sudo apt-get -y install python2 -sudo apt-get -y install python2-dev # in order to be able to pip3 install pycrypto -sudo apt-get -y install python2-pip - -echo -e "\nPython 3 version: " -python3 --version -echo -e "\nPython 2 version: " -python2 --version - -echo -e "\nIPMI version" -ipmitool -V - -curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash - -sudo apt-get install -y nodejs - -echo -e "\nNode version" -npm version - -echo " - - - Central - https://repo1.maven.org/maven2 - central - - - -" > ~/.m2/settings.xml - -echo -e "\nChecking PIP Version: " -python3 -m pip --version -echo -e "\nUpgrading PIP if necessary: " -python3 -m pip install --upgrade pip -python3 -m pip --version - -echo -e "\nInstalling some python packages: " - -for ((i=0;i<$RETRY_COUNT;i++)) -do - python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid > /tmp/piplog - python2 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid >> /tmp/piplog - if [[ $? -eq 0 ]]; then - echo -e "\npython packages installed successfully" - break; - fi - echo -e "\npython packages failed to install" - cat /tmp/piplog -done - -echo -e "\nVersion of pip packages:\n" -echo $(pip freeze) diff --git a/tools/travis/before_script.sh b/tools/travis/before_script.sh deleted file mode 100755 index e0a021856e7..00000000000 --- a/tools/travis/before_script.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# This script should be used to bring up the environment. -# -export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/urandom" -echo -e "\nStarting simulator" -mvn -Dsimulator -Dorg.eclipse.jetty.annotations.maxWait=120 -pl :cloud-client-ui jetty:run 2>&1 > /tmp/jetty-log & - -while ! nc -vzw 5 localhost 8096 2>&1 > /dev/null; do grep Exception /tmp/jetty-log; sleep 10; done -echo -e "\nStarting DataCenter deployment" -python3 tools/marvin/marvin/deployDataCenter.py -i setup/dev/advanced.cfg 2>&1 || true diff --git a/tools/travis/install.sh b/tools/travis/install.sh deleted file mode 100755 index 57f4e315dfc..00000000000 --- a/tools/travis/install.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# This should be used to create the build. -# - -export TEST_SEQUENCE_NUMBER=`echo $TRAVIS_JOB_NUMBER | cut -d. -f2` -export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/./urandom" - -set -e - -DIR=$(pwd) - -cd ~ -wget https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -tar zxvf apache-maven-3.6.3-bin.tar.gz -export PATH=`pwd`/apache-maven-3.6.3/bin:$PATH -cd $DIR - -echo -e "\nJDK version" -export JAVA_HOME=$(readlink -f /usr/lib/jvm/java-11-openjdk-amd64/bin/java | sed "s:bin/java::") -mvn -v - -if [ $TEST_SEQUENCE_NUMBER -eq 1 ]; then - # npm lint, test and build - cd ui && npm install && npm run lint && npm run test:unit && npm run build - cd $DIR - -# TODO reinstate this when the systemvm code is fully ported to python3 (or some other processing system) - # Pylint/pep8 systemvm python codebase -# python3 -m pip install --user --upgrade setuptools pycodestyle pylint flask nose -# pip install virtualenv -# sudo virtualenv -p `which python2` py2 -# source py2/bin/activate -# cd $DIR -# which python -# which python2 -# python --version -# python2 -m pip install --user --upgrade setuptools pycodestyle pylint flask nose -# cd systemvm/test && bash -x runtests.sh - - # Build noredist - git clone https://github.com/rhtyd/cloudstack-nonoss.git nonoss && cd nonoss && bash -x install-non-oss.sh - cd $DIR && echo $DIR - git clean -fdx . - # Perform rat checks - mvn -P developer,systemvm -Dsimulator -Dnoredist -pl . org.apache.rat:apache-rat-plugin:0.12:check - mvn -q -B -P developer,systemvm -Dsimulator -Dnoredist clean install -else - mvn -Pdeveloper -Dsimulator clean install -DskipTests=true -T4 -fi - -# Install mysql-connector-python -python3 -m pip install --user --upgrade http://cdn.mysql.com/Downloads/Connector-Python/mysql-connector-python-2.0.4.zip#md5=3df394d89300db95163f17c843ef49df 2>&1 > /dev/null - -# Install marvin -python3 -m pip install --user --upgrade tools/marvin/dist/Marvin-*.tar.gz - -# Deploy the database -mvn -q -Pdeveloper -pl developer -Ddeploydb -mvn -q -Pdeveloper -pl developer -Ddeploydb-simulator diff --git a/ui/README.md b/ui/README.md index b72eedb3389..99b04a3e3aa 100644 --- a/ui/README.md +++ b/ui/README.md @@ -8,15 +8,19 @@ A modern role-based progressive CloudStack UI based on VueJS and Ant Design. Install node: (Debian/Ubuntu) - curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash - + curl -sL https://deb.nodesource.com/setup_14.x | sudo -E bash - sudo apt-get install -y nodejs # Or use distro provided: sudo apt-get install npm nodejs Install node: (CentOS/Fedora/RHEL) - curl -sL https://rpm.nodesource.com/setup_12.x | sudo bash - + curl -sL https://rpm.nodesource.com/setup_14.x | sudo bash - sudo yum install nodejs +Install node: (Mac OS) + + brew install node@16 + Optionally, you may also install system-wide dev tools: sudo npm install -g @vue/cli npm-check-updates @@ -36,6 +40,10 @@ Override the default `CS_URL` to a running CloudStack management server: To configure https, you may use `.env.local.https.example`. +To use the community Simulator QA server you can do this: + + echo "CS_URL=https://qa.cloudstack.cloud" > .env.local + Build and run: npm run serve @@ -45,9 +53,8 @@ Upgrade dependencies to the latest versions: ncu -u -Run Tests: +Run Lint and Unit Tests: - npm run test npm run lint npm run test:unit @@ -125,7 +132,7 @@ The following is tested to work on any Ubuntu 18.04/20.04 base installation or docker container: # Install nodejs (lts) - curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash - + curl -sL https://deb.nodesource.com/setup_14.x | sudo -E bash - sudo apt-get install -y nodejs debhelper rpm # Install build tools npm install -g @vue/cli webpack eslint @@ -135,7 +142,7 @@ docker container: ## Documentation -- VueJS Guide: https://vuejs.org/v2/guide/ +- VueJS Guide: https://vuejs.org/guide/ - Vue Ant Design: https://www.antdv.com/docs/vue/introduce/ - UI Developer [Docs](docs) - JavaScript ES6 Reference: https://www.tutorialspoint.com/es6/ diff --git a/ui/package.json b/ui/package.json index e27ec671fa1..3ceab3df260 100644 --- a/ui/package.json +++ b/ui/package.json @@ -60,9 +60,9 @@ "vue-i18n": "^9.1.6", "vue-loader": "^16.2.0", "vue-router": "^4.0.14", + "vue-uuid": "^3.0.0", "vue-web-storage": "^6.1.0", "vue3-clipboard": "^1.0.0", - "vue-uuid": "^3.0.0", "vuedraggable": "^4.0.3", "vuex": "^4.0.0-0" },