vyos/vyos-build
1
0
Fork 0
mirror of https://github.com/vyos/vyos-build.git synced 2025-10-01 20:28:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Testsuite: add RAID-1 test case for disk recovery

Browse Source 
(cherry picked from commit 986fd18e11ac19470e17ad786d5cc908339ccb43)
This commit is contained in:
Christian Poessinger 2021-12-25 22:01:46 +01:00
parent 35d432c281
commit 67a6f3e3c9
1 changed files with 128 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
scripts/check-qemu-install
View File

@ -42,6 +42,7 @@ import random
import traceback
import logging
import re
import json
from io import BytesIO
from io import StringIO
@ -65,13 +66,18 @@ parser.add_argument('--logfile', help='Log to file')
parser.add_argument('--uefi', help='Boot using UEFI', action='store_true', default=False)
parser.add_argument('--raid', help='Perform a RAID-1 install', action='store_true', default=False)
parser.add_argument('--no-kvm', help='Disable use of kvm', action='store_true', default=False)
parser.add_argument('--configd', help='Execute testsuite with config daemon',
action='store_true', default=False)
parser.add_argument('--configd', help='Execute testsuite with config daemon', action='store_true',
default=False)
parser.add_argument('--no-interfaces', help='Execute testsuite without interface tests to save time',
action='store_true', default=False)
parser.add_argument('--configtest', help='Execute load/commit config tests',
action='store_true', default=False)
args = parser.parse_args()
with open('data/defaults.json') as f:
vyos_defaults = json.load(f)
class StreamToLogger(object):
"""
Fake file-like stream object that redirects writes to a logger instance.
@ -119,7 +125,7 @@ def get_qemu_cmd(name, enable_kvm, enable_uefi, disk_img, raid=None, iso_img=Non
cdrom = ""
if iso_img:
cdrom = "-boot d -cdrom {}".format(iso_img)
cdrom = f"-boot d -cdrom {iso_img}"
# test using half of the available CPUs on the system
cpucount = get_half_cpus()
@ -143,10 +149,13 @@ def get_qemu_cmd(name, enable_kvm, enable_uefi, disk_img, raid=None, iso_img=Non
-machine accel=kvm \
-uuid {uuid} \
-nographic {cpu} {cdrom} {kvm} \
-drive format=raw,file={disk_img}'
-device virtio-scsi-pci,id=scsi0 \
-drive format=raw,file={disk_img},if=none,media=disk,id=drive-hd1,readonly=off \
-device scsi-hd,bus=scsi0.0,drive=drive-hd1,id=hd1,bootindex=1'
if raid:
cmd += f' -drive format=raw,file={raid}'
cmd += f' -drive format=raw,file={raid},if=none,media=disk,id=drive-hd2,readonly=off \
-device scsi-hd,bus=scsi0.0,drive=drive-hd2,id=hd2,bootindex=2'
return cmd
@ -157,9 +166,7 @@ def shutdownVM(c, log, message=''):
if message:
log.info(message)
c.sendline('poweroff')
c.expect(r'\nAre you sure you want to poweroff this system.*\]')
c.sendline('Y')
c.sendline('poweroff now')
log.info('Shutting down virtual machine')
for i in range(30):
log.info('Waiting for shutdown...')
@ -243,6 +250,7 @@ if args.raid:
# must be called after the raid disk as args.disk name is altered in the RAID path
gen_disk(args.disk)
test_timeout = 3 *3600 # 3 hours (in seconds)
try:
#################################################
# Installing image to disk
@ -341,7 +349,7 @@ try:
# Always load the WiFi simulation module
################################################
c.sendline('sudo modprobe mac80211_hwsim')
c.expect(r'vyos@vyos:~\$')
c.expect(op_mode_prompt)
#################################################
# Start/stop config daemon
@ -350,35 +358,116 @@ try:
c.sendline('sudo systemctl start vyos-configd.service &> /dev/null')
else:
c.sendline('sudo systemctl stop vyos-configd.service &> /dev/null')
c.expect(r'vyos@vyos:~\$')
c.expect(op_mode_prompt)
#################################################
# Basic Configmode/Opmode switch
#################################################
log.info('Basic CLI configuration mode test')
c.sendline('configure')
c.expect(r'vyos@vyos#')
c.expect(cfg_mode_prompt)
c.sendline('exit')
c.expect(r'vyos@vyos:~\$')
c.expect(op_mode_prompt)
c.sendline('show version')
c.expect(r'vyos@vyos:~\$')
c.expect(op_mode_prompt)
c.sendline('show version kernel')
c.expect(f'{vyos_defaults["kernel_version"]}-{vyos_defaults["kernel_flavor"]}')
c.expect(op_mode_prompt)
c.sendline('show version frr')
c.expect(op_mode_prompt)
c.sendline('show interfaces')
c.expect(r'vyos@vyos:~\$')
if args.raid:
c.sendline('cat /proc/mdstat')
c.expect(op_mode_prompt)
c.expect(op_mode_prompt)
#################################################
# Executing test-suite
#################################################
if args.raid:
# Verify RAID subsystem - by deleting a disk and re-create the array
# from scratch
c.sendline('cat /proc/mdstat')
c.expect(op_mode_prompt)
shutdownVM(c, log, f'Shutdown VM and start with empty RAID member "{args.disk}"')
if os.path.exists(args.disk):
os.unlink(args.disk)
gen_disk(args.disk)
#################################################
# Booting RAID-1 system with one missing disk
#################################################
log.info('Booting RAID-1 system')
cmd = get_qemu_cmd('TESTVM', kvm, args.uefi, args.disk, diskname_raid)
# We need to swap boot indexes to boot from second harddisk so we can
# recreate the RAID on the first disk
cmd = cmd.replace('bootindex=1', 'bootindex=X')
cmd = cmd.replace('bootindex=2', 'bootindex=1')
cmd = cmd.replace('bootindex=X', 'bootindex=2')
log.debug(f'Executing command: {cmd}')
c = pexpect.spawn(cmd, logfile=stl)
#################################################
# Logging into VyOS system
#################################################
try:
c.expect('The highlighted entry will be executed automatically in', timeout=10)
c.sendline('')
except pexpect.TIMEOUT:
log.warning('Did not find GRUB countdown window, ignoring')
loginVM(c, log)
c.sendline('cat /proc/mdstat')
c.expect(op_mode_prompt)
log.info('Re-format new RAID member')
c.sendline('format disk sda like sdb')
c.sendline('yes')
c.expect(op_mode_prompt)
log.info('Add member to RAID1 (md0)')
c.sendline('add raid md0 member sda1')
c.expect(op_mode_prompt)
log.info('Now we need to wait for re-sync to complete')
start_time = time.time()
timeout = 60
while True:
if (start_time + timeout) < time.time():
break
c.sendline('cat /proc/mdstat')
c.expect(op_mode_prompt)
time.sleep(20)
# Reboot system with new primary RAID1 disk
shutdownVM(c, log, f'Shutdown VM and start from recovered RAID member "{args.disk}"')
log.info('Booting RAID-1 system')
cmd = get_qemu_cmd('TESTVM', kvm, args.uefi, args.disk, diskname_raid)
log.debug(f'Executing command: {cmd}')
c = pexpect.spawn(cmd, logfile=stl)
loginVM(c, log)
c.sendline('cat /proc/mdstat')
c.expect(op_mode_prompt)
elif not args.configtest:
# run default smoketest suite
if args.no_interfaces:
# remove interface tests as they consume a lot of time
c.sendline('sudo rm -f /usr/libexec/vyos/tests/smoke/cli/test_interfaces_*')
c.expect(op_mode_prompt)
# run default smoketest suite
if not args.raid and not args.configtest:
log.info('Executing VyOS smoketests')
c.sendline('/usr/bin/vyos-smoketest')
i = c.expect(['\n +Invalid command:', '\n +Set failed',
'No such file or directory', r'\n\S+@\S+[$#]'], timeout=7200)
'No such file or directory', r'\n\S+@\S+[$#]'], timeout=test_timeout)
if i == 0:
raise Exception('Invalid command detected')
@ -399,10 +488,14 @@ try:
raise Exception("Smoketest-failed, please look into debug output")
# else, run configtest suite
elif not args.raid:
log.info('Generating a WireGuard default keypair')
c.sendline('generate wireguard default-keypair')
c.expect(r'vyos@vyos:~\$')
else:
log.info('Adding a legacy WireGuard default keypair for migrations')
c.sendline('sudo mkdir -p /config/auth/wireguard/default')
c.expect(op_mode_prompt)
c.sendline('echo "aGx+fvW916Ej7QRnBbW3QMoldhNv1u95/WHz45zDmF0=" | sudo tee /config/auth/wireguard/default/private.key')
c.expect(op_mode_prompt)
c.sendline('echo "x39C77eavJNpvYbNzPSG3n1D68rHYei6q3AEBEyL1z8=" | sudo tee /config/auth/wireguard/default/public.key')
c.expect(op_mode_prompt)
log.info('Generating some OpenVPN keys')
subject = '/C=DE/ST=BY/O=VyOS/localityName=Cloud/commonName=vyos/' \
@ -416,28 +509,28 @@ try:
c.sendline(f'openssl req -newkey rsa:4096 -new -nodes -x509 -days 3650 '\
f'-keyout {ssl_key} -out {ssl_cert} -subj {subject}')
c.expect(r'vyos@vyos:~\$', timeout=600)
c.expect(op_mode_prompt, timeout=600)
c.sendline(f'openssl req -new -x509 -key {ssl_key} -out {ca_cert} -subj {subject}')
c.expect(r'vyos@vyos:~\$', timeout=600)
c.expect(op_mode_prompt, timeout=600)
c.sendline(f'openssl dhparam -out {dh_pem} 2048')
c.expect(r'vyos@vyos:~\$', timeout=600)
c.sendline(f'generate openvpn key {s2s_key}')
c.expect(r'vyos@vyos:~\$')
c.sendline(f'generate openvpn key {auth_key}')
c.expect(r'vyos@vyos:~\$')
c.expect(op_mode_prompt, timeout=600)
c.sendline(f'openvpn --genkey secret {s2s_key}')
c.expect(op_mode_prompt)
c.sendline(f'openvpn --genkey secret {auth_key}')
c.expect(op_mode_prompt)
script_file = '/config/scripts/vyos-foo-update.script'
c.sendline(f'echo "#!/bin/sh" > {script_file}; chmod 775 {script_file}')
c.expect(r'vyos@vyos:~\$')
c.expect(op_mode_prompt)
for file in [ca_cert, ssl_cert, ssl_key, dh_pem, s2s_key, auth_key]:
c.sendline(f'sudo chown openvpn:openvpn {file}')
c.expect(r'vyos@vyos:~\$')
c.expect(op_mode_prompt)
log.info('Executing load config tests')
c.sendline('/usr/bin/vyos-configtest')
i = c.expect(['\n +Invalid command:', 'No such file or directory',
r'\n\S+@\S+[$#]'], timeout=3600)
r'\n\S+@\S+[$#]'], timeout=test_timeout)
if i==0:
raise Exception('Invalid command detected')