diff --git a/virttest/vt_utils/block.py b/virttest/vt_utils/block.py index 9fc73e56b5..78a191bc3b 100644 --- a/virttest/vt_utils/block.py +++ b/virttest/vt_utils/block.py @@ -24,14 +24,14 @@ PARTITION_TYPE_PRIMARY = "primary" -def get_disks(partition=False): +def get_disks_info(has_partition=False): """ List all disks or disks with no partition. - :param partition: if true, list all disks; otherwise, - list only disks with no partition - :type partition: boolean - :return: the disks info.( e.g. {kname: [kname, size, type, serial, wwn]} ) + :param has_partition: If true, list disks info; + otherwise, list disks info which do NOT have partition. + :type has_partition: Boolean + :return: The disks info.( e.g. {kname: [kname, size, type, serial, wwn]} ) :rtype: dict """ disks_dict = {} @@ -40,24 +40,30 @@ def get_disks(partition=False): if platform.machine() == "s390x": driver = "css0" block_info = process.run( - 'ls /sys/dev/block -l | grep "/%s"' % driver, verbose=False + 'ls /sys/dev/block -l | grep "/%s"' % driver, + verbose=False, + shell=True, ).stdout_text for matched in re.finditer(r"/block/(\S+)\s^", block_info, re.M): knames = matched.group(1).split("/") if len(knames) == 2: parent_disks.add(knames[0]) - if partition is False and knames[0] in parent_disks: + if has_partition is False and knames[0] in parent_disks: if knames[0] in disks_dict: del disks_dict[knames[0]] continue disks_dict[knames[-1]] = [knames[-1]] o = process.run( - 'lsblk -o KNAME,SIZE | grep "%s "' % knames[-1], verbose=False + 'lsblk -o KNAME,SIZE | grep "%s "' % knames[-1], + verbose=False, + shell=True, ).stdout_text disks_dict[knames[-1]].append(o.split()[-1]) o = process.run( - "udevadm info -q all -n %s" % knames[-1], verbose=False + "udevadm info -q all -n %s" % knames[-1], + verbose=False, + shell=True, ).stdout_text for parttern in ( r"DEVTYPE=(\w+)\s^", @@ -69,6 +75,26 @@ def get_disks(partition=False): return disks_dict +def get_disks_path(partition_path=False): + """ + List disk path in Linux host. + + :param partition_path: If true, list disk name including partition; otherwise, + list disk name excluding partition. + :type partition_path: Boolean + + :return: The disks path in set(). ( e.g. {'/dev/sda2', '/dev/sda1', '/dev/sda'} or {'/dev/sda'} ) + :rtype: Set + """ + cmd = "ls /dev/[vhs]d*" + if not partition_path: + cmd = "%s | grep -v [0-9]$" % cmd + status, output = process.getstatusoutput(cmd, shell=True) + if status != 0: + raise RuntimeError("Get disks failed with output %s" % output) + return set(output.split()) + + def create_partition(did, size, start, part_type=PARTITION_TYPE_PRIMARY, timeout=360): """ Create single partition on disk. @@ -175,14 +201,18 @@ def get_disk_size(did): :param did: disk kname. e.g. 'sdb', 'sdc' :return: disk size. """ - disks_info = get_disks(partition=True) + disks_info = get_disks_info(True) disk_size = disks_info["%s" % did][1] return int(utils_numeric.normalize_data_size(disk_size, "B").split(".")[0]) -def get_partions_list(): +def get_partitions_list(): """ - Get all partition lists. + Get all partition list. + + :return: All partition list. + e.g. ['sda', 'sda1', 'sda2', 'dm-0', 'dm-1', 'dm-2'] + :rtype: List """ parts_cmd = "cat /proc/partitions" parts_out = process.run(parts_cmd, verbose=False).stdout_text @@ -201,10 +231,12 @@ def get_disk_by_serial(serial_str): """ Get disk by serial in host. - :param serial_str: ID_SERIAL of disk, string value - :return: Disk name if find one with serial_str, else None + :param serial_str: ID_SERIAL of disk, string value. + :type serial_str: String + :return: Disk name if find one with serial_str, else None. + :rtype: String """ - parts_list = get_partions_list() + parts_list = get_partitions_list() for disk in parts_list: cmd = "udevadm info --query=all --name=/dev/{} | grep ID_SERIAL={}".format( disk, serial_str @@ -215,3 +247,30 @@ def get_disk_by_serial(serial_str): if not status: return disk return None + + +def get_drive_path(did, timeout=120): + """ + Get drive path( devname ) on host by drive serial or wwn + + :param did: A drive serial( ID_SERIAL or ID_SERIAL_SHORT ) + or a wwn( ID_WWN ). + :type did: String + :param timeout: Time out. + :type timeout: Integer + + :return: A drive path( devname ) + :rtype: String + + :raises: An RuntimeError will be raised when cmd exit code is NOT 0. + """ + cmd = "for dev_path in `ls -d /sys/block/*`; do " + cmd += "echo `udevadm info -q property -p $dev_path`; done" + status, output = process.getstatusoutput(cmd, timeout=timeout) + if status != 0: + raise RuntimeError("Command running was failed. Output: %s" % output) + p = r"DEVNAME=([^\s]+)\s.*(?:ID_SERIAL|ID_SERIAL_SHORT|ID_WWN)=%s" % did + dev = re.search(p, output, re.M) + if dev: + return dev.groups()[0] + return "" diff --git a/virttest/vt_utils/filesystem.py b/virttest/vt_utils/filesystem.py index d37fbb1ea2..3e18aeee46 100644 --- a/virttest/vt_utils/filesystem.py +++ b/virttest/vt_utils/filesystem.py @@ -13,6 +13,7 @@ # # Copyright: Red Hat (c) 2023 and Avocado contributors # Author: Houqi Zuo +import os import re from avocado.utils import process @@ -121,11 +122,11 @@ def umount(src, dst, fstype=None): return True -def create_filesyetem(partition_name, fstype, timeout=360): +def create_filesyetem(partition, fstype, timeout=360): """ create file system. - :param partition_name: partition name that to be formatted. e.g. sdb1 + :param partition: partition name that to be formatted. e.g. /dev/sdb1 :param fstype: filesystem type for the disk. :param timeout: Timeout for cmd execution in seconds. """ @@ -133,8 +134,8 @@ def create_filesyetem(partition_name, fstype, timeout=360): mkfs_cmd = "mkfs.%s -f" % fstype else: mkfs_cmd = "mkfs.%s -F" % fstype - format_cmd = "yes|%s '/dev/%s'" % (mkfs_cmd, partition_name) - process.system(format_cmd, timeout=timeout, verbose=False) + format_cmd = "yes|%s '%s'" % (mkfs_cmd, partition) + process.run(format_cmd, timeout=timeout, verbose=False) def resize_filesystem(partition, size): @@ -147,7 +148,7 @@ def resize_filesystem(partition, size): :param size: resize file system to size. size unit can be 'B', 'K', 'M', 'G'. support transfer size with SIZE_AVAILABLE, - enlarge to maximun available size. + enlarge to maximum available size. """ def get_start_size(): @@ -193,7 +194,7 @@ def resize_ext_fs(size): ) * bsize size = utils_numeric.normalize_data_size(str(size).split(".")[0], "K") resize_fs_cmd = "resize2fs %s %sK" % (partition, int(size.split(".")[0])) - process.system(resize_fs_cmd, verbose=False) + process.run(resize_fs_cmd, verbose=False) if flag: mount(partition, mountpoint, fstype=fstype) @@ -216,3 +217,65 @@ def get_mpoint_fstype(partition): mount_list = process.run("cat /proc/mounts", verbose=False).stdout_text mount_info = re.search(r"%s\s(.+?)\s(.+?)\s" % partition, mount_list) return mount_info.groups() + + +def format_disk( + did, + all_disks_did, + partition=False, + mountpoint=None, + size=None, + fstype="ext3", +): + """ + Create a partition on disk in Linux host and format and mount it. + + :param did: Disk kname, serial or wwn. + :type did: String + :param all_disks_did: All disks did lists each include disk kname, + serial and wwn. + :type all_disks_did: List + :param partition: If true, can format all disks; otherwise, + only format the ones with no partition originally. + :type partition: Boolean + :param mountpoint: Mount point for the disk. + :type mountpoint: String + :param size: Partition size( such as 6G, 500M ). + :type size: String + :param fstype: Filesystem type for the disk. + :type fstype: String + + :return: If disk is usable, return True. Otherwise, return False. + :rtype: Boolean + """ + disks = block.get_disks_path(partition) + for line in disks: + kname = line.split("/")[-1] + did_list = all_disks_did[kname] + if did not in did_list: + # Continue to search target disk + continue + if not size: + size_output = process.run( + "lsblk -o KNAME,SIZE|grep %s" % kname, + verbose=False, + shell=True, + ).stdout_text + size = size_output.splitlines()[0].split()[1] + all_disks_before = block.get_disks_path(True) + devname = line + block.create_partition( + devname.split("/")[-1], + size, + "0M", + ) + all_disks_after = block.get_disks_path(True) + partname = (all_disks_after - all_disks_before).pop() + create_filesyetem(partname, fstype) + if not mountpoint: + process.run("mkdir /mnt/%s" % kname) + mountpoint = os.path.join("/mnt", kname) + mount(src=partname, dst=mountpoint, fstype=fstype) + if is_mounted(src=partname, dst=mountpoint, fstype=fstype): + return True + return False diff --git a/virttest/vt_utils/interrupted_thread.py b/virttest/vt_utils/interrupted_thread.py new file mode 100644 index 0000000000..72f4240f5c --- /dev/null +++ b/virttest/vt_utils/interrupted_thread.py @@ -0,0 +1,90 @@ +# +# Library for interrupted thread related helper functions +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; specifically version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat (c) 2024 and Avocado contributors +# Author: Houqi Zuo +import sys +import threading + +from virttest import error_context + + +class InterruptedThread(threading.Thread): + """ + Run a function in a background thread. + """ + + def __init__(self, target, args=(), kwargs={}): + """ + Initialize the instance. + + :param target: Function to run in the thread. + :type target: Function object + :param args: Arguments to pass to target. + :type args: Tuple + :param kwargs: Keyword arguments to pass to target. + :type kwargs: Dictionary + """ + threading.Thread.__init__(self) + self._target = target + self._args = args + self._kwargs = kwargs + + def run(self): + """ + Run target (passed to the constructor). No point in calling this + function directly. Call start() to make this function run in a new + thread. + + :raises: An Exception from run() of threading.Thread. + """ + self._e = None + self._retval = None + try: + try: + self._retval = self._target(*self._args, **self._kwargs) + except Exception: + self._e = sys.exc_info() + raise + finally: + # Avoid circular references (start() may be called only once so + # it's OK to delete these) + del self._target, self._args, self._kwargs + + def join(self, timeout=None, suppress_exception=False): + """ + Join the thread. If target raised an exception, re-raise it. + Otherwise, return the value returned by target. + + :param timeout: Timeout value to pass to threading.Thread.join(). + :type timeout: Integer + :param suppress_exception: If True, don't re-raise the exception. + :type suppress_exception: Boolean + """ + threading.Thread.join(self, timeout) + try: + if self._e: + if not suppress_exception: + # Because the exception was raised in another thread, we + # need to explicitly insert the current context into it + s = error_context.exception_context(self._e[1]) + s = error_context.join_contexts(error_context.get_context(), s) + error_context.set_exception_context(self._e[1], s) + raise self._e.with_traceback(*self._e) + else: + return self._retval + finally: + # Avoid circular references (join() may be called multiple times + # so we can't delete these) + self._e = None + self._retval = None diff --git a/virttest/vt_utils/memory.py b/virttest/vt_utils/memory.py new file mode 100644 index 0000000000..b2774aeb25 --- /dev/null +++ b/virttest/vt_utils/memory.py @@ -0,0 +1,95 @@ +# +# Library for memory related helper functions +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; specifically version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat (c) 2024 and Avocado contributors +# Author: Houqi Zuo +import math +import re + +from avocado.utils import memory, process + +from virttest import utils_numeric + + +def get_usable_memory_size(align=None): + """ + Sync, then drop host caches, then return host free memory size. + + :param align: MB use to align free memory size. + :type align: Integer + :return: host free memory size in MB. + :rtype: Float + """ + memory.drop_caches() + usable_mem = memory.read_from_meminfo("MemFree") + usable_mem = float(utils_numeric.normalize_data_size("%s KB" % usable_mem)) + if align: + usable_mem = math.floor(usable_mem / align) * align + return usable_mem + + +def get_mem_info(attr="MemTotal"): + """ + Get memory information attributes in Linux host. + + :param attr: Memory information attribute. + :type attr: String + + :return: Memory information of attribute in kB. + :rtype: Integer + """ + cmd = "grep '%s:' /proc/meminfo" % attr + output = process.run(cmd, shell=True).stdout_text + output = re.findall(r"\d+\s\w", output)[0] + output = float(utils_numeric.normalize_data_size(output, order_magnitude="K")) + return int(output) + + +def get_used_mem(): + """ + Get Used memory for Linux. + + :return: Used space memory in M-bytes. + :rtype: Integer + """ + cmd = "free -m | grep 'Mem'" + pattern = r"Mem:\s+(\d+)\s+(\d+)\s+" + output = process.run(cmd, shell=True).stdout_text + match = re.search(pattern, output, re.M | re.I) + used = "%sM" % "".join(match.group(2).split(",")) + used = float(utils_numeric.normalize_data_size(used, order_magnitude="M")) + return int(used) + + +def get_cache_mem(): + """ + Get cache memory for Linux. + + :return: Memory cache in M-bytes. + :rtype: Integer + """ + cache = "%s kB" % (get_mem_info("Cached") + get_mem_info("Buffers")) + cache = float(utils_numeric.normalize_data_size(cache, order_magnitude="M")) + return int(cache) + + +def get_free_mem(): + """ + Get Free memory for Linux. + + :return: Free space memory in M-bytes. + :rtype: Integer + """ + free = "%s kB" % get_mem_info("MemFree") + free = float(utils_numeric.normalize_data_size(free, order_magnitude="M")) + return int(free) diff --git a/virttest/vt_utils/pci_utils.py b/virttest/vt_utils/pci_utils.py new file mode 100644 index 0000000000..128b6f9c57 --- /dev/null +++ b/virttest/vt_utils/pci_utils.py @@ -0,0 +1,100 @@ +# +# Library for pci device related helper functions +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; specifically version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat (c) 2024 and Avocado contributors +# Author: Houqi Zuo +import re + +from avocado.utils import process + + +def get_full_pci_id(pci_id): + """ + Get full PCI ID of pci_id. + + :param pci_id: PCI ID of a device. + :type pci_id: String + + :return: A full PCI ID. If exception happens, return None + :rtype: String + """ + cmd = "lspci -D | awk '/%s/ {print $1}'" % pci_id + try: + return process.run(cmd, shell=True).stdout_text.strip() + except process.CmdError: + return None + + +def get_pci_id_using_filter(pci_filter): + """ + Get PCI ID from pci filter in host. + + :param pci_filter: PCI filter regex of a device (adapter name) + :type pci_filter: string + + :return: list of pci ids with adapter name regex + :rtype: List + """ + cmd = "lspci | grep -F '%s' | awk '{print $1}'" % pci_filter + cmd_obj = process.run(cmd, shell=True) + status, output = cmd_obj.exit_status, cmd_obj.stdout_text.strip() + if status != 0 or not output: + return [] + return output.split() + + +def get_interface_from_pci_id(pci_id, nic_regex=""): + """ + Get interface from pci id in host. + + :param pci_id: PCI id of the interface to be identified. + :type pci_id: String + :param nic_regex: regex to match nic interfaces. + :type nic_regex: String + + :return: interface name associated with the pci id. + :rtype: String + """ + if not nic_regex: + nic_regex = "\w+(?=: flags)|\w+(?=\s*Link)" + cmd = "ifconfig -a" + cmd_obj = process.run(cmd, shell=True) + status, output = cmd_obj.exit_status, cmd_obj.stdout_text.strip() + if status != 0: + return None + ethnames = re.findall(nic_regex, output) + for each_interface in ethnames: + cmd = "ethtool -i %s | awk '/bus-info/ {print $2}'" % each_interface + cmd_obj = process.run(cmd, shell=True) + status, output = cmd_obj.exit_status, cmd_obj.stdout_text.strip() + if status: + continue + if pci_id in output: + return each_interface + return None + + +def get_vendor_from_pci_id(pci_id): + """ + Check out the device vendor ID according to pci_id. + + :param pci_id: PCI ID of a device. + :type pci_id: String + + :return: The device vendor ID. + :rtype: String + """ + cmd = "lspci -n | awk '/%s/ {print $3}'" % pci_id + return re.sub( + ":", " ", process.run(cmd, shell=True, ignore_status=True).stdout_text + ) diff --git a/virttest/vt_utils/qemu_utils.py b/virttest/vt_utils/qemu_utils.py new file mode 100644 index 0000000000..2a3f7a1172 --- /dev/null +++ b/virttest/vt_utils/qemu_utils.py @@ -0,0 +1,68 @@ +# +# Library for qemu option related helper functions +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; specifically version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat (c) 2024 and Avocado contributors +# Author: Houqi Zuo +import re + +from avocado.utils import process + + +def has_option(option, qemu_path="/usr/bin/qemu-kvm"): + """ + Helper function for command line option wrappers. + + :param option: Option need check. + :type option: String + :param qemu_path: Path for qemu-kvm. + :type option: String + + :return: Return true if the qemu has the given option. Otherwise, return + false. + :rtype: Boolean + """ + hlp = process.run( + "%s -help" % qemu_path, shell=True, ignore_status=True, verbose=False + ).stdout_text + return bool(re.search(r"^-%s(\s|$)" % option, hlp, re.MULTILINE)) + + +def get_support_machine_type(qemu_binary="/usr/libexec/qemu-kvm", remove_alias=False): + """ + Get each machine types supported by host. + + :param qemu_binary: qemu-kvm binary file path. + :type qemu_binary: String + :param remove_alias: If it's True, remove alias or not. Otherwise, do NOT + remove alias. + :type remove_alias: Boolean + + :return: A tuple(machine_name, machine_type, machine_alias). + :rtype: Tuple[List, List, List] + """ + o = process.run("%s -M ?" % qemu_binary).stdout_text.splitlines() + machine_name = [] + machine_type = [] + machine_alias = [] + split_pattern = re.compile( + r"^(\S+)\s+(.*?)(?: (\((?:alias|default|deprecated).*))?$" + ) + for item in o[1:]: + if "none" in item: + continue + machine_list = split_pattern.search(item).groups() + machine_name.append(machine_list[0]) + machine_type.append(machine_list[1]) + val = None if remove_alias else machine_list[2] + machine_alias.append(val) + return machine_name, machine_type, machine_alias