diff --git a/.buildkite/pipeline_cpu_template.py b/.buildkite/pipeline_cpu_template.py index 2de782320c1..72a67ea5e7d 100755 --- a/.buildkite/pipeline_cpu_template.py +++ b/.buildkite/pipeline_cpu_template.py @@ -23,7 +23,7 @@ class BkStep(str, Enum): cpu_template_test = { "rdmsr": { BkStep.COMMAND: [ - "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features.py -k 'test_cpu_rdmsr' " + "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features_x86_64.py -k 'test_cpu_rdmsr' " ], BkStep.LABEL: "📖 rdmsr", "instances": ["c5n.metal", "m5n.metal", "m6a.metal", "m6i.metal"], @@ -40,7 +40,7 @@ class BkStep(str, Enum): "cpuid_wrmsr": { "snapshot": { BkStep.COMMAND: [ - "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features.py -k 'test_cpu_wrmsr_snapshot or test_cpu_cpuid_snapshot'", + "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features_x86_64.py -k 'test_cpu_wrmsr_snapshot or test_cpu_cpuid_snapshot'", "mkdir -pv tests/snapshot_artifacts_upload/{instance}_{os}_{kv}", "sudo mv tests/snapshot_artifacts/* tests/snapshot_artifacts_upload/{instance}_{os}_{kv}", ], @@ -52,7 +52,7 @@ class BkStep(str, Enum): BkStep.COMMAND: [ "buildkite-agent artifact download tests/snapshot_artifacts_upload/{instance}_{os}_{kv}/**/* .", "mv tests/snapshot_artifacts_upload/{instance}_{os}_{kv} tests/snapshot_artifacts", - "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features.py -k 'test_cpu_wrmsr_restore or test_cpu_cpuid_restore'", + "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features_x86_64.py -k 'test_cpu_wrmsr_restore or test_cpu_cpuid_restore'", ], BkStep.LABEL: "📸 load snapshot artifacts created on {instance} {snapshot_os} {snapshot_kv} to {restore_instance} {restore_os} {restore_kv}", BkStep.TIMEOUT: 30, diff --git a/tests/framework/utils_cpuid.py b/tests/framework/utils_cpuid.py index 65e953f43ac..4303e3ba967 100644 --- a/tests/framework/utils_cpuid.py +++ b/tests/framework/utils_cpuid.py @@ -10,6 +10,8 @@ from framework.utils import check_output from framework.utils_imdsv2 import imdsv2_get +CPU_FEATURES_CMD = r"lscpu |grep -oP '^Flags:\s+\K.+'" + class CpuVendor(Enum): """CPU vendors enum.""" diff --git a/tests/integration_tests/functional/test_cpu_features_aarch64.py b/tests/integration_tests/functional/test_cpu_features_aarch64.py index a1066da21a5..3bfa2357d8b 100644 --- a/tests/integration_tests/functional/test_cpu_features_aarch64.py +++ b/tests/integration_tests/functional/test_cpu_features_aarch64.py @@ -2,13 +2,15 @@ # SPDX-License-Identifier: Apache-2.0 """Tests for the CPU features for aarch64.""" +import os import platform import re import pytest import framework.utils_cpuid as cpuid_utils -from framework.utils_cpuid import CpuModel +from framework import utils +from framework.utils_cpuid import CPU_FEATURES_CMD, CpuModel PLATFORM = platform.machine() @@ -48,7 +50,7 @@ def _check_cpu_features_arm(test_microvm, guest_kv, template_name=None): case CpuModel.ARM_NEOVERSE_V1, _, None: expected_cpu_features = DEFAULT_G3_FEATURES_5_10 - _, stdout, _ = test_microvm.ssh.check_output(r"lscpu |grep -oP '^Flags:\s+\K.+'") + _, stdout, _ = test_microvm.ssh.check_output(CPU_FEATURES_CMD) flags = set(stdout.strip().split(" ")) assert flags == expected_cpu_features @@ -63,6 +65,46 @@ def get_cpu_template_dir(cpu_template): return cpu_template if cpu_template else "none" +@pytest.mark.skipif( + PLATFORM != "aarch64", + reason="This is aarch64 specific test.", +) +def test_host_vs_guest_cpu_features_aarch64(uvm_nano): + """Check CPU features host vs guest""" + + vm = uvm_nano + vm.add_net_iface() + vm.start() + host_feats = set(utils.check_output(CPU_FEATURES_CMD).stdout.strip().split(" ")) + guest_feats = set(vm.ssh.check_output(CPU_FEATURES_CMD).stdout.strip().split(" ")) + + cpu_model = cpuid_utils.get_cpu_model_name() + match cpu_model: + case CpuModel.ARM_NEOVERSE_N1: + assert host_feats - guest_feats == set() + # Kernel should hide this feature, but our guest kernel + # currently lacks the commit with this change. + # The commit that introduces the change: + # https://github.com/torvalds/linux/commit/7187bb7d0b5c7dfa18ca82e9e5c75e13861b1d88 + assert guest_feats - host_feats == {"ssbs"} + case CpuModel.ARM_NEOVERSE_V1: + # KVM does not enable PAC or SVE features by default + # and Firecracker does not enable them either. + assert host_feats - guest_feats == { + "paca", + "pacg", + "sve", + "svebf16", + "svei8mm", + } + # kernel should hide this feature, but our guest kernel + # is not recent enough for this. + assert guest_feats - host_feats == {"ssbs"} + case _: + if os.environ.get("BUILDKITE") is not None: + assert False, f"Cpu model {cpu_model} is not supported" + + @pytest.mark.skipif( PLATFORM != "aarch64", reason="This is aarch64 specific test.", diff --git a/tests/integration_tests/functional/test_cpu_features.py b/tests/integration_tests/functional/test_cpu_features_x86_64.py similarity index 82% rename from tests/integration_tests/functional/test_cpu_features.py rename to tests/integration_tests/functional/test_cpu_features_x86_64.py index 225bec620eb..3cea004a6b4 100644 --- a/tests/integration_tests/functional/test_cpu_features.py +++ b/tests/integration_tests/functional/test_cpu_features_x86_64.py @@ -22,6 +22,7 @@ from framework.defs import SUPPORTED_HOST_KERNELS from framework.properties import global_props from framework.utils_cpu_templates import SUPPORTED_CPU_TEMPLATES +from framework.utils_cpuid import CPU_FEATURES_CMD, CpuModel PLATFORM = platform.machine() UNSUPPORTED_HOST_KERNEL = ( @@ -203,6 +204,272 @@ def test_brand_string(uvm_plain_any): assert False +@pytest.mark.skipif( + PLATFORM != "x86_64", + reason="This is x86_64 specific test.", +) +def test_host_vs_guest_cpu_features_x86_64(uvm_nano): + """Check CPU features host vs guest""" + + vm = uvm_nano + vm.add_net_iface() + vm.start() + host_feats = set(utils.check_output(CPU_FEATURES_CMD).stdout.strip().split(" ")) + guest_feats = set(vm.ssh.check_output(CPU_FEATURES_CMD).stdout.strip().split(" ")) + + cpu_model = cpuid_utils.get_cpu_codename() + match cpu_model: + case CpuModel.AMD_MILAN: + host_guest_diff_5_10 = { + "amd_ppin", + "aperfmperf", + "bpext", + "cat_l3", + "cdp_l3", + "cpb", + "cqm", + "cqm_llc", + "cqm_mbm_local", + "cqm_mbm_total", + "cqm_occup_llc", + "decodeassists", + "extapic", + "extd_apicid", + "flushbyasid", + "hw_pstate", + "ibs", + "irperf", + "lbrv", + "mba", + "monitor", + "mwaitx", + "overflow_recov", + "pausefilter", + "perfctr_llc", + "perfctr_nb", + "pfthreshold", + "rdpru", + "rdt_a", + "sev", + "sev_es", + "skinit", + "smca", + "sme", + "succor", + "svm_lock", + "tce", + "tsc_scale", + "v_vmsave_vmload", + "vgif", + "vmcb_clean", + "wdt", + } + + host_guest_diff_6_1 = host_guest_diff_5_10 - { + "lbrv", + "pausefilter", + "pfthreshold", + "sme", + "tsc_scale", + "v_vmsave_vmload", + "vgif", + "vmcb_clean", + } | {"brs", "rapl", "v_spec_ctrl"} + + if global_props.host_linux_version_tpl < (6, 1): + assert host_feats - guest_feats == host_guest_diff_5_10 + else: + assert host_feats - guest_feats == host_guest_diff_6_1 + + assert guest_feats - host_feats == { + "hypervisor", + "tsc_adjust", + "tsc_deadline_timer", + "tsc_known_freq", + } + case CpuModel.INTEL_SKYLAKE: + assert host_feats - guest_feats == { + "acpi", + "aperfmperf", + "arch_perfmon", + "art", + "bts", + "cat_l3", + "cdp_l3", + "cqm", + "cqm_llc", + "cqm_mbm_local", + "cqm_mbm_total", + "cqm_occup_llc", + "dca", + "ds_cpl", + "dtes64", + "dtherm", + "dts", + "epb", + "ept", + "ept_ad", + "est", + "flexpriority", + "flush_l1d", + "hwp", + "hwp_act_window", + "hwp_epp", + "hwp_pkg_req", + "ida", + "intel_ppin", + "intel_pt", + "mba", + "monitor", + "pbe", + "pdcm", + "pebs", + "pln", + "pts", + "rdt_a", + "sdbg", + "smx", + "tm", + "tm2", + "tpr_shadow", + "vmx", + "vnmi", + "vpid", + "xtpr", + } + assert guest_feats - host_feats == { + "hypervisor", + "tsc_known_freq", + "umip", + } + case CpuModel.INTEL_CASCADELAKE: + assert host_feats - guest_feats == { + "acpi", + "aperfmperf", + "arch_perfmon", + "art", + "bts", + "cat_l3", + "cdp_l3", + "cqm", + "cqm_llc", + "cqm_mbm_local", + "cqm_mbm_total", + "cqm_occup_llc", + "dca", + "ds_cpl", + "dtes64", + "dtherm", + "dts", + "epb", + "ept", + "ept_ad", + "est", + "flexpriority", + "flush_l1d", + "hwp", + "hwp_act_window", + "hwp_epp", + "hwp_pkg_req", + "ida", + "intel_ppin", + "intel_pt", + "mba", + "monitor", + "pbe", + "pdcm", + "pebs", + "pln", + "pts", + "rdt_a", + "sdbg", + "smx", + "tm", + "tm2", + "tpr_shadow", + "vmx", + "vnmi", + "vpid", + "xtpr", + } + assert guest_feats - host_feats == { + "hypervisor", + "tsc_known_freq", + "umip", + } + case CpuModel.INTEL_ICELAKE: + host_guest_diff_5_10 = { + "dtes64", + "hwp_act_window", + "pdcm", + "acpi", + "aperfmperf", + "arch_perfmon", + "art", + "bts", + "cat_l3", + "cqm", + "cqm_llc", + "cqm_mbm_local", + "cqm_mbm_total", + "cqm_occup_llc", + "dca", + "ds_cpl", + "dtherm", + "dts", + "epb", + "ept", + "ept_ad", + "est", + "flexpriority", + "flush_l1d", + "hwp", + "hwp_epp", + "hwp_pkg_req", + "ida", + "intel_ppin", + "intel_pt", + "mba", + "monitor", + "pbe", + "pconfig", + "pebs", + "pln", + "pts", + "rdt_a", + "sdbg", + "smx", + "split_lock_detect", + "tm", + "tm2", + "tme", + "tpr_shadow", + "vmx", + "vnmi", + "vpid", + "xtpr", + } + host_guest_diff_6_1 = host_guest_diff_5_10 - { + "bts", + "dtes64", + "dts", + "pebs", + } + + if global_props.host_linux_version_tpl < (6, 1): + assert host_feats - guest_feats == host_guest_diff_5_10 + else: + assert host_feats - guest_feats == host_guest_diff_6_1 + + assert guest_feats - host_feats == { + "hypervisor", + "tsc_known_freq", + } + case _: + if os.environ.get("BUILDKITE") is not None: + assert False, f"Cpu model {cpu_model} is not supported" + + # From the `Intel® 64 Architecture x2APIC Specification` # (https://courses.cs.washington.edu/courses/cse451/24wi/documentation/x2apic.pdf): # > The X2APIC MSRs cannot to be loaded and stored on VMX transitions. A VMX transition fails