From 157c2bd1ff685d6d84234b1fb55dcc747bc85c15 Mon Sep 17 00:00:00 2001 From: hjx_gitff Date: Mon, 30 Jun 2025 04:40:45 -0400 Subject: [PATCH 1/6] add dynamic wfx trap method for virtcca cvm Signed-off-by: hjx_gitff --- arch/arm64/kvm/virtcca_cvm.c | 41 ++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c index 3a5d1b50f2fd..1538992da9f4 100644 --- a/arch/arm64/kvm/virtcca_cvm.c +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -762,6 +762,41 @@ static int tmi_check_version(void) return 0; } +struct cpumask cvm_wfx_no_trap_mask; +static DEFINE_SPINLOCK(cvm_wfx_config_lock); +static ssize_t cvm_wfx_trap_config_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + spin_lock(&cvm_wfx_config_lock); + if (cpumask_parse(buf, &cvm_wfx_no_trap_mask) < 0) + return -EINVAL; + spin_unlock(&cvm_wfx_config_lock); + + return count; +} + +static ssize_t cvm_wfx_trap_config_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + int ret; + + ret = scnprintf(buf, PAGE_SIZE, "%*pb\n", cpumask_pr_args(&cvm_wfx_no_trap_mask)); + + return ret; +} + +static struct kobj_attribute cvm_wfx_trap_config_attr = __ATTR_RW(cvm_wfx_trap_config); + +static int __init cvm_wfx_trap_config_init(void) +{ + cpumask_clear(&cvm_wfx_no_trap_mask); + + return sysfs_create_file(kernel_kobj, &cvm_wfx_trap_config_attr.attr); +} +late_initcall(cvm_wfx_trap_config_init); + int kvm_tec_enter(struct kvm_vcpu *vcpu) { struct tmi_tec_run *run; @@ -773,12 +808,14 @@ int kvm_tec_enter(struct kvm_vcpu *vcpu) run = tec->tec_run; /* set/clear TWI TWE flags */ - if (vcpu->arch.hcr_el2 & HCR_TWI) + if ((vcpu->arch.hcr_el2 & HCR_TWI) && + !cpumask_test_cpu(vcpu->vcpu_id, &cvm_wfx_no_trap_mask)) run->tec_entry.flags |= TEC_ENTRY_FLAG_TRAP_WFI; else run->tec_entry.flags &= ~TEC_ENTRY_FLAG_TRAP_WFI; - if (vcpu->arch.hcr_el2 & HCR_TWE) + if ((vcpu->arch.hcr_el2 & HCR_TWE) && + !cpumask_test_cpu(vcpu->vcpu_id, &cvm_wfx_no_trap_mask)) run->tec_entry.flags |= TEC_ENTRY_FLAG_TRAP_WFE; else run->tec_entry.flags &= ~TEC_ENTRY_FLAG_TRAP_WFE; -- Gitee From 1b8d4e9ee9ffe2f276c4ff71819951d3ce6d33be Mon Sep 17 00:00:00 2001 From: hjx_gitff Date: Mon, 30 Jun 2025 04:41:40 -0400 Subject: [PATCH 2/6] add poll method for nfs Signed-off-by: hjx_gitff --- arch/arm64/kvm/virtcca_cvm.c | 2 +- drivers/char/Kconfig | 7 + drivers/char/Makefile | 1 + drivers/char/ib_core_cq_poll.c | 316 ++++++++++++++++++++++ drivers/char/ib_core_cq_poll.h | 20 ++ drivers/infiniband/core/cq.c | 41 +++ drivers/infiniband/core/ib_core_cq_poll.h | 25 ++ 7 files changed, 411 insertions(+), 1 deletion(-) create mode 100644 drivers/char/ib_core_cq_poll.c create mode 100644 drivers/char/ib_core_cq_poll.h create mode 100644 drivers/infiniband/core/ib_core_cq_poll.h diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c index 1538992da9f4..67db7f7503d5 100644 --- a/arch/arm64/kvm/virtcca_cvm.c +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -762,7 +762,7 @@ static int tmi_check_version(void) return 0; } -struct cpumask cvm_wfx_no_trap_mask; +static struct cpumask cvm_wfx_no_trap_mask; static DEFINE_SPINLOCK(cvm_wfx_config_lock); static ssize_t cvm_wfx_trap_config_store(struct kobject *kobj, struct kobj_attribute *attr, diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index ea7ace87a9df..d3784e20204e 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -442,4 +442,11 @@ config ADI and SSM (Silicon Secured Memory). Intended consumers of this driver include crash and makedumpfile. +config IB_CORE_CQ_POLL + tristate "Improve NFS using poll method" + default m + help + A Network File System read/write acceleration method + based on dynamic polling of partner kernel threads. + endmenu diff --git a/drivers/char/Makefile b/drivers/char/Makefile index 109af71c5416..df91b10a233b 100644 --- a/drivers/char/Makefile +++ b/drivers/char/Makefile @@ -46,3 +46,4 @@ obj-$(CONFIG_PS3_FLASH) += ps3flash.o obj-$(CONFIG_XILLYBUS_CLASS) += xillybus/ obj-$(CONFIG_POWERNV_OP_PANEL) += powernv-op-panel.o obj-$(CONFIG_ADI) += adi.o +obj-$(CONFIG_IB_CORE_CQ_POLL) += ib_core_cq_poll.o diff --git a/drivers/char/ib_core_cq_poll.c b/drivers/char/ib_core_cq_poll.c new file mode 100644 index 000000000000..7cb825d4e6d6 --- /dev/null +++ b/drivers/char/ib_core_cq_poll.c @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025. Huawei Technologies Co., Ltd. All rights reserved. + */ +#include +#include + +#include +#include + +#include +#include +#include + +#include "./ib_core_cq_poll.h" +#include "../infiniband/core/ib_core_cq_poll.h" + +#define INVALID_IB_POLL_CORE -1 + +static DEFINE_SPINLOCK(polling_kthread_spinlock); +static unsigned int poll_core; + +static struct task_struct *poll_cq_thread; +static struct task_struct *waker_polling_thread; +struct polling_kthread ib_cq_polling_kthread = { + .use_polling_kthread = 0, + .debug_cq_poll_stat = 0, + .cqe_polling_cnt = 0, +}; + +LIST_HEAD(ib_cq_poll_list); +static DEFINE_SPINLOCK(cq_list_lock); + +void add_cq_to_poll_list(void *cq) +{ + unsigned long flags; + cq_poll_node_t *cq_node; + + cq_node = (cq_poll_node_t *)kmalloc(sizeof(cq_poll_node_t), GFP_ATOMIC); + if (!cq_node) { + pr_err("add cq to poll list failed: kmalloc for cq_node failed.\n"); + return; + } + cq_node->cq = cq; + cq_node->time_used_ns = 0; + cq_node->poll_cq_cnt = 0; + cq_node->max_time_ns = 0; + + spin_lock_irqsave(&cq_list_lock, flags); + list_add_tail(&cq_node->list, &ib_cq_poll_list); + spin_unlock_irqrestore(&cq_list_lock, flags); +} + +void del_cq_from_poll_list(void *del_cq) +{ + cq_poll_node_t *poll_node_entry, *poll_node_next; + void *curr_cq; + void *cq = del_cq; + unsigned long flags; + + spin_lock_irqsave(&cq_list_lock, flags); + list_for_each_entry_safe(poll_node_entry, poll_node_next, + &ib_cq_poll_list, list) { + curr_cq = poll_node_entry->cq; + if (curr_cq == cq) { + list_del(&poll_node_entry->list); + kfree(poll_node_entry); + break; + } + } + spin_unlock_irqrestore(&cq_list_lock, flags); +} + +void clear_cq_poll_list(void) +{ + unsigned long flags; + cq_poll_node_t *entry, *next; + + spin_lock_irqsave(&cq_list_lock, flags); + list_for_each_entry_safe(entry, next, &ib_cq_poll_list, list) { + list_del(&entry->list); + kfree(entry); + } + spin_unlock_irqrestore(&cq_list_lock, flags); + + INIT_LIST_HEAD(&ib_cq_poll_list); +} + +void cq_polling(void *data) +{ + void *cq; + int completed = 0; + unsigned long flags; + u64 time_interval; + ktime_t start_time_stamp, end_time_stamp; + cq_poll_node_t *poll_node_entry, *poll_node_next; + + spin_lock_irqsave(&cq_list_lock, flags); + list_for_each_entry_safe(poll_node_entry, poll_node_next, + &ib_cq_poll_list, list) { + cq = poll_node_entry->cq; + if (!cq) { + WARN_ONCE(1, "got NULL CQ 0x%p in poll list\n", cq); + continue; + } + start_time_stamp = ktime_get(); + completed = ib_poll_cq_thread(cq); + end_time_stamp = ktime_get(); + if (ib_cq_polling_kthread.debug_cq_poll_stat && completed) { + time_interval = ktime_to_ns(ktime_sub(end_time_stamp, start_time_stamp)); + poll_node_entry->time_used_ns += time_interval; + poll_node_entry->poll_cq_cnt++; + if (poll_node_entry->max_time_ns < time_interval) + poll_node_entry->max_time_ns = time_interval; + } + } + spin_unlock_irqrestore(&cq_list_lock, flags); +} + +void wakeup_and_poll(struct task_struct *awakened_thread) +{ + wake_up_process(awakened_thread); + + cq_polling(NULL); +} + +int polling_thread(void *data) +{ + while (true) { + if (ib_cq_polling_kthread.use_polling_kthread) + wakeup_and_poll(waker_polling_thread); + + set_current_state(TASK_INTERRUPTIBLE); + if (unlikely(kthread_should_stop())) { + set_current_state(TASK_RUNNING); + break; + } + schedule(); + } + + return 0; +} + +int polling_awaken_thread(void *data) +{ + while (true) { + if (ib_cq_polling_kthread.use_polling_kthread) + wakeup_and_poll(poll_cq_thread); + + set_current_state(TASK_INTERRUPTIBLE); + if (unlikely(kthread_should_stop())) { + set_current_state(TASK_RUNNING); + break; + } + schedule(); + } + + return 0; +} + +static ssize_t ib_core_poll_cpu_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + int ret; + + if (ib_cq_polling_kthread.use_polling_kthread) + ret = sprintf(buf, "Current polling core is: %u\n", poll_core); + else + ret = sprintf(buf, "Current polling thread is down\n"); + + return ret; +} + +static ssize_t ib_core_poll_cpu_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int input_poll_core; + unsigned int max_nr_cores = num_possible_cpus(); + + if (sscanf(buf, "%d", &input_poll_core) < 0) { + pr_err("Invalid input, input format: e.g. 8\n"); + return -EINVAL; + } + + spin_lock(&polling_kthread_spinlock); + if (input_poll_core == INVALID_IB_POLL_CORE) { + ib_cq_polling_kthread.use_polling_kthread = 0; + set_kthread_polling_ctx(&ib_cq_polling_kthread); + clear_cq_poll_list(); + goto out; + } + if (input_poll_core < 0 || input_poll_core >= max_nr_cores) { + pr_err("Invalid CPU core ID. Valid range is 0 to %u.\n", max_nr_cores - 1); + goto err_inval; + } + ib_cq_polling_kthread.use_polling_kthread = 1; + set_kthread_polling_ctx(&ib_cq_polling_kthread); + poll_core = (unsigned int)input_poll_core; + set_cpus_allowed_ptr(poll_cq_thread, cpumask_of(poll_core)); + set_cpus_allowed_ptr(waker_polling_thread, cpumask_of(poll_core)); + wake_up_process(waker_polling_thread); + +out: + spin_unlock(&polling_kthread_spinlock); + return count; + +err_inval: + spin_unlock(&polling_kthread_spinlock); + return -EINVAL; +} + +static ssize_t ib_core_poll_stat_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + int ret; + unsigned long flags; + unsigned long avg_poll_time; + cq_poll_node_t *poll_node_entry, *poll_node_next; + + ib_cq_polling_kthread = get_kthread_polling_ctx(); + ret = sprintf(buf, "kthread polled cqes cnt: %lu\n", + ib_cq_polling_kthread.cqe_polling_cnt); + + spin_lock_irqsave(&cq_list_lock, flags); + pr_info("cq\t\tpoll cnt\tavg poll time(ns)\tmax poll time(ns)\n"); + list_for_each_entry_safe(poll_node_entry, poll_node_next, + &ib_cq_poll_list, list) { + avg_poll_time = poll_node_entry->time_used_ns / poll_node_entry->poll_cq_cnt; + pr_info("%p\t%lu\t\t%lu\t\t\t%lu", poll_node_entry->cq, + poll_node_entry->poll_cq_cnt, avg_poll_time, + poll_node_entry->max_time_ns); + } + spin_unlock_irqrestore(&cq_list_lock, flags); + + return ret; +} + +static ssize_t ib_core_poll_stat_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + unsigned int input_debug_cq_poll_stat; + + if (sscanf(buf, "%u", &input_debug_cq_poll_stat) < 0) { + pr_err("Invalid input, input format: <0/1> for on/off\n"); + return -EINVAL; + } + spin_lock(&polling_kthread_spinlock); + ib_cq_polling_kthread.debug_cq_poll_stat = input_debug_cq_poll_stat; + set_kthread_polling_ctx(&ib_cq_polling_kthread); + spin_unlock(&polling_kthread_spinlock); + + return count; +} + +static struct kobj_attribute ib_core_poll_cpu_attr = __ATTR_RW(ib_core_poll_cpu); +static struct kobj_attribute ib_core_poll_stat_attr = __ATTR_RW(ib_core_poll_stat); + +static int __init mod_poll_cq_kthread_init(void) +{ + int ret; + + ret = sysfs_create_file(kernel_kobj, &ib_core_poll_cpu_attr.attr); + if (ret) + return ret; + ret = sysfs_create_file(kernel_kobj, &ib_core_poll_stat_attr.attr); + if (ret) + return ret; + + /* init poll thread */ + if (!poll_cq_thread) + poll_cq_thread = kthread_create(polling_thread, NULL, "polling_thread"); + if (IS_ERR(poll_cq_thread)) { + ret = PTR_ERR(poll_cq_thread); + poll_cq_thread = NULL; + goto out; + } + if (!waker_polling_thread) + waker_polling_thread = kthread_create(polling_awaken_thread, + NULL, "polling_awaken_thread"); + if (IS_ERR(waker_polling_thread)) { + ret = PTR_ERR(waker_polling_thread); + waker_polling_thread = NULL; + goto out; + } + ib_cq_polling_kthread.add_to_poll_list = add_cq_to_poll_list; + ib_cq_polling_kthread.del_from_poll_list = del_cq_from_poll_list; + set_kthread_polling_ctx(&ib_cq_polling_kthread); + +out: + return 0; +} +late_initcall(mod_poll_cq_kthread_init); + +static void mod_poll_cq_kthread_exit(void) +{ + if (poll_cq_thread) + kthread_stop(poll_cq_thread); + if (waker_polling_thread) + kthread_stop(waker_polling_thread); + if (ib_cq_polling_kthread.use_polling_kthread) { + ib_cq_polling_kthread.use_polling_kthread = 0; + ib_cq_polling_kthread.debug_cq_poll_stat = 0; + set_kthread_polling_ctx(&ib_cq_polling_kthread); + clear_cq_poll_list(); + } + + sysfs_remove_file(kernel_kobj, &ib_core_poll_cpu_attr.attr); + sysfs_remove_file(kernel_kobj, &ib_core_poll_stat_attr.attr); +} +module_exit(mod_poll_cq_kthread_exit); + +MODULE_DESCRIPTION("cq polling kthread init and config"); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Chen Haotian "); diff --git a/drivers/char/ib_core_cq_poll.h b/drivers/char/ib_core_cq_poll.h new file mode 100644 index 000000000000..674e0391a156 --- /dev/null +++ b/drivers/char/ib_core_cq_poll.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2025. Huawei Technologies Co., Ltd. All rights reserved. + */ +#ifndef __POLL_CQ_KTHREAD_H +#define __POLL_CQ_KTHREAD_H + +typedef struct cq_poll_node { + struct list_head list; + void *cq; + unsigned long time_used_ns; + unsigned long poll_cq_cnt; + unsigned long max_time_ns; +} cq_poll_node_t; + +extern void add_cq_to_poll_list(void *cq); +extern void del_cq_from_poll_list(void *del_cq); +extern void clear_cq_poll_list(void); + +#endif /* __POLL_CQ_KTHREAD_H */ diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c index a70876a0a231..1eae36bb0a51 100644 --- a/drivers/infiniband/core/cq.c +++ b/drivers/infiniband/core/cq.c @@ -7,6 +7,7 @@ #include #include "core_priv.h" +#include "./ib_core_cq_poll.h" #include /* Max size for shared CQ, may require tuning */ @@ -195,6 +196,39 @@ static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private) queue_work(cq->comp_wq, &cq->work); } +static struct polling_kthread ib_cq_polling_kthread = { + .use_polling_kthread = 0, + .debug_cq_poll_stat = 0, + .cqe_polling_cnt = 0, +}; + +struct polling_kthread get_kthread_polling_ctx(void) +{ + return ib_cq_polling_kthread; +} +EXPORT_SYMBOL(get_kthread_polling_ctx); + +void set_kthread_polling_ctx(struct polling_kthread *polling_ctx) +{ + memcpy(&ib_cq_polling_kthread, polling_ctx, sizeof(struct polling_kthread)); +} +EXPORT_SYMBOL(set_kthread_polling_ctx); + +int ib_poll_cq_thread(void *data) +{ + int completed; + struct ib_wc wcs[IB_POLL_BATCH]; + struct ib_cq *cq = (struct ib_cq *)data; + + memcpy(wcs, cq->wc, sizeof(wcs)); + completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE, wcs, + IB_POLL_BATCH); + if (ib_cq_polling_kthread.debug_cq_poll_stat) + ib_cq_polling_kthread.cqe_polling_cnt += completed; + return completed; +} +EXPORT_SYMBOL(ib_poll_cq_thread); + /** * __ib_alloc_cq - allocate a completion queue * @dev: device to allocate the CQ for @@ -243,6 +277,10 @@ struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe, rdma_dim_init(cq); + if (ib_cq_polling_kthread.use_polling_kthread) { + cq->poll_ctx = IB_POLL_DIRECT; + ib_cq_polling_kthread.add_to_poll_list(cq); + } switch (cq->poll_ctx) { case IB_POLL_DIRECT: cq->comp_handler = ib_cq_completion_direct; @@ -338,6 +376,9 @@ void ib_free_cq(struct ib_cq *cq) WARN_ON_ONCE(1); } + if (ib_cq_polling_kthread.use_polling_kthread) + ib_cq_polling_kthread.del_from_poll_list(cq); + rdma_dim_destroy(cq); trace_cq_free(cq); ret = cq->device->ops.destroy_cq(cq, NULL); diff --git a/drivers/infiniband/core/ib_core_cq_poll.h b/drivers/infiniband/core/ib_core_cq_poll.h new file mode 100644 index 000000000000..05f89c7d574a --- /dev/null +++ b/drivers/infiniband/core/ib_core_cq_poll.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2025. Huawei Technologies Co., Ltd. All rights reserved. + */ +#ifndef __IB_CORE_POLL_KTHREAD_H +#define __IB_CORE_POLL_KTHREAD_H + +struct polling_kthread { + /* configs */ + int use_polling_kthread; + unsigned int debug_cq_poll_stat; + + /* vars */ + unsigned long cqe_polling_cnt; + + /* ops */ + void (*add_to_poll_list)(void *new_entry); + void (*del_from_poll_list)(void *deleted_entry); +}; + +extern int ib_poll_cq_thread(void *data); +extern struct polling_kthread get_kthread_polling_ctx(void); +extern void set_kthread_polling_ctx(struct polling_kthread *polling_ctx); + +#endif /* __IB_CORE_POLL_KTHREAD_H */ -- Gitee From 74fbc346a171d54ac192feaabf02ff1b4bb62e2f Mon Sep 17 00:00:00 2001 From: yxk Date: Tue, 12 Aug 2025 23:47:35 +0800 Subject: [PATCH 3/6] improve cvm vtimer irq inject method --- arch/arm64/include/asm/kvm_tmi.h | 3 ++- arch/arm64/kvm/arch_timer.c | 3 ++- arch/arm64/kvm/tmi.c | 4 +++- arch/arm64/kvm/virtcca_cvm.c | 7 ++++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/arch/arm64/include/asm/kvm_tmi.h b/arch/arm64/include/asm/kvm_tmi.h index 6e599df75c19..45b04b51bdfa 100644 --- a/arch/arm64/include/asm/kvm_tmi.h +++ b/arch/arm64/include/asm/kvm_tmi.h @@ -313,6 +313,7 @@ struct tmi_tec_run { #define TMI_ABI_VERSION_GET_MINOR(_version) ((_version) & 0xFFFF) #define TMI_ABI_VERSION_MAJOR U(0x2) +#define TMI_ABI_VERSION_MINOR U(0x2) /* KVM_CAP_ARM_TMM on VM fd */ #define KVM_CAP_ARM_TMM_CONFIG_CVM_HOST 0 @@ -406,7 +407,7 @@ static inline bool is_armv8_4_sel2_present(void) u64 tmi_version(void); u64 tmi_data_create(u64 data, u64 rd, u64 map_addr, u64 src, u64 level); u64 tmi_cvm_activate(u64 rd); -u64 tmi_cvm_create(u64 params_ptr, u64 numa_set); +u64 tmi_cvm_create(u64 params_ptr, u64 numa_set, bool vtimer_adjust); u64 tmi_cvm_destroy(u64 rd); u64 tmi_tec_create(u64 numa_set, u64 rd, u64 mpidr, u64 params_ptr); u64 tmi_tec_destroy(u64 tec); diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 27032290094d..fd958be83dfe 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -177,11 +177,12 @@ static void timer_set_cval(struct arch_timer_context *ctxt, u64 cval) } #ifdef CONFIG_HISI_VIRTCCA_HOST + static bool cvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx) { return timer_ctx && ((timer_get_ctl(timer_ctx) & - (ARCH_TIMER_CTRL_IT_MASK | ARCH_TIMER_CTRL_ENABLE)) == ARCH_TIMER_CTRL_ENABLE); + (ARCH_TIMER_CTRL_ENABLE)) == ARCH_TIMER_CTRL_ENABLE); } void kvm_cvm_timers_update(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/kvm/tmi.c b/arch/arm64/kvm/tmi.c index 3c1425c62d31..0be0f5d7dd3a 100644 --- a/arch/arm64/kvm/tmi.c +++ b/arch/arm64/kvm/tmi.c @@ -81,10 +81,12 @@ u64 tmi_cvm_activate(u64 rd) return res.a1; } -u64 tmi_cvm_create(u64 params_ptr, u64 numa_set) +u64 tmi_cvm_create(u64 params_ptr, u64 numa_set, bool vtimer_adjust) { struct arm_smccc_res res; + /* vtimer_adjust enable to reduce vtimer irq */ + params_ptr = params_ptr | (vtimer_adjust ? 1UL : 0UL); arm_smccc_1_1_smc(TMI_TMM_CVM_CREATE, params_ptr, numa_set, &res); return res.a1; } diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c index 67db7f7503d5..bf6c635a89af 100644 --- a/arch/arm64/kvm/virtcca_cvm.c +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -23,6 +23,7 @@ static DEFINE_SPINLOCK(cvm_vmid_lock); static unsigned long *cvm_vmid_bitmap; DEFINE_STATIC_KEY_FALSE(virtcca_cvm_is_available); +static bool virtcca_vtimer_adjust; #define SIMD_PAGE_SIZE 0x3000 #define UEFI_MAX_SIZE 0x8000000 #define UEFI_DTB_START 0x40000000 @@ -167,7 +168,7 @@ int kvm_arm_create_cvm(struct kvm *kvm) cvm->params->ns_vtcr = kvm->arch.vtcr; cvm->params->vttbr_el2 = kvm->arch.mmu.pgd_phys; memcpy(cvm->params->rpv, &cvm->cvm_vmid, sizeof(cvm->cvm_vmid)); - cvm->rd = tmi_cvm_create(__pa(cvm->params), numa_set); + cvm->rd = tmi_cvm_create(__pa(cvm->params), numa_set, virtcca_vtimer_adjust); if (!cvm->rd) { kvm_err("KVM creates cVM failed: %d\n", cvm->cvm_vmid); ret = -ENOMEM; @@ -757,6 +758,10 @@ static int tmi_check_version(void) version_minor); return -ENXIO; } + if (version_minor < TMI_ABI_VERSION_MINOR) + virtcca_vtimer_adjust = false; + else + virtcca_vtimer_adjust = true; kvm_info("TMI ABI version %d,%d\n", version_major, version_minor); return 0; -- Gitee From a00a6e7bfa6627d88c2bba84438bc5a4c9a2a1f0 Mon Sep 17 00:00:00 2001 From: Haotian Chen Date: Fri, 3 Jan 2025 10:47:14 +0800 Subject: [PATCH 4/6] add ipi direct inject method --- arch/arm64/include/asm/virtcca_cvm_guest.h | 7 +++ arch/arm64/kernel/smp.c | 17 +++++- arch/arm64/kernel/virtcca_cvm_guest.c | 62 ++++++++++++++++++++++ include/linux/virtcca_cvm_domain.h | 46 ++++++++++++++++ kernel/sched/idle.c | 30 +++++++++++ 5 files changed, 160 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/virtcca_cvm_guest.h b/arch/arm64/include/asm/virtcca_cvm_guest.h index a074bd65dab9..aaa728b2dfb6 100644 --- a/arch/arm64/include/asm/virtcca_cvm_guest.h +++ b/arch/arm64/include/asm/virtcca_cvm_guest.h @@ -8,6 +8,13 @@ #ifdef CONFIG_HISI_VIRTCCA_GUEST struct device; +DECLARE_PER_CPU(unsigned int, virtcca_unpark_idle_notify); +DECLARE_PER_CPU(unsigned int, virtcca_park_idle_state); + +extern void smp_cross_call(const struct cpumask *target, unsigned int ipinr); + +extern void cvm_send_call_function_single_ipi(int cpu, int ipi_type); + extern int set_cvm_memory_encrypted(unsigned long addr, int numpages); extern int set_cvm_memory_decrypted(unsigned long addr, int numpages); diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 50f6576f1b31..7819ad96f17f 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -805,7 +806,7 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = { [IPI_WAKEUP] = "CPU wake-up interrupts", }; -static void smp_cross_call(const struct cpumask *target, unsigned int ipinr); +void smp_cross_call(const struct cpumask *target, unsigned int ipinr); unsigned long irq_err_count; @@ -832,6 +833,12 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask) void arch_send_call_function_single_ipi(int cpu) { +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (virtcca_cvm_domain()) { + cvm_send_call_function_single_ipi(cpu, IPI_CALL_FUNC); + return; + } +#endif smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC); } @@ -958,7 +965,7 @@ static irqreturn_t ipi_handler(int irq, void *data) return IRQ_HANDLED; } -static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) +void smp_cross_call(const struct cpumask *target, unsigned int ipinr) { trace_ipi_raise(target, ipi_types[ipinr]); __ipi_send_mask(ipi_desc[ipinr], target); @@ -1027,6 +1034,12 @@ void __init set_smp_ipi_range(int ipi_base, int n) void arch_smp_send_reschedule(int cpu) { +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (virtcca_cvm_domain()) { + cvm_send_call_function_single_ipi(cpu, IPI_RESCHEDULE); + return; + } +#endif smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); } diff --git a/arch/arm64/kernel/virtcca_cvm_guest.c b/arch/arm64/kernel/virtcca_cvm_guest.c index da2c4a17f837..e6b41cf47350 100644 --- a/arch/arm64/kernel/virtcca_cvm_guest.c +++ b/arch/arm64/kernel/virtcca_cvm_guest.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -72,6 +73,67 @@ bool is_virtcca_cvm_world(void) } EXPORT_SYMBOL_GPL(is_virtcca_cvm_world); +struct cpumask cvm_spin_cpumask; +static DEFINE_SPINLOCK(ipi_passthrough_lock); +DEFINE_PER_CPU(unsigned int, virtcca_unpark_idle_notify); +DEFINE_PER_CPU(unsigned int, virtcca_park_idle_state); + +bool virtcca_spin_cpumask_test_cpu(int cpu) +{ + return cpumask_test_cpu(cpu, &cvm_spin_cpumask); +} + +static inline bool is_cvm_in_park_idle_state(int cpu) +{ + return (per_cpu(virtcca_park_idle_state, cpu) == CVM_PARK_IDLE); +} + +void cvm_send_call_function_single_ipi(int cpu, int ipi_type) { + if (virtcca_spin_cpumask_test_cpu(cpu) && is_cvm_in_park_idle_state(cpu)) + virtcca_set_unpark_idle_notify(cpu); + else + smp_cross_call(cpumask_of(cpu), ipi_type); +} + +static ssize_t soft_ipi_passthrough_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + spin_lock(&ipi_passthrough_lock); + if (cpumask_parse(buf, &cvm_spin_cpumask) < 0) { + return -EINVAL; + } + spin_unlock(&ipi_passthrough_lock); + + return count; +} + +static ssize_t soft_ipi_passthrough_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + int ret; + + ret = scnprintf(buf, PAGE_SIZE, "%*pb\n", cpumask_pr_args(&cvm_spin_cpumask)); + + return ret; +} + +static struct kobj_attribute soft_ipi_passthrough_attr = __ATTR_RW(soft_ipi_passthrough); + +static int __init soft_ipi_passthrough_init(void) +{ + unsigned int cpu; + unsigned int max_nr_cpus = num_possible_cpus(); + + cpumask_clear(&cvm_spin_cpumask); + for (cpu = 0; cpu < max_nr_cpus; cpu++) { + virtcca_clear_unpark_idle_notify(cpu); + } + + return sysfs_create_file(kernel_kobj, &soft_ipi_passthrough_attr.attr); +} +late_initcall(soft_ipi_passthrough_init); + static int change_page_range_cvm(pte_t *ptep, unsigned long addr, void *data) { bool encrypt = (bool)data; diff --git a/include/linux/virtcca_cvm_domain.h b/include/linux/virtcca_cvm_domain.h index 9a24a031d12c..d41153c428d6 100644 --- a/include/linux/virtcca_cvm_domain.h +++ b/include/linux/virtcca_cvm_domain.h @@ -9,6 +9,12 @@ #ifdef CONFIG_HISI_VIRTCCA_GUEST #include + +enum cvm_park_idle_state { + CVM_PARK_RUNNING, + CVM_PARK_IDLE +}; + static inline bool virtcca_cvm_domain(void) { return is_virtcca_cvm_world(); @@ -16,6 +22,29 @@ static inline bool virtcca_cvm_domain(void) extern void enable_swiotlb_for_cvm_dev(struct device *dev, bool enable); +extern bool virtcca_spin_cpumask_test_cpu(int cpu); + +static inline void virtcca_set_park_idle_state(int cpu, + enum cvm_park_idle_state state) +{ + per_cpu(virtcca_park_idle_state, cpu) = state; +} + +static inline bool is_virtcca_unpark_idle_notify_set(int cpu) +{ + return (per_cpu(virtcca_unpark_idle_notify, cpu) == 1); +} + +static inline void virtcca_set_unpark_idle_notify(int cpu) +{ + per_cpu(virtcca_unpark_idle_notify, cpu) = 1; +} + +static inline void virtcca_clear_unpark_idle_notify(int cpu) +{ + per_cpu(virtcca_unpark_idle_notify, cpu) = 0; +} + #else static inline bool virtcca_cvm_domain(void) { @@ -34,6 +63,23 @@ static inline struct page *virtcca_its_alloc_shared_pages_node(int node, gfp_t g static inline void virtcca_its_free_shared_pages(void *addr, int order) {} +static inline bool virtcca_spin_cpumask_test_cpu(int cpu) +{ + return false; +} + +static inline void virtcca_set_park_idle_state(int cpu, + enum cvm_park_idle_state state) {} + +static inline bool is_virtcca_unpark_idle_notify_set(int cpu) +{ + return false; +} + +static inline void virtcca_set_unpark_idle_notify(int cpu) {} + +static inline void virtcca_clear_unpark_idle_notify(int cpu) {} + #endif #ifdef CONFIG_HISI_VIRTCCA_HOST diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 5007b25c5bc6..35506cf62026 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -7,6 +7,8 @@ * tasks which are handled in sched/fair.c ) */ +#include + /* Linker adds these: start and end of __cpuidle functions */ extern char __cpuidle_text_start[], __cpuidle_text_end[]; @@ -229,6 +231,28 @@ static void cpuidle_idle_call(void) local_irq_enable(); } +#ifdef CONFIG_HISI_VIRTCCA_GUEST +static noinline int cvm_cpu_idle_poll(void) +{ + unsigned int cpu = smp_processor_id(); + + if (!virtcca_cvm_domain() || !virtcca_spin_cpumask_test_cpu(cpu)) { + return 0; + } + trace_cpu_idle(0, smp_processor_id()); + local_irq_enable(); + + virtcca_set_park_idle_state(cpu, CVM_PARK_IDLE); + while (!tif_need_resched() && !is_virtcca_unpark_idle_notify_set(cpu)) { + cpu_relax(); + } + virtcca_clear_unpark_idle_notify(cpu); + virtcca_set_park_idle_state(cpu, CVM_PARK_RUNNING); + trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id()); + + return 1; +} +#endif /* * Generic idle loop implementation * @@ -269,6 +293,12 @@ static void do_idle(void) arch_cpu_idle_enter(); rcu_nocb_flush_deferred_wakeup(); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (cvm_cpu_idle_poll()) { + arch_cpu_idle_exit(); + break; + } +#endif /* * In poll mode we reenable interrupts and spin. Also if we * detected in the wakeup from idle path that the tick -- Gitee From 1b15e56212b16835a2831373f6de6a29b3493445 Mon Sep 17 00:00:00 2001 From: yxk Date: Tue, 19 Aug 2025 01:02:53 +0800 Subject: [PATCH 5/6] Revise cvm_cpu_idle_poll --- kernel/sched/idle.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 35506cf62026..292c3d4134ea 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -232,7 +232,7 @@ static void cpuidle_idle_call(void) } #ifdef CONFIG_HISI_VIRTCCA_GUEST -static noinline int cvm_cpu_idle_poll(void) +static noinline int virtcca_cvm_cpu_idle_poll(void) { unsigned int cpu = smp_processor_id(); @@ -240,19 +240,27 @@ static noinline int cvm_cpu_idle_poll(void) return 0; } trace_cpu_idle(0, smp_processor_id()); - local_irq_enable(); + stop_critical_timings(); + ct_cpuidle_enter(); virtcca_set_park_idle_state(cpu, CVM_PARK_IDLE); + raw_local_irq_enable(); while (!tif_need_resched() && !is_virtcca_unpark_idle_notify_set(cpu)) { cpu_relax(); } + raw_local_irq_disable(); virtcca_clear_unpark_idle_notify(cpu); virtcca_set_park_idle_state(cpu, CVM_PARK_RUNNING); + + ct_cpuidle_exit(); + start_critical_timings(); trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id()); + local_irq_enable(); return 1; } #endif + /* * Generic idle loop implementation * @@ -294,7 +302,7 @@ static void do_idle(void) rcu_nocb_flush_deferred_wakeup(); #ifdef CONFIG_HISI_VIRTCCA_GUEST - if (cvm_cpu_idle_poll()) { + if (virtcca_cvm_cpu_idle_poll()) { arch_cpu_idle_exit(); break; } -- Gitee From 19e57f6bd889d4defe5957471224a78ed7302253 Mon Sep 17 00:00:00 2001 From: y00878127 Date: Tue, 19 Aug 2025 18:15:48 +0800 Subject: [PATCH 6/6] Revise cvm_send_call_function_single_ipi --- arch/arm64/include/asm/virtcca_cvm_guest.h | 4 ---- arch/arm64/kernel/smp.c | 11 ++++++++++- arch/arm64/kernel/virtcca_cvm_guest.c | 12 ------------ include/linux/virtcca_cvm_domain.h | 10 ++++++++++ 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/arch/arm64/include/asm/virtcca_cvm_guest.h b/arch/arm64/include/asm/virtcca_cvm_guest.h index aaa728b2dfb6..9e3e49452e45 100644 --- a/arch/arm64/include/asm/virtcca_cvm_guest.h +++ b/arch/arm64/include/asm/virtcca_cvm_guest.h @@ -11,10 +11,6 @@ struct device; DECLARE_PER_CPU(unsigned int, virtcca_unpark_idle_notify); DECLARE_PER_CPU(unsigned int, virtcca_park_idle_state); -extern void smp_cross_call(const struct cpumask *target, unsigned int ipinr); - -extern void cvm_send_call_function_single_ipi(int cpu, int ipi_type); - extern int set_cvm_memory_encrypted(unsigned long addr, int numpages); extern int set_cvm_memory_decrypted(unsigned long addr, int numpages); diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 7819ad96f17f..b8c9a3776f0b 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -806,7 +806,7 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = { [IPI_WAKEUP] = "CPU wake-up interrupts", }; -void smp_cross_call(const struct cpumask *target, unsigned int ipinr); +static void smp_cross_call(const struct cpumask *target, unsigned int ipinr); unsigned long irq_err_count; @@ -831,6 +831,15 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask) smp_cross_call(mask, IPI_CALL_FUNC); } +#ifdef CONFIG_HISI_VIRTCCA_GUEST +static void cvm_send_call_function_single_ipi(int cpu, int ipi_type) { + if (virtcca_spin_cpumask_test_cpu(cpu) && is_cvm_in_park_idle_state(cpu)) + virtcca_set_unpark_idle_notify(cpu); + else + smp_cross_call(cpumask_of(cpu), ipi_type); +} +#endif + void arch_send_call_function_single_ipi(int cpu) { #ifdef CONFIG_HISI_VIRTCCA_GUEST diff --git a/arch/arm64/kernel/virtcca_cvm_guest.c b/arch/arm64/kernel/virtcca_cvm_guest.c index e6b41cf47350..44623d7057e6 100644 --- a/arch/arm64/kernel/virtcca_cvm_guest.c +++ b/arch/arm64/kernel/virtcca_cvm_guest.c @@ -83,18 +83,6 @@ bool virtcca_spin_cpumask_test_cpu(int cpu) return cpumask_test_cpu(cpu, &cvm_spin_cpumask); } -static inline bool is_cvm_in_park_idle_state(int cpu) -{ - return (per_cpu(virtcca_park_idle_state, cpu) == CVM_PARK_IDLE); -} - -void cvm_send_call_function_single_ipi(int cpu, int ipi_type) { - if (virtcca_spin_cpumask_test_cpu(cpu) && is_cvm_in_park_idle_state(cpu)) - virtcca_set_unpark_idle_notify(cpu); - else - smp_cross_call(cpumask_of(cpu), ipi_type); -} - static ssize_t soft_ipi_passthrough_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) diff --git a/include/linux/virtcca_cvm_domain.h b/include/linux/virtcca_cvm_domain.h index d41153c428d6..4072f369b854 100644 --- a/include/linux/virtcca_cvm_domain.h +++ b/include/linux/virtcca_cvm_domain.h @@ -15,6 +15,11 @@ enum cvm_park_idle_state { CVM_PARK_IDLE }; +static inline bool is_cvm_in_park_idle_state(int cpu) +{ + return (per_cpu(virtcca_park_idle_state, cpu) == CVM_PARK_IDLE); +} + static inline bool virtcca_cvm_domain(void) { return is_virtcca_cvm_world(); @@ -46,6 +51,11 @@ static inline void virtcca_clear_unpark_idle_notify(int cpu) } #else +static inline bool is_cvm_in_park_idle_state(int cpu) +{ + return false; +} + static inline bool virtcca_cvm_domain(void) { return false; -- Gitee