From 6ac98b6d57e7d8cd13fb92f7d11941039e84b432 Mon Sep 17 00:00:00 2001 From: vvzxy <145555693+vvzxy@users.noreply.github.com> Date: Fri, 21 Jun 2024 10:49:09 +0800 Subject: [PATCH] =?UTF-8?q?cpu=5Fwatcher=EF=BC=9Aschedule=5Fdelay=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=E6=89=A9=E5=85=85&sar=E5=8A=9F=E8=83=BD=E9=80=82?= =?UTF-8?q?=E9=85=8Dcontroller=20(#830)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cpu_watcher：schedule_delay增加阈值选项&&workflow增加测试流程 * . * cpu_watcher：增加controller功能 * cpu_watcher:schedule_delay增加dump出调度延迟过大task的前两个task * sar功能适配controller * . --- .../cpu_watcher/bpf/cs_delay.bpf.c | 11 +++ .../cpu_watcher/bpf/mq_delay.bpf.c | 17 +++++ .../cpu_watcher/bpf/preempt.bpf.c | 15 +++- .../CPU_Subsystem/cpu_watcher/bpf/sar.bpf.c | 23 +++++- .../cpu_watcher/bpf/sc_delay.bpf.c | 40 +++++----- .../cpu_watcher/bpf/schedule_delay.bpf.c | 48 ++++++++---- .../CPU_Subsystem/cpu_watcher/controller.c | 23 +++--- .../CPU_Subsystem/cpu_watcher/cpu_watcher.c | 73 +++++++++++++------ .../cpu_watcher/include/cpu_watcher.h | 13 +++- 9 files changed, 187 insertions(+), 76 deletions(-) diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/cs_delay.bpf.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/cs_delay.bpf.c index 4770a3c15..caaf8c408 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/cs_delay.bpf.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/cs_delay.bpf.c @@ -30,9 +30,19 @@ struct { __uint(max_entries, 256 * 1024); } rb SEC(".maps"); +static inline struct cs_ctrl *get_cs_ctrl(void) { + struct cs_ctrl *cs_ctrl; + cs_ctrl = bpf_map_lookup_elem(&cs_ctrl_map, &ctrl_key); + if (!cs_ctrl || !cs_ctrl->cs_func) { + return NULL; + } + return cs_ctrl; +} + SEC("kprobe/schedule") int BPF_KPROBE(schedule) { + struct cs_ctrl *cs_ctrl = get_cs_ctrl(); u64 t1; t1 = bpf_ktime_get_ns()/1000; int key =0; @@ -43,6 +53,7 @@ int BPF_KPROBE(schedule) SEC("kretprobe/schedule") int BPF_KRETPROBE(schedule_exit) { + struct cs_ctrl *cs_ctrl = get_cs_ctrl(); u64 t2 = bpf_ktime_get_ns()/1000; u64 t1,delay; int key = 0; diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/mq_delay.bpf.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/mq_delay.bpf.c index b86631002..3aa9dd92e 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/mq_delay.bpf.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/mq_delay.bpf.c @@ -33,6 +33,16 @@ struct { __uint(max_entries, 256 * 1024); } rb SEC(".maps"); +static inline struct mq_ctrl *get_mq_ctrl(void) { + struct mq_ctrl *mq_ctrl; + mq_ctrl = bpf_map_lookup_elem(&mq_ctrl_map, &ctrl_key); + if (!mq_ctrl || !mq_ctrl->mq_func) { + return NULL; + } + return mq_ctrl; +} + + // int print_send_info(struct send_events * mq_send_info,int flag){ // bpf_printk("---------------------test----------------------------test--------------------------test--------------------------------------------test---------------------test---------------------test\n"); // bpf_printk("send_msg_prio: %-8lu\n",mq_send_info->msg_prio); @@ -64,6 +74,7 @@ int BPF_KPROBE(mq_timedsend,mqd_t mqdes, const char *u_msg_ptr, size_t msg_len, unsigned int msg_prio, struct timespec64 *ts) { + struct mq_ctrl *mq_ctrl = get_mq_ctrl(); u64 send_enter_time = bpf_ktime_get_ns();//开始发送信息时间； int pid = bpf_get_current_pid_tgid();//发送端pid @@ -83,6 +94,7 @@ int BPF_KPROBE(mq_timedsend,mqd_t mqdes, const char *u_msg_ptr, /*仅获取mq_send_info -> src*/ SEC("kprobe/load_msg") int BPF_KPROBE(load_msg_enter,const void *src, size_t len){ + struct mq_ctrl *mq_ctrl = get_mq_ctrl(); int pid = bpf_get_current_pid_tgid();//发送端pid /*记录load入参src*/ struct send_events *mq_send_info = bpf_map_lookup_elem(&send_msg1, &pid); @@ -97,6 +109,7 @@ int BPF_KPROBE(load_msg_enter,const void *src, size_t len){ /*获取消息块作为key，并建立 message -> mq_send_info 的哈希表*/ SEC("kretprobe/load_msg") int BPF_KRETPROBE(load_msg_exit,void *ret){ + struct mq_ctrl *mq_ctrl = get_mq_ctrl(); int pid = bpf_get_current_pid_tgid();//发送端pid /*构建消息块结构体，作为key*/ struct send_events *mq_send_info = bpf_map_lookup_elem(&send_msg1, &pid); @@ -122,6 +135,7 @@ int BPF_KRETPROBE(load_msg_exit,void *ret){ SEC("kretprobe/do_mq_timedsend") int BPF_KRETPROBE(do_mq_timedsend_exit,void *ret) { + struct mq_ctrl *mq_ctrl = get_mq_ctrl(); bpf_printk("do_mq_timedsend_exit----------------------------------------------------------------\n"); u64 send_exit_time = bpf_ktime_get_ns();//开始发送信息时间； int pid = bpf_get_current_pid_tgid();//发送端pid @@ -149,6 +163,7 @@ int BPF_KPROBE(mq_timedreceive_entry,mqd_t mqdes, const char __user *u_msg_ptr, size_t msg_len, unsigned int msg_prio, struct timespec64 *ts) { + struct mq_ctrl *mq_ctrl = get_mq_ctrl(); u64 rcv_enter_time = bpf_ktime_get_ns(); int pid = bpf_get_current_pid_tgid(); @@ -166,6 +181,7 @@ int BPF_KPROBE(mq_timedreceive_entry,mqd_t mqdes, const char __user *u_msg_ptr, SEC("kprobe/store_msg") int BPF_KPROBE(store_msg,void __user *dest, struct msg_msg *msg, size_t len) { + struct mq_ctrl *mq_ctrl = get_mq_ctrl(); int pid = bpf_get_current_pid_tgid(); /*make key*/ @@ -193,6 +209,7 @@ int BPF_KPROBE(store_msg,void __user *dest, struct msg_msg *msg, size_t len) SEC("kretprobe/do_mq_timedreceive") int BPF_KRETPROBE(do_mq_timedreceive_exit,void *ret){ + struct mq_ctrl *mq_ctrl = get_mq_ctrl(); u64 rcv_exit_time = bpf_ktime_get_ns(); int pid = bpf_get_current_pid_tgid(); u64 send_enter_time,delay; diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/preempt.bpf.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/preempt.bpf.c index 57043e3bb..bf650a6e0 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/preempt.bpf.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/preempt.bpf.c @@ -23,17 +23,27 @@ char LICENSE[] SEC("license") = "Dual BSD/GPL"; #define TIF_NEED_RESCHED 3 - +const int ctrl_key = 0; // 记录时间戳 BPF_HASH(preemptTime, pid_t, u64, 4096); - +BPF_ARRAY(preempt_ctrl_map,int,struct preempt_ctrl,1); struct { __uint(type, BPF_MAP_TYPE_RINGBUF); __uint(max_entries, 256 * 1024); } rb SEC(".maps"); +static inline struct preempt_ctrl *get_preempt_ctrl(void) { + struct preempt_ctrl *preempt_ctrl; + preempt_ctrl = bpf_map_lookup_elem(&preempt_ctrl_map, &ctrl_key); + if (!preempt_ctrl || !preempt_ctrl->preempt_func) { + return NULL; + } + return preempt_ctrl; +} + SEC("tp_btf/sched_switch") int BPF_PROG(sched_switch, bool preempt, struct task_struct *prev, struct task_struct *next) { + struct preempt_ctrl *preempt_ctrl = get_preempt_ctrl(); u64 start_time = bpf_ktime_get_ns(); pid_t prev_pid = BPF_CORE_READ(prev, pid); @@ -52,6 +62,7 @@ int BPF_PROG(sched_switch, bool preempt, struct task_struct *prev, struct task_s // SEC("kprobe/finish_task_switch") SEC("kprobe/finish_task_switch.isra.0") int BPF_KPROBE(finish_task_switch, struct task_struct *prev) { + struct preempt_ctrl *preempt_ctrl = get_preempt_ctrl(); u64 end_time = bpf_ktime_get_ns(); pid_t pid = BPF_CORE_READ(prev, pid); u64 *val; diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/sar.bpf.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/sar.bpf.c index 2fe6b43f5..eb4012def 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/sar.bpf.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/sar.bpf.c @@ -23,7 +23,7 @@ char LICENSE[] SEC("license") = "Dual BSD/GPL"; const volatile long long unsigned int forks_addr = 0; - +const int ctrl_key = 0; #define PF_IDLE 0x00000002 /* I am an IDLE thread */ #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ @@ -51,11 +51,23 @@ BPF_ARRAY(kt_LastTime,u32,u64,1); BPF_ARRAY(ut_LastTime,u32,u64,1); BPF_ARRAY(tick_user,u32,u64,1); BPF_ARRAY(symAddr,u32,u64,1); +BPF_ARRAY(sar_ctrl_map,int,struct sar_ctrl,1); + +static inline struct sar_ctrl *get_sar_ctrl(void) { + struct sar_ctrl *sar_ctrl; + sar_ctrl = bpf_map_lookup_elem(&sar_ctrl_map, &ctrl_key); + if (!sar_ctrl || !sar_ctrl->sar_func) { + return NULL; + } + return sar_ctrl; +} + // 统计fork数 SEC("kprobe/finish_task_switch.isra.0") // SEC("kprobe/finish_task_switch") int kprobe__finish_task_switch(struct pt_regs *ctx) { + struct sar_ctrl *sar_ctrl = get_sar_ctrl(); u32 key = 0; u64 val, *valp = NULL; unsigned long total_forks; @@ -73,6 +85,7 @@ int kprobe__finish_task_switch(struct pt_regs *ctx) //获取进程切换数; SEC("tracepoint/sched/sched_switch") int trace_sched_switch2(struct cswch_args *info) { + struct sar_ctrl *sar_ctrl = get_sar_ctrl(); pid_t prev = info->prev_pid, next = info->next_pid; if (prev != next) { u32 key = 0; @@ -94,6 +107,7 @@ int trace_sched_switch2(struct cswch_args *info) { // SEC("kprobe/finish_task_switch") SEC("kprobe/finish_task_switch.isra.0") int BPF_KPROBE(finish_task_switch,struct task_struct *prev){ + struct sar_ctrl *sar_ctrl = get_sar_ctrl(); pid_t pid=BPF_CORE_READ(prev,pid); u64 *val, time = bpf_ktime_get_ns(); u64 delta; @@ -124,6 +138,7 @@ int BPF_KPROBE(finish_task_switch,struct task_struct *prev){ //统计运行队列长度 SEC("kprobe/update_rq_clock") int BPF_KPROBE(update_rq_clock,struct rq *rq){ + struct sar_ctrl *sar_ctrl = get_sar_ctrl(); u32 key = 0; u64 val = BPF_CORE_READ(rq,nr_running); bpf_map_update_elem(&runqlen,&key,&val,BPF_ANY); @@ -133,6 +148,7 @@ int BPF_KPROBE(update_rq_clock,struct rq *rq){ //软中断 SEC("tracepoint/irq/softirq_entry") int trace_softirq_entry(struct __softirq_info *info) { + struct sar_ctrl *sar_ctrl = get_sar_ctrl(); u32 key = info->vec; u64 val = bpf_ktime_get_ns(); bpf_map_update_elem(&softirqCpuEnterTime, &key, &val, BPF_ANY); @@ -141,6 +157,7 @@ int trace_softirq_entry(struct __softirq_info *info) { SEC("tracepoint/irq/softirq_exit") int trace_softirq_exit(struct __softirq_info *info) { + struct sar_ctrl *sar_ctrl = get_sar_ctrl(); u32 key = info->vec; u64 now = bpf_ktime_get_ns(), *valp = 0; valp =bpf_map_lookup_elem(&softirqCpuEnterTime, &key); @@ -159,6 +176,7 @@ int trace_softirq_exit(struct __softirq_info *info) { 注意这是所有CPU时间的叠加，平均到每个CPU应该除以CPU个数。*/ SEC("tracepoint/irq/irq_handler_entry") int trace_irq_handler_entry(struct __irq_info *info) { + struct sar_ctrl *sar_ctrl = get_sar_ctrl(); u32 key = info->irq; u64 ts = bpf_ktime_get_ns(); bpf_map_update_elem(&irq_cpu_enter_start, &key, &ts, BPF_ANY); @@ -167,6 +185,7 @@ int trace_irq_handler_entry(struct __irq_info *info) { SEC("tracepoint/irq/irq_handler_exit") int trace_irq_handler_exit(struct __irq_info *info) { + struct sar_ctrl *sar_ctrl = get_sar_ctrl(); u32 key = info->irq; u64 now = bpf_ktime_get_ns(), *ts = 0; ts = bpf_map_lookup_elem(&irq_cpu_enter_start, &key); @@ -186,6 +205,7 @@ int trace_irq_handler_exit(struct __irq_info *info) { //tracepoint:power_cpu_idle 表征了CPU进入IDLE的状态，比较准确 SEC("tracepoint/power/cpu_idle") int trace_cpu_idle(struct idleStruct *pIDLE) { + struct sar_ctrl *sar_ctrl = get_sar_ctrl(); u64 delta, time = bpf_ktime_get_ns(); u32 key = pIDLE->cpu_id; if (pIDLE->state == -1) { @@ -215,6 +235,7 @@ static __always_inline int user_mode(struct pt_regs *regs) // 两个CPU各自会产生一个调用，这正好方便我们使用 SEC("perf_event") int tick_update(struct pt_regs *ctx) { + struct sar_ctrl *sar_ctrl = get_sar_ctrl(); // bpf_trace_printk("cs_rpl = %x\n", ctx->cs & 3); u32 key = 0; diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/sc_delay.bpf.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/sc_delay.bpf.c index 23eaf2200..ee224283a 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/sc_delay.bpf.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/sc_delay.bpf.c @@ -15,27 +15,38 @@ // author: albert_xuu@163.com zhangxy1016304@163.com zhangziheng0525@163.com #include "vmlinux.h" -#include //包含了BPF 辅助函数 +#include #include #include "cpu_watcher.h" char LICENSE[] SEC("license") = "Dual BSD/GPL"; -// 定义数组映射 -//BPF_PERCPU_HASH(SyscallEnterTime,pid_t,struct syscall_flags,512);//记录时间戳 -BPF_PERCPU_HASH(SyscallEnterTime,pid_t,u64,512);//记录时间戳 + +const int ctrl_key = 0; +BPF_PERCPU_HASH(SyscallEnterTime,pid_t,u64,512); BPF_PERCPU_HASH(Events,pid_t,u64,10); +BPF_ARRAY(sc_ctrl_map,int,struct sc_ctrl,1); struct { __uint(type, BPF_MAP_TYPE_RINGBUF); __uint(max_entries, 256 * 1024); } rb SEC(".maps");//环形缓冲区； +static inline struct sc_ctrl *get_sc_ctrl(void) { + struct sc_ctrl *sc_ctrl; + sc_ctrl = bpf_map_lookup_elem(&sc_ctrl_map, &ctrl_key); + if (!sc_ctrl || !sc_ctrl->sc_func) { + return NULL; + } + return sc_ctrl; +} + -SEC("tracepoint/raw_syscalls/sys_enter")//进入系统调用 +SEC("tracepoint/raw_syscalls/sys_enter") int tracepoint__syscalls__sys_enter(struct trace_event_raw_sys_enter *args){ - u64 start_time = bpf_ktime_get_ns()/1000;//ms - pid_t pid = bpf_get_current_pid_tgid();//获取到当前进程的pid + struct sc_ctrl *sc_ctrl = get_sc_ctrl(); + u64 start_time = bpf_ktime_get_ns()/1000; + pid_t pid = bpf_get_current_pid_tgid(); u64 syscall_id = (u64)args->id; //bpf_printk("ID:%ld\n",syscall_id); @@ -44,13 +55,13 @@ int tracepoint__syscalls__sys_enter(struct trace_event_raw_sys_enter *args){ return 0; } -SEC("tracepoint/raw_syscalls/sys_exit")//退出系统调用 +SEC("tracepoint/raw_syscalls/sys_exit") int tracepoint__syscalls__sys_exit(struct trace_event_raw_sys_exit *args){ - u64 exit_time = bpf_ktime_get_ns()/1000;//ms - pid_t pid = bpf_get_current_pid_tgid() ;//获取到当前进程的pid + struct sc_ctrl *sc_ctrl = get_sc_ctrl(); + u64 exit_time = bpf_ktime_get_ns()/1000; + pid_t pid = bpf_get_current_pid_tgid() ; u64 syscall_id; u64 start_time, delay; - u64 *val = bpf_map_lookup_elem(&SyscallEnterTime, &pid); if(val !=0){ start_time = *val; @@ -59,7 +70,6 @@ int tracepoint__syscalls__sys_exit(struct trace_event_raw_sys_exit *args){ }else{ return 0; } - u64 *val2 = bpf_map_lookup_elem(&Events, &pid); if(val2 !=0){ syscall_id = *val2; @@ -67,19 +77,13 @@ int tracepoint__syscalls__sys_exit(struct trace_event_raw_sys_exit *args){ }else{ return 0; } - - struct syscall_events *e; e = bpf_ringbuf_reserve(&rb, sizeof(*e), 0); if (!e) return 0; - e->pid = pid; e->delay = delay; bpf_get_current_comm(&e->comm, sizeof(e->comm)); e->syscall_id = syscall_id; - bpf_ringbuf_submit(e, 0); - - return 0; } diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/schedule_delay.bpf.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/schedule_delay.bpf.c index 596158c6e..db1b2a363 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/schedule_delay.bpf.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/bpf/schedule_delay.bpf.c @@ -28,14 +28,21 @@ BPF_HASH(has_scheduled,struct proc_id, bool, 10240);//记录该进程是否调 BPF_HASH(enter_schedule,struct proc_id, struct schedule_event, 10240);//记录该进程上运行队列的时间 BPF_ARRAY(sys_schedule,int,struct sum_schedule,1);//记录整个系统的调度延迟 BPF_ARRAY(threshold_schedule,int,struct proc_schedule,10240);//记录每个进程的调度延迟 +BPF_HASH(proc_histories,struct proc_id, struct proc_history, 10240);//记录每个进程运行前的两个进程 BPF_ARRAY(schedule_ctrl_map,int,struct schedule_ctrl,1); +static inline struct schedule_ctrl *get_schedule_ctrl(void) { + struct schedule_ctrl *sched_ctrl; + sched_ctrl = bpf_map_lookup_elem(&schedule_ctrl_map, &ctrl_key); + if (!sched_ctrl || !sched_ctrl->schedule_func) { + return NULL; + } + return sched_ctrl; +}//查找控制结构体 + SEC("tp_btf/sched_wakeup") int BPF_PROG(sched_wakeup, struct task_struct *p) { - struct schedule_ctrl *sched_ctrl; - sched_ctrl = bpf_map_lookup_elem(&schedule_ctrl_map,&ctrl_key); - if(!sched_ctrl || !sched_ctrl->schedule_func) - return 0; + struct schedule_ctrl *sched_ctrl = get_schedule_ctrl(); pid_t pid = p->pid; int cpu = bpf_get_smp_processor_id(); struct schedule_event *schedule_event; @@ -62,7 +69,7 @@ int BPF_PROG(sched_wakeup, struct task_struct *p) { SEC("tp_btf/sched_wakeup_new") int BPF_PROG(sched_wakeup_new, struct task_struct *p) { - struct schedule_ctrl *sched_ctrl; + struct schedule_ctrl *sched_ctrl = get_schedule_ctrl(); sched_ctrl = bpf_map_lookup_elem(&schedule_ctrl_map,&ctrl_key); if(!sched_ctrl || !sched_ctrl->schedule_func) return 0; @@ -86,11 +93,9 @@ int BPF_PROG(sched_wakeup_new, struct task_struct *p) { SEC("tp_btf/sched_switch") int BPF_PROG(sched_switch, bool preempt, struct task_struct *prev, struct task_struct *next) { - struct schedule_ctrl *sched_ctrl; - sched_ctrl = bpf_map_lookup_elem(&schedule_ctrl_map,&ctrl_key); - if(!sched_ctrl || !sched_ctrl->schedule_func) - return 0; - + struct schedule_ctrl *sched_ctrl = get_schedule_ctrl(); + struct proc_history *history; + struct proc_history new_history; u64 current_time = bpf_ktime_get_ns(); pid_t prev_pid = prev->pid; unsigned int prev_state = prev->__state; @@ -138,8 +143,8 @@ int BPF_PROG(sched_switch, bool preempt, struct task_struct *prev, struct task_s } delay = current_time - schedule_event->enter_time; struct proc_schedule proc_schedule; - proc_schedule.pid = next_pid; proc_schedule.delay = delay; + proc_schedule.id= next_id; bpf_probe_read_kernel_str(&proc_schedule.proc_name, sizeof(proc_schedule.proc_name), next->comm); bpf_map_update_elem(&threshold_schedule, &key, &proc_schedule, BPF_ANY); sum_schedule = bpf_map_lookup_elem(&sys_schedule, &key); @@ -172,16 +177,27 @@ int BPF_PROG(sched_switch, bool preempt, struct task_struct *prev, struct task_s } } } + history = bpf_map_lookup_elem(&proc_histories, &next_id); + if (history) { + // 如果找到了，更新历史记录 + new_history.last[0] = history->last[1]; + new_history.last[1].pid = prev->pid; + bpf_probe_read_kernel_str(&new_history.last[1].comm, sizeof(new_history.last[1].comm), prev->comm); + bpf_map_update_elem(&proc_histories, &next_id, &new_history, BPF_ANY); + } else { + // 如果没有找到，初始化新的历史记录 + new_history.last[0].pid = 0; // 初始化为0，表示没有历史信息 + new_history.last[0].comm[0] = '\0'; + new_history.last[1].pid = prev->pid; + bpf_probe_read_kernel_str(&new_history.last[1].comm, sizeof(new_history.last[1].comm), prev->comm); + bpf_map_update_elem(&proc_histories, &next_id, &new_history, BPF_ANY); + } return 0; } SEC("tracepoint/sched/sched_process_exit") int sched_process_exit(void *ctx) { - struct schedule_ctrl *sched_ctrl; - sched_ctrl = bpf_map_lookup_elem(&schedule_ctrl_map,&ctrl_key); - if(!sched_ctrl || !sched_ctrl->schedule_func) - return 0; - + struct schedule_ctrl *sched_ctrl = get_schedule_ctrl(); struct task_struct *p = (struct task_struct *)bpf_get_current_task(); pid_t pid = BPF_CORE_READ(p, pid); int cpu = bpf_get_smp_processor_id(); diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/controller.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/controller.c index b02a986ce..7c8cde155 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/controller.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/controller.c @@ -30,6 +30,7 @@ static struct env { // 1代表activate；2代表unactivate；3代表finish int usemode; bool SAR; + bool percent; bool CS_DELAY; bool SYSCALL_DELAY; bool MIN_US_SET; @@ -41,6 +42,7 @@ static struct env { } env = { .usemode = 0, .SAR = false, + .percent = false, .CS_DELAY = false, .SYSCALL_DELAY = false, .MIN_US_SET = false, @@ -58,22 +60,13 @@ static const struct argp_option opts[] = { { "unactivate", 'u', NULL, 0, "Initialize to the original unactivated state" }, { "finish", 'f', NULL, 0, "Finish to run eBPF tool" }, {"libbpf_sar", 's', 0, 0, "Print sar_info (the data of cpu)" }, + {"percent", 'P', 0, 0, "Format data as percentages" }, {"cs_delay", 'c', 0, 0, "Print cs_delay (the data of cpu)" }, {"syscall_delay", 'S', 0, 0, "Print syscall_delay (the data of syscall)" }, {"preempt_time", 'p', 0, 0, "Print preempt_time (the data of preempt_schedule)" }, {"schedule_delay", 'd', 0, 0, "Print schedule_delay (the data of cpu)" }, {"schedule_delay_min_us_set", 'e', "THRESHOLD", 0, "Print scheduling delays that exceed the threshold (the data of cpu)" }, {"mq_delay", 'm', 0, 0, "Print mq_delay(the data of proc)" }, - // { "pid", 'p', "PID", 0, "Process ID to trace" }, - // { "tgid", 'P', "TGID", 0, "Thread group to trace" }, - // { "cpuid", 'c', "CPUID", 0, "Set For Tracing per-CPU Process(other processes don't need to set this parameter)" }, - // { "time", 't', "TIME-SEC", 0, "Max Running Time(0 for infinite)" }, - // { "myproc", 'm', NULL, 0, "Trace the process of the tool itself (not tracked by default)" }, - // { "resource", 'r', NULL, 0, "Collects resource usage information about processes" }, - // { "keytime", 'k', "KEYTIME", 0, "Collects keytime information about processes(0:except CPU kt_info,1:all kt_info,any 0 or 1 when deactivated)" }, - // { "lock", 'l', NULL, 0, "Collects lock information about processes" }, - // { "syscall", 's', "SYSCALLS", 0, "Collects syscall sequence (1~50) information about processes(any 1~50 when deactivated)" }, - // { "schedule", 'S', NULL, 0, "Collects schedule information about processes (trace tool process)" }, { NULL, 'h', NULL, OPTION_HIDDEN, "show the full help" }, {}, }; @@ -93,6 +86,8 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) case 's': env.SAR = true; break; + case 'P': + env.percent = true; case 'c': env.CS_DELAY = true; break; @@ -134,7 +129,7 @@ int deactivate_mode(){ int err; if(env.SAR){ - struct sar_ctrl sar_ctrl = {false,0}; + struct sar_ctrl sar_ctrl = {false,false,0}; err = update_sar_ctrl_map(sar_ctrl); if(err < 0) return err; } @@ -190,9 +185,9 @@ int main(int argc, char **argv) if(env.usemode == 1){ // activate mode if(env.SAR){ - struct sar_ctrl sar_ctrl = {true,SAR_WACTHER}; - err = update_sar_ctrl_map(sar_ctrl); - if(err < 0) return err; + struct sar_ctrl sar_ctrl = {true,env.percent,SAR_WACTHER+env.percent}; + err = update_sar_ctrl_map(sar_ctrl); + if(err < 0) return err; } if(env.CS_DELAY){ diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c index 23114d93c..0cdf445f5 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher.c @@ -55,13 +55,10 @@ struct msg_msg { static struct env { int time; int period; - bool percent; bool enable_proc; bool SAR; bool CS_DELAY; bool SYSCALL_DELAY; - bool MIN_US_SET; - int MIN_US; bool PREEMPT; bool SCHEDULE_DELAY; bool MQ_DELAY; @@ -71,13 +68,10 @@ static struct env { } env = { .time = 0, .period = 1, - .percent = false, .enable_proc = false, .SAR = false, .CS_DELAY = false, .SYSCALL_DELAY = false, - .MIN_US_SET = false, - .MIN_US = 10000, .PREEMPT = false, .SCHEDULE_DELAY = false, .MQ_DELAY = false, @@ -97,9 +91,11 @@ struct mq_delay_bpf *mq_skel; static int csmap_fd; static int sarmap_fd; +struct sar_ctrl sar_ctrl= {}; static int scmap_fd; static int preemptmap_fd; static int schedulemap_fd; +struct schedule_ctrl sd_ctrl = {}; static int mqmap_fd; //static int prev_watcher = 0;//上一个使用的工具，用于在切换使用功能时，打印不用功能的表头； @@ -130,7 +126,6 @@ const char argp_program_doc[] = "cpu watcher is in use ....\n"; static const struct argp_option opts[] = { { "time", 't', "TIME-SEC", 0, "Max Running Time(0 for infinite)" }, { "period", 'i', "INTERVAL", 0, "Period interval in seconds" }, - {"percent", 'P', 0, 0, "Format data as percentages" }, {"libbpf_sar", 's', 0, 0, "Print sar_info (the data of cpu)" }, {"cs_delay", 'c', 0, 0, "Print cs_delay (the data of cpu)" }, {"syscall_delay", 'S', 0, 0, "Print syscall_delay (the data of syscall)" }, @@ -153,9 +148,6 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) case 'i': env.period = strtol(arg, NULL, 10); break; - case 'P': - env.percent = true; - break; case 's': env.SAR = true; break; @@ -264,6 +256,28 @@ u64 find_ksym(const char* target_symbol) { static int print_all() { + int err,key=0; + err = bpf_map_lookup_elem(sarmap_fd, &key, &sar_ctrl); + if (err < 0) { + fprintf(stderr, "failed to lookup infos: %d\n", err); + return -1; + } + if(!sar_ctrl.sar_func) return 0; + if(sar_ctrl.prev_watcher == SAR_WACTHER + 1) { + printf(" time proc/s cswch/s runqlen irqTime/%% softirq/%% idle/%% kthread/%% sysc/%% utime/%% sys/%% \n"); + sar_ctrl.prev_watcher = SAR_WACTHER + 2; + err = bpf_map_update_elem(sarmap_fd, &key, &sar_ctrl, 0); + if(err < 0){ + fprintf(stderr, "Failed to update elem\n"); + } + }else if (sar_ctrl.prev_watcher == SAR_WACTHER){ + printf(" time proc/s cswch/s runqlen irqTime/us softirq/us idle/ms kthread/us sysc/ms utime/ms sys/ms \n"); + sar_ctrl.prev_watcher = SAR_WACTHER + 2; + err = bpf_map_update_elem(sarmap_fd, &key, &sar_ctrl, 0); + if(err < 0){ + fprintf(stderr, "Failed to update elem\n"); + } + } int nprocs = get_nprocs(); /*proc:*/ int key_proc = 1; @@ -380,7 +394,7 @@ static int print_all() if(env.enable_proc){ time_t now = time(NULL); struct tm *localTime = localtime(&now); - if (env.percent == true){ + if (sar_ctrl.percent == true){ printf("%02d:%02d:%02d %8llu %8llu %6d ",localTime->tm_hour, localTime->tm_min, localTime->tm_sec,__proc, __sched, runqlen); // 大于百分之60的标红输出 double values[7] = { @@ -567,9 +581,7 @@ void add_entry(int pid, const char *comm, long long delay) { } static int schedule_print() { - int err,key = 0; - struct schedule_ctrl sd_ctrl = {}; err = bpf_map_lookup_elem(schedulemap_fd,&key,&sd_ctrl); if (err < 0) { fprintf(stderr, "failed to lookup infos: %d\n", err); @@ -620,18 +632,34 @@ static int schedule_print() } else{ struct proc_schedule info; + struct proc_id id_key; + struct proc_history prev_info; int key = 0; - int err, fd = bpf_map__fd(sd_skel->maps.threshold_schedule); - err = bpf_map_lookup_elem(fd, &key, &info); + int err, fd1 = bpf_map__fd(sd_skel->maps.threshold_schedule),fd2 = bpf_map__fd(sd_skel->maps.proc_histories); + err = bpf_map_lookup_elem(fd1, &key, &info); if (err < 0) { fprintf(stderr, "failed to lookup infos: %d\n", err); return -1; } - if (info.delay / 1000>sd_ctrl.min_us&&info.pid!=0) { - if (!entry_exists(info.pid, info.proc_name, info.delay / 1000)) { - printf("%-10d %-16s %15lld\n", info.pid, info.proc_name, info.delay / 1000); - add_entry(info.pid, info.proc_name, info.delay / 1000); + if (info.delay / 1000 > sd_ctrl.min_us&&info.id.pid!=0) { + id_key.pid = info.id.pid; + id_key.cpu_id = info.id.cpu_id; + err = bpf_map_lookup_elem(fd2, &id_key, &prev_info); + if (err < 0) { + fprintf(stderr, "Failed to lookup proc_histories with PID %d and CPU ID %d: %d\n", id_key.pid, id_key.cpu_id, err); + return -1; + } + if (!entry_exists(info.id.pid, info.proc_name, info.delay / 1000)) { + printf("%-10d %-16s %15lld", info.id.pid, info.proc_name, info.delay / 1000); + add_entry(info.id.pid, info.proc_name, info.delay / 1000); + for (int i = 0; i < 2; i++) { + if (prev_info.last[i].pid != 0) { + printf(" Previous Process %d: PID=%-10d Name=%-16s ", i+1, prev_info.last[i].pid, prev_info.last[i].comm); + } + } + printf("\n"); } + } } @@ -878,7 +906,7 @@ int main(int argc, char **argv) goto sar_cleanup; } sarmap_fd = bpf_map__fd(sar_ctrl_map); - struct sar_ctrl init_value = {false,SAR_WACTHER}; + struct sar_ctrl init_value = {false,false,SAR_WACTHER}; err = bpf_map_update_elem(sarmap_fd, &key, &init_value, 0); if(err < 0){ fprintf(stderr, "Failed to update elem\n"); @@ -891,9 +919,6 @@ int main(int argc, char **argv) fprintf(stderr, "Failed to attach BPF skeleton\n"); goto sar_cleanup; } - if (env.percent){ - printf(" time proc/s cswch/s runqlen irqTime/%% softirq/%% idle/%% kthread/%% sysc/%% utime/%% sys/%% \n"); - }else{printf(" time proc/s cswch/s runqlen irqTime/us softirq/us idle/ms kthread/us sysc/ms utime/ms sys/ms \n");} }else if(env.MQ_DELAY){ /* Load and verify BPF application */ mq_skel = mq_delay_bpf__open(); @@ -1013,7 +1038,7 @@ int main(int argc, char **argv) if (err < 0) { break; } - if(env.SCHEDULE_DELAY){ + if(env.SCHEDULE_DELAY&&!sd_ctrl.min_us_set){ sleep(1); } } diff --git a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h index 00a6463ec..eadd0d874 100644 --- a/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h +++ b/eBPF_Supermarket/CPU_Subsystem/cpu_watcher/include/cpu_watcher.h @@ -141,11 +141,21 @@ struct sum_schedule { }; struct proc_schedule { - int pid; + struct proc_id id; unsigned long long delay; char proc_name[TASK_COMM_LEN]; }; +struct proc_info { + pid_t pid; + char comm[TASK_COMM_LEN]; +}; + +struct proc_history { + struct proc_info last[2]; // 存储最后两个调度的进程信息 +}; + + /*----------------------------------------------*/ /* mq_delay相关结构体 */ /*----------------------------------------------*/ @@ -226,6 +236,7 @@ struct idleStruct { /*----------------------------------------------*/ struct sar_ctrl{ bool sar_func; + bool percent; int prev_watcher; };