Loading drivers/soc/qcom/msm_performance.c +281 −5 Original line number Diff line number Diff line Loading @@ -20,9 +20,11 @@ #include <linux/sched/core_ctl.h> #include <soc/qcom/msm_performance.h> #include <linux/spinlock.h> #include <linux/jiffies.h> #include <linux/circ_buf.h> #include <linux/ktime.h> #include <linux/perf_event.h> #include <linux/errno.h> #include <linux/topology.h> /* * Sched will provide the data for every 20ms window, Loading @@ -32,6 +34,16 @@ #define POLL_INT 25 #define NODE_NAME_MAX_CHARS 16 #define QUEUE_POOL_SIZE 512 /*2^8 always keep in 2^x */ #define INST_EV 0x08 /* 0th event*/ #define CYC_EV 0x11 /* 1st event*/ static DEFINE_PER_CPU(bool, cpu_is_idle); static DEFINE_PER_CPU(bool, cpu_is_hp); enum event_idx { INST_EVENT, CYC_EVENT, NO_OF_EVENT }; enum cpu_clusters { MIN = 0, Loading Loading @@ -68,6 +80,13 @@ static struct queue_indicies curr_pos; static DEFINE_SPINLOCK(gfx_circ_buff_lock); static struct event_data { struct perf_event *pevent; u64 prev_count; u64 cur_delta; u64 cached_total_count; } pmu_events[NO_OF_EVENT][NR_CPUS]; struct events { spinlock_t cpu_hotplug_lock; bool cpu_hotplug; Loading @@ -80,6 +99,7 @@ static unsigned int aggr_big_nr; static unsigned int aggr_top_load; static unsigned int top_load[CLUSTER_MAX]; static unsigned int curr_cap[CLUSTER_MAX]; static bool max_cap_cpus[NR_CPUS]; static cpumask_var_t limit_mask_min; static cpumask_var_t limit_mask_max; Loading Loading @@ -441,13 +461,221 @@ static struct attribute_group notify_attr_group = { static struct kobject *notify_kobj; /*******************************sysfs ends************************************/ static int hotplug_notify(unsigned int cpu) /*****************PMU Data Collection*****************/ static struct perf_event_attr *msm_perf_alloc_attr(void) { struct perf_event_attr *attr = NULL; attr = kzalloc(sizeof(struct perf_event_attr), GFP_KERNEL); if (!attr) return attr; attr->type = PERF_TYPE_RAW; attr->size = sizeof(struct perf_event_attr); attr->pinned = 1; return attr; } static int set_event(struct event_data *ev, int cpu, struct perf_event_attr *attr) { struct perf_event *pevent; pevent = perf_event_create_kernel_counter(attr, cpu, NULL, NULL, NULL); if (IS_ERR(pevent)) { pr_err("msm_perf: %s failed, eventId:0x%x, cpu:%d, error code:%ld\n", __func__, attr->config, cpu, PTR_ERR(pevent)); return PTR_ERR(pevent); } ev->pevent = pevent; perf_event_enable(pevent); return 0; } static void free_pmu_counters(unsigned int cpu) { int i = 0; for (i = 0; i < NO_OF_EVENT; i++) { pmu_events[i][cpu].prev_count = 0; pmu_events[i][cpu].cur_delta = 0; pmu_events[i][cpu].cached_total_count = 0; if (pmu_events[i][cpu].pevent) { perf_event_disable(pmu_events[i][cpu].pevent); perf_event_release_kernel(pmu_events[i][cpu].pevent); pmu_events[i][cpu].pevent = NULL; } } } static int init_pmu_counter(void) { struct perf_event_attr *attr = msm_perf_alloc_attr(); int cpu; unsigned long cpu_capacity[NR_CPUS]; unsigned long min_cpu_capacity = ULONG_MAX; int ret = 0; if (!attr) return -ENOMEM; /* Create events per CPU */ for_each_possible_cpu(cpu) { /* create Instruction event */ attr->config = INST_EV; ret = set_event(&pmu_events[INST_EVENT][cpu], cpu, attr); if (ret < 0) { kfree(attr); return ret; } /* create cycle event */ attr->config = CYC_EV; ret = set_event(&pmu_events[CYC_EVENT][cpu], cpu, attr); if (ret < 0) { free_pmu_counters(cpu); kfree(attr); return ret; } /* find capacity per cpu */ cpu_capacity[cpu] = arch_scale_cpu_capacity(cpu); if (cpu_capacity[cpu] < min_cpu_capacity) min_cpu_capacity = cpu_capacity[cpu]; } /* determine cpu index for maximum capacity cpus */ for_each_possible_cpu(cpu) { if (cpu_capacity[cpu] > min_cpu_capacity) max_cap_cpus[cpu] = true; } kfree(attr); return 0; } static inline void msm_perf_read_event(struct event_data *event) { u64 ev_count = 0; u64 total, enabled, running; if (!event->pevent) return; if (!per_cpu(cpu_is_idle, event->pevent->cpu) && !per_cpu(cpu_is_hp, event->pevent->cpu)) total = perf_event_read_value(event->pevent, &enabled, &running); else total = event->cached_total_count; ev_count = total - event->prev_count; event->prev_count = total; event->cur_delta = ev_count; } static int get_cpu_total_instruction(char *buf, const struct kernel_param *kp) { u64 instruction = 0; u64 cycles = 0; u64 total_inst_big = 0; u64 total_inst_little = 0; u64 ipc_big = 0; u64 ipc_little = 0; int cnt = 0, cpu; for_each_possible_cpu(cpu) { /* Read Instruction event */ msm_perf_read_event(&pmu_events[INST_EVENT][cpu]); /* Read Cycle event */ msm_perf_read_event(&pmu_events[CYC_EVENT][cpu]); instruction = pmu_events[INST_EVENT][cpu].cur_delta; cycles = pmu_events[CYC_EVENT][cpu].cur_delta; /* collecting max inst and ipc for max cap and min cap cpus */ if (max_cap_cpus[cpu]) { if (cycles) ipc_big = max(ipc_big, ((instruction*100)/cycles)); total_inst_big += instruction; } else { if (cycles) ipc_little = max(ipc_little, ((instruction*100)/cycles)); total_inst_little += instruction; } } cnt += scnprintf(buf, PAGE_SIZE, "%llu:%llu:%llu:%llu\n", total_inst_big, ipc_big, total_inst_little, ipc_little); return cnt; } static const struct kernel_param_ops param_ops_cpu_total_instruction = { .set = NULL, .get = get_cpu_total_instruction, }; module_param_cb(inst, ¶m_ops_cpu_total_instruction, NULL, 0444); static int restart_events(unsigned int cpu, bool cpu_up) { struct perf_event_attr *attr = msm_perf_alloc_attr(); int ret = 0; if (!attr) return -ENOMEM; if (cpu_up) { /* create Instruction event */ attr->config = INST_EV; ret = set_event(&pmu_events[INST_EVENT][cpu], cpu, attr); if (ret < 0) { kfree(attr); return ret; } /* create cycle event */ attr->config = CYC_EV; ret = set_event(&pmu_events[CYC_EVENT][cpu], cpu, attr); if (ret < 0) { free_pmu_counters(cpu); kfree(attr); return ret; } } else { free_pmu_counters(cpu); } kfree(attr); return 0; } static int hotplug_notify_down(unsigned int cpu) { unsigned long flags; if (events_group.init_success) { spin_lock_irqsave(&(events_group.cpu_hotplug_lock), flags); per_cpu(cpu_is_hp, cpu) = true; restart_events(cpu, false); spin_unlock_irqrestore(&(events_group.cpu_hotplug_lock), flags); } return 0; } static int hotplug_notify_up(unsigned int cpu) { unsigned long flags; if (events_group.init_success) { spin_lock_irqsave(&(events_group.cpu_hotplug_lock), flags); events_group.cpu_hotplug = true; restart_events(cpu, true); per_cpu(cpu_is_hp, cpu) = false; spin_unlock_irqrestore(&(events_group.cpu_hotplug_lock), flags); wake_up_process(events_notify_thread); } Loading @@ -455,6 +683,42 @@ static int hotplug_notify(unsigned int cpu) return 0; } static int msm_perf_idle_read_events(unsigned int cpu) { int ret = 0, i; for (i = 0; i < NO_OF_EVENT; i++) { if (pmu_events[i][cpu].pevent) ret = perf_event_read_local(pmu_events[i][cpu].pevent, &pmu_events[i][cpu].cached_total_count, NULL, NULL); } return ret; } static int msm_perf_idle_notif(struct notifier_block *nb, unsigned long action, void *data) { int ret = NOTIFY_OK; int cpu = smp_processor_id(); switch (action) { case IDLE_START: __this_cpu_write(cpu_is_idle, true); if (!per_cpu(cpu_is_hp, cpu)) ret = msm_perf_idle_read_events(cpu); break; case IDLE_END: __this_cpu_write(cpu_is_idle, false); break; } return NOTIFY_OK; } static struct notifier_block msm_perf_event_idle_nb = { .notifier_call = msm_perf_idle_notif, }; static int events_notify_userspace(void *data) { unsigned long flags; Loading Loading @@ -714,7 +978,9 @@ module_param_cb(splh_notif, ¶m_ops_splh_notification, &splh_notif, 0644); static int __init msm_performance_init(void) { unsigned int cpu; int ret; if (!alloc_cpumask_var(&limit_mask_min, GFP_KERNEL)) return -ENOMEM; Loading @@ -723,14 +989,24 @@ static int __init msm_performance_init(void) return -ENOMEM; } ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE, get_online_cpus(); for_each_possible_cpu(cpu) { if (!cpumask_test_cpu(cpu, cpu_online_mask)) per_cpu(cpu_is_hp, cpu) = true; } ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "msm_performance_cpu_hotplug", hotplug_notify, NULL); hotplug_notify_up, hotplug_notify_down); put_online_cpus(); init_events_group(); init_notify_group(); init_pmu_counter(); idle_notifier_register(&msm_perf_event_idle_nb); return 0; } late_initcall(msm_performance_init); Loading
drivers/soc/qcom/msm_performance.c +281 −5 Original line number Diff line number Diff line Loading @@ -20,9 +20,11 @@ #include <linux/sched/core_ctl.h> #include <soc/qcom/msm_performance.h> #include <linux/spinlock.h> #include <linux/jiffies.h> #include <linux/circ_buf.h> #include <linux/ktime.h> #include <linux/perf_event.h> #include <linux/errno.h> #include <linux/topology.h> /* * Sched will provide the data for every 20ms window, Loading @@ -32,6 +34,16 @@ #define POLL_INT 25 #define NODE_NAME_MAX_CHARS 16 #define QUEUE_POOL_SIZE 512 /*2^8 always keep in 2^x */ #define INST_EV 0x08 /* 0th event*/ #define CYC_EV 0x11 /* 1st event*/ static DEFINE_PER_CPU(bool, cpu_is_idle); static DEFINE_PER_CPU(bool, cpu_is_hp); enum event_idx { INST_EVENT, CYC_EVENT, NO_OF_EVENT }; enum cpu_clusters { MIN = 0, Loading Loading @@ -68,6 +80,13 @@ static struct queue_indicies curr_pos; static DEFINE_SPINLOCK(gfx_circ_buff_lock); static struct event_data { struct perf_event *pevent; u64 prev_count; u64 cur_delta; u64 cached_total_count; } pmu_events[NO_OF_EVENT][NR_CPUS]; struct events { spinlock_t cpu_hotplug_lock; bool cpu_hotplug; Loading @@ -80,6 +99,7 @@ static unsigned int aggr_big_nr; static unsigned int aggr_top_load; static unsigned int top_load[CLUSTER_MAX]; static unsigned int curr_cap[CLUSTER_MAX]; static bool max_cap_cpus[NR_CPUS]; static cpumask_var_t limit_mask_min; static cpumask_var_t limit_mask_max; Loading Loading @@ -441,13 +461,221 @@ static struct attribute_group notify_attr_group = { static struct kobject *notify_kobj; /*******************************sysfs ends************************************/ static int hotplug_notify(unsigned int cpu) /*****************PMU Data Collection*****************/ static struct perf_event_attr *msm_perf_alloc_attr(void) { struct perf_event_attr *attr = NULL; attr = kzalloc(sizeof(struct perf_event_attr), GFP_KERNEL); if (!attr) return attr; attr->type = PERF_TYPE_RAW; attr->size = sizeof(struct perf_event_attr); attr->pinned = 1; return attr; } static int set_event(struct event_data *ev, int cpu, struct perf_event_attr *attr) { struct perf_event *pevent; pevent = perf_event_create_kernel_counter(attr, cpu, NULL, NULL, NULL); if (IS_ERR(pevent)) { pr_err("msm_perf: %s failed, eventId:0x%x, cpu:%d, error code:%ld\n", __func__, attr->config, cpu, PTR_ERR(pevent)); return PTR_ERR(pevent); } ev->pevent = pevent; perf_event_enable(pevent); return 0; } static void free_pmu_counters(unsigned int cpu) { int i = 0; for (i = 0; i < NO_OF_EVENT; i++) { pmu_events[i][cpu].prev_count = 0; pmu_events[i][cpu].cur_delta = 0; pmu_events[i][cpu].cached_total_count = 0; if (pmu_events[i][cpu].pevent) { perf_event_disable(pmu_events[i][cpu].pevent); perf_event_release_kernel(pmu_events[i][cpu].pevent); pmu_events[i][cpu].pevent = NULL; } } } static int init_pmu_counter(void) { struct perf_event_attr *attr = msm_perf_alloc_attr(); int cpu; unsigned long cpu_capacity[NR_CPUS]; unsigned long min_cpu_capacity = ULONG_MAX; int ret = 0; if (!attr) return -ENOMEM; /* Create events per CPU */ for_each_possible_cpu(cpu) { /* create Instruction event */ attr->config = INST_EV; ret = set_event(&pmu_events[INST_EVENT][cpu], cpu, attr); if (ret < 0) { kfree(attr); return ret; } /* create cycle event */ attr->config = CYC_EV; ret = set_event(&pmu_events[CYC_EVENT][cpu], cpu, attr); if (ret < 0) { free_pmu_counters(cpu); kfree(attr); return ret; } /* find capacity per cpu */ cpu_capacity[cpu] = arch_scale_cpu_capacity(cpu); if (cpu_capacity[cpu] < min_cpu_capacity) min_cpu_capacity = cpu_capacity[cpu]; } /* determine cpu index for maximum capacity cpus */ for_each_possible_cpu(cpu) { if (cpu_capacity[cpu] > min_cpu_capacity) max_cap_cpus[cpu] = true; } kfree(attr); return 0; } static inline void msm_perf_read_event(struct event_data *event) { u64 ev_count = 0; u64 total, enabled, running; if (!event->pevent) return; if (!per_cpu(cpu_is_idle, event->pevent->cpu) && !per_cpu(cpu_is_hp, event->pevent->cpu)) total = perf_event_read_value(event->pevent, &enabled, &running); else total = event->cached_total_count; ev_count = total - event->prev_count; event->prev_count = total; event->cur_delta = ev_count; } static int get_cpu_total_instruction(char *buf, const struct kernel_param *kp) { u64 instruction = 0; u64 cycles = 0; u64 total_inst_big = 0; u64 total_inst_little = 0; u64 ipc_big = 0; u64 ipc_little = 0; int cnt = 0, cpu; for_each_possible_cpu(cpu) { /* Read Instruction event */ msm_perf_read_event(&pmu_events[INST_EVENT][cpu]); /* Read Cycle event */ msm_perf_read_event(&pmu_events[CYC_EVENT][cpu]); instruction = pmu_events[INST_EVENT][cpu].cur_delta; cycles = pmu_events[CYC_EVENT][cpu].cur_delta; /* collecting max inst and ipc for max cap and min cap cpus */ if (max_cap_cpus[cpu]) { if (cycles) ipc_big = max(ipc_big, ((instruction*100)/cycles)); total_inst_big += instruction; } else { if (cycles) ipc_little = max(ipc_little, ((instruction*100)/cycles)); total_inst_little += instruction; } } cnt += scnprintf(buf, PAGE_SIZE, "%llu:%llu:%llu:%llu\n", total_inst_big, ipc_big, total_inst_little, ipc_little); return cnt; } static const struct kernel_param_ops param_ops_cpu_total_instruction = { .set = NULL, .get = get_cpu_total_instruction, }; module_param_cb(inst, ¶m_ops_cpu_total_instruction, NULL, 0444); static int restart_events(unsigned int cpu, bool cpu_up) { struct perf_event_attr *attr = msm_perf_alloc_attr(); int ret = 0; if (!attr) return -ENOMEM; if (cpu_up) { /* create Instruction event */ attr->config = INST_EV; ret = set_event(&pmu_events[INST_EVENT][cpu], cpu, attr); if (ret < 0) { kfree(attr); return ret; } /* create cycle event */ attr->config = CYC_EV; ret = set_event(&pmu_events[CYC_EVENT][cpu], cpu, attr); if (ret < 0) { free_pmu_counters(cpu); kfree(attr); return ret; } } else { free_pmu_counters(cpu); } kfree(attr); return 0; } static int hotplug_notify_down(unsigned int cpu) { unsigned long flags; if (events_group.init_success) { spin_lock_irqsave(&(events_group.cpu_hotplug_lock), flags); per_cpu(cpu_is_hp, cpu) = true; restart_events(cpu, false); spin_unlock_irqrestore(&(events_group.cpu_hotplug_lock), flags); } return 0; } static int hotplug_notify_up(unsigned int cpu) { unsigned long flags; if (events_group.init_success) { spin_lock_irqsave(&(events_group.cpu_hotplug_lock), flags); events_group.cpu_hotplug = true; restart_events(cpu, true); per_cpu(cpu_is_hp, cpu) = false; spin_unlock_irqrestore(&(events_group.cpu_hotplug_lock), flags); wake_up_process(events_notify_thread); } Loading @@ -455,6 +683,42 @@ static int hotplug_notify(unsigned int cpu) return 0; } static int msm_perf_idle_read_events(unsigned int cpu) { int ret = 0, i; for (i = 0; i < NO_OF_EVENT; i++) { if (pmu_events[i][cpu].pevent) ret = perf_event_read_local(pmu_events[i][cpu].pevent, &pmu_events[i][cpu].cached_total_count, NULL, NULL); } return ret; } static int msm_perf_idle_notif(struct notifier_block *nb, unsigned long action, void *data) { int ret = NOTIFY_OK; int cpu = smp_processor_id(); switch (action) { case IDLE_START: __this_cpu_write(cpu_is_idle, true); if (!per_cpu(cpu_is_hp, cpu)) ret = msm_perf_idle_read_events(cpu); break; case IDLE_END: __this_cpu_write(cpu_is_idle, false); break; } return NOTIFY_OK; } static struct notifier_block msm_perf_event_idle_nb = { .notifier_call = msm_perf_idle_notif, }; static int events_notify_userspace(void *data) { unsigned long flags; Loading Loading @@ -714,7 +978,9 @@ module_param_cb(splh_notif, ¶m_ops_splh_notification, &splh_notif, 0644); static int __init msm_performance_init(void) { unsigned int cpu; int ret; if (!alloc_cpumask_var(&limit_mask_min, GFP_KERNEL)) return -ENOMEM; Loading @@ -723,14 +989,24 @@ static int __init msm_performance_init(void) return -ENOMEM; } ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE, get_online_cpus(); for_each_possible_cpu(cpu) { if (!cpumask_test_cpu(cpu, cpu_online_mask)) per_cpu(cpu_is_hp, cpu) = true; } ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "msm_performance_cpu_hotplug", hotplug_notify, NULL); hotplug_notify_up, hotplug_notify_down); put_online_cpus(); init_events_group(); init_notify_group(); init_pmu_counter(); idle_notifier_register(&msm_perf_event_idle_nb); return 0; } late_initcall(msm_performance_init);
