Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit cf23d003 authored by qctecmdr's avatar qctecmdr Committed by Gerrit - the friendly Code Review server
Browse files

Merge "PM / devfreq: memlat: Look for min stall% in addition to ratio criteria"

parents ec6b9bfb 93c29d13

drivers/devfreq/Kconfig

+19 −0
Original line number Diff line number Diff line
@@ -83,6 +83,15 @@ config QCOM_BIMC_BWMON 
	  has the capability to raise an IRQ when the count exceeds a

	  programmable limit.



config ARM_MEMLAT_MON

	tristate "ARM CPU Memory Latency monitor hardware"

	depends on ARCH_QCOM

	help

	  The PMU present on these ARM cores allow for the use of counters to

	  monitor the memory latency characteristics of an ARM CPU workload.

	  This driver uses these counters to implement the APIs needed by

	  the mem_latency devfreq governor.



config DEVFREQ_GOV_QCOM_BW_HWMON

	tristate "HW monitor based governor for device BW"

	depends on QCOM_BIMC_BWMON
@@ -102,6 +111,16 @@ config DEVFREQ_GOV_QCOM_CACHE_HWMON 
	  it can conflict with existing profiling tools. This governor is

	  unlikely to be useful for other devices.



config DEVFREQ_GOV_MEMLAT

	tristate "HW monitor based governor for device BW"

	depends on ARM_MEMLAT_MON

	help

	  HW monitor based governor for device to DDR bandwidth voting.

	  This governor sets the CPU BW vote based on stats obtained from memalat

	  monitor if it determines that a workload is memory latency bound. Since

	  this uses target specific counters it can conflict with existing profiling

	  tools.



comment "DEVFREQ Drivers"



config ARM_EXYNOS_BUS_DEVFREQ

drivers/devfreq/Makefile

+2 −0
Original line number Diff line number Diff line
@@ -7,8 +7,10 @@ obj-$(CONFIG_DEVFREQ_GOV_POWERSAVE) += governor_powersave.o 
obj-$(CONFIG_DEVFREQ_GOV_USERSPACE)	+= governor_userspace.o

obj-$(CONFIG_DEVFREQ_GOV_PASSIVE)	+= governor_passive.o

obj-$(CONFIG_QCOM_BIMC_BWMON)		+= bimc-bwmon.o

obj-$(CONFIG_ARM_MEMLAT_MON)		+= arm-memlat-mon.o

obj-$(CONFIG_DEVFREQ_GOV_QCOM_BW_HWMON)	+= governor_bw_hwmon.o

obj-$(CONFIG_DEVFREQ_GOV_QCOM_CACHE_HWMON)	+= governor_cache_hwmon.o

obj-$(CONFIG_DEVFREQ_GOV_MEMLAT)       += governor_memlat.o



# DEVFREQ Drivers

obj-$(CONFIG_ARM_EXYNOS_BUS_DEVFREQ)	+= exynos-bus.o

drivers/devfreq/arm-memlat-mon.c

0 → 100644
+338 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

/*

 * Copyright (c) 2014-2017, 2019, The Linux Foundation. All rights reserved.

 */



#define pr_fmt(fmt) "arm-memlat-mon: " fmt



#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/init.h>

#include <linux/io.h>

#include <linux/delay.h>

#include <linux/err.h>

#include <linux/errno.h>

#include <linux/interrupt.h>

#include <linux/platform_device.h>

#include <linux/of.h>

#include <linux/of_irq.h>

#include <linux/slab.h>

#include <linux/irq.h>

#include <linux/cpu_pm.h>

#include <linux/cpu.h>

#include "governor.h"

#include "governor_memlat.h"

#include <linux/perf_event.h>



enum ev_index {

	INST_IDX,

	CM_IDX,

	CYC_IDX,

	STALL_CYC_IDX,

	NUM_EVENTS

};

#define INST_EV		0x08

#define L2DM_EV		0x17

#define CYC_EV		0x11



struct event_data {

	struct perf_event *pevent;

	unsigned long prev_count;

};



struct cpu_pmu_stats {

	struct event_data events[NUM_EVENTS];

	ktime_t prev_ts;

};



struct cpu_grp_info {

	cpumask_t cpus;

	unsigned int event_ids[NUM_EVENTS];

	struct cpu_pmu_stats *cpustats;

	struct memlat_hwmon hw;

};



#define to_cpustats(cpu_grp, cpu) \

	(&cpu_grp->cpustats[cpu - cpumask_first(&cpu_grp->cpus)])

#define to_devstats(cpu_grp, cpu) \

	(&cpu_grp->hw.core_stats[cpu - cpumask_first(&cpu_grp->cpus)])

#define to_cpu_grp(hwmon) container_of(hwmon, struct cpu_grp_info, hw)





static unsigned long compute_freq(struct cpu_pmu_stats *cpustats,

						unsigned long cyc_cnt)

{

	ktime_t ts;

	unsigned int diff;

	unsigned long freq = 0;



	ts = ktime_get();

	diff = ktime_to_us(ktime_sub(ts, cpustats->prev_ts));

	if (!diff)

		diff = 1;

	cpustats->prev_ts = ts;

	freq = cyc_cnt;

	do_div(freq, diff);



	return freq;

}



#define MAX_COUNT_LIM 0xFFFFFFFFFFFFFFFF

static inline unsigned long read_event(struct event_data *event)

{

	unsigned long ev_count;

	u64 total, enabled, running;



	total = perf_event_read_value(event->pevent, &enabled, &running);

	ev_count = total - event->prev_count;

	event->prev_count = total;

	return ev_count;

}



static void read_perf_counters(int cpu, struct cpu_grp_info *cpu_grp)

{

	struct cpu_pmu_stats *cpustats = to_cpustats(cpu_grp, cpu);

	struct dev_stats *devstats = to_devstats(cpu_grp, cpu);

	unsigned long cyc_cnt, stall_cnt;



	devstats->inst_count = read_event(&cpustats->events[INST_IDX]);

	devstats->mem_count = read_event(&cpustats->events[CM_IDX]);

	cyc_cnt = read_event(&cpustats->events[CYC_IDX]);

	devstats->freq = compute_freq(cpustats, cyc_cnt);

	if (cpustats->events[STALL_CYC_IDX].pevent) {

		stall_cnt = read_event(&cpustats->events[STALL_CYC_IDX]);

		stall_cnt = min(stall_cnt, cyc_cnt);

		devstats->stall_pct = mult_frac(100, stall_cnt, cyc_cnt);

	} else {

		devstats->stall_pct = 100;

	}

}



static unsigned long get_cnt(struct memlat_hwmon *hw)

{

	int cpu;

	struct cpu_grp_info *cpu_grp = to_cpu_grp(hw);



	for_each_cpu(cpu, &cpu_grp->cpus)

		read_perf_counters(cpu, cpu_grp);



	return 0;

}



static void delete_events(struct cpu_pmu_stats *cpustats)

{

	int i;



	for (i = 0; i < ARRAY_SIZE(cpustats->events); i++) {

		cpustats->events[i].prev_count = 0;

		if (cpustats->events[i].pevent) {

			perf_event_release_kernel(cpustats->events[i].pevent);

			cpustats->events[i].pevent = NULL;

		}

	}

}



static void stop_hwmon(struct memlat_hwmon *hw)

{

	int cpu;

	struct cpu_grp_info *cpu_grp = to_cpu_grp(hw);

	struct dev_stats *devstats;



	for_each_cpu(cpu, &cpu_grp->cpus) {

		delete_events(to_cpustats(cpu_grp, cpu));



		/* Clear governor data */

		devstats = to_devstats(cpu_grp, cpu);

		devstats->inst_count = 0;

		devstats->mem_count = 0;

		devstats->freq = 0;

		devstats->stall_pct = 0;

	}

}



static struct perf_event_attr *alloc_attr(void)

{

	struct perf_event_attr *attr;



	attr = kzalloc(sizeof(struct perf_event_attr), GFP_KERNEL);

	if (!attr)

		return attr;



	attr->type = PERF_TYPE_RAW;

	attr->size = sizeof(struct perf_event_attr);

	attr->pinned = 1;



	return attr;

}



static int set_events(struct cpu_grp_info *cpu_grp, int cpu)

{

	struct perf_event *pevent;

	struct perf_event_attr *attr;

	int err, i;

	unsigned int event_id;

	struct cpu_pmu_stats *cpustats = to_cpustats(cpu_grp, cpu);



	/* Allocate an attribute for event initialization */

	attr = alloc_attr();

	if (!attr)

		return -ENOMEM;



	for (i = 0; i < ARRAY_SIZE(cpustats->events); i++) {

		event_id = cpu_grp->event_ids[i];

		if (!event_id)

			continue;



		attr->config = event_id;

		pevent = perf_event_create_kernel_counter(attr, cpu, NULL,

							  NULL, NULL);

		if (IS_ERR(pevent))

			goto err_out;

		cpustats->events[i].pevent = pevent;

		perf_event_enable(pevent);

	}



	kfree(attr);

	return 0;



err_out:

	err = PTR_ERR(pevent);

	kfree(attr);

	return err;

}



static int start_hwmon(struct memlat_hwmon *hw)

{

	int cpu, ret = 0;

	struct cpu_grp_info *cpu_grp = to_cpu_grp(hw);



	for_each_cpu(cpu, &cpu_grp->cpus) {

		ret = set_events(cpu_grp, cpu);

		if (ret < 0) {

			pr_warn("Perf event init failed on CPU%d: %d\n", cpu,

					ret);

			break;

		}

	}



	return ret;

}



static int get_mask_from_dev_handle(struct platform_device *pdev,

					cpumask_t *mask)

{

	struct device *dev = &pdev->dev;

	struct device_node *dev_phandle;

	struct device *cpu_dev;

	int cpu, i = 0;

	int ret = -ENOENT;



	dev_phandle = of_parse_phandle(dev->of_node, "qcom,cpulist", i++);

	while (dev_phandle) {

		for_each_possible_cpu(cpu) {

			cpu_dev = get_cpu_device(cpu);

			if (cpu_dev && cpu_dev->of_node == dev_phandle) {

				cpumask_set_cpu(cpu, mask);

				ret = 0;

				break;

			}

		}

		dev_phandle = of_parse_phandle(dev->of_node,

						"qcom,cpulist", i++);

	}



	return ret;

}



static int arm_memlat_mon_driver_probe(struct platform_device *pdev)

{

	struct device *dev = &pdev->dev;

	struct memlat_hwmon *hw;

	struct cpu_grp_info *cpu_grp;

	int cpu, ret;

	u32 event_id;



	cpu_grp = devm_kzalloc(dev, sizeof(*cpu_grp), GFP_KERNEL);

	if (!cpu_grp)

		return -ENOMEM;

	hw = &cpu_grp->hw;



	hw->dev = dev;

	hw->of_node = of_parse_phandle(dev->of_node, "qcom,target-dev", 0);

	if (!hw->of_node) {

		dev_err(dev, "Couldn't find a target device\n");

		return -ENODEV;

	}



	if (get_mask_from_dev_handle(pdev, &cpu_grp->cpus)) {

		dev_err(dev, "CPU list is empty\n");

		return -ENODEV;

	}



	hw->num_cores = cpumask_weight(&cpu_grp->cpus);

	hw->core_stats = devm_kzalloc(dev, hw->num_cores *

				sizeof(*(hw->core_stats)), GFP_KERNEL);

	if (!hw->core_stats)

		return -ENOMEM;



	cpu_grp->cpustats = devm_kzalloc(dev, hw->num_cores *

			sizeof(*(cpu_grp->cpustats)), GFP_KERNEL);

	if (!cpu_grp->cpustats)

		return -ENOMEM;



	cpu_grp->event_ids[CYC_IDX] = CYC_EV;



	ret = of_property_read_u32(dev->of_node, "qcom,cachemiss-ev",

				   &event_id);

	if (ret < 0) {

		dev_dbg(dev, "Cache Miss event not specified. Using def:0x%x\n",

			L2DM_EV);

		event_id = L2DM_EV;

	}

	cpu_grp->event_ids[CM_IDX] = event_id;



	ret = of_property_read_u32(dev->of_node, "qcom,inst-ev", &event_id);

	if (ret < 0) {

		dev_dbg(dev, "Inst event not specified. Using def:0x%x\n",

			INST_EV);

		event_id = INST_EV;

	}

	cpu_grp->event_ids[INST_IDX] = event_id;



	ret = of_property_read_u32(dev->of_node, "qcom,stall-cycle-ev",

				   &event_id);

	if (ret)

		dev_dbg(dev, "Stall cycle event not specified. Event ignored.\n");

	else

		cpu_grp->event_ids[STALL_CYC_IDX] = event_id;



	for_each_cpu(cpu, &cpu_grp->cpus)

		to_devstats(cpu_grp, cpu)->id = cpu;



	hw->start_hwmon = &start_hwmon;

	hw->stop_hwmon = &stop_hwmon;

	hw->get_cnt = &get_cnt;



	ret = register_memlat(dev, hw);

	if (ret < 0) {

		pr_err("Mem Latency Gov registration failed: %d\n", ret);

		return ret;

	}



	return 0;

}



static const struct of_device_id memlat_match_table[] = {

	{ .compatible = "qcom,arm-memlat-mon" },

	{}

};



static struct platform_driver arm_memlat_mon_driver = {

	.probe = arm_memlat_mon_driver_probe,

	.driver = {

		.name = "arm-memlat-mon",

		.of_match_table = memlat_match_table,

	},

};



module_platform_driver(arm_memlat_mon_driver);

drivers/devfreq/bimc-bwmon.c

+886 −118

File changed.

Preview size limit exceeded, changes collapsed.

drivers/devfreq/devfreq_simple_dev.c

+53 −23
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

/*

 * Copyright (c) 2014, 2019, The Linux Foundation. All rights reserved.

 * Copyright (c) 2014-2015, 2017, 2019, The Linux Foundation. All rights reserved.

 */



#define pr_fmt(fmt) "devfreq-simple-dev: " fmt
@@ -26,6 +26,7 @@ struct dev_data { 
	struct clk			*clk;

	struct devfreq			*df;

	struct devfreq_dev_profile	profile;

	bool				freq_in_khz;

};



static void find_freq(struct devfreq_dev_profile *p, unsigned long *freq,
@@ -57,7 +58,7 @@ static int dev_target(struct device *dev, unsigned long *freq, u32 flags) 


	find_freq(&d->profile, freq, flags);



	rfreq = clk_round_rate(d->clk, *freq * 1000);

	rfreq = clk_round_rate(d->clk, d->freq_in_khz ? *freq * 1000 : *freq);

	if (IS_ERR_VALUE(rfreq)) {

		dev_err(dev, "devfreq: Cannot find matching frequency for %lu\n",

			*freq);
@@ -75,39 +76,30 @@ static int dev_get_cur_freq(struct device *dev, unsigned long *freq) 
	f = clk_get_rate(d->clk);

	if (IS_ERR_VALUE(f))

		return f;

	*freq = f / 1000;

	*freq = d->freq_in_khz ? f / 1000 : f;

	return 0;

}



#define PROP_TBL "freq-tbl-khz"

static int devfreq_clock_probe(struct platform_device *pdev)

static int parse_freq_table(struct device *dev, struct dev_data *d)

{

	struct device *dev = &pdev->dev;

	struct dev_data *d;

	struct devfreq_dev_profile *p;

	u32 *data, poll;

	const char *gov_name;

	struct devfreq_dev_profile *p = &d->profile;

	int ret, len, i, j;

	u32 *data;

	unsigned long f;



	d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL);

	if (!d)

		return -ENOMEM;

	platform_set_drvdata(pdev, d);



	d->clk = devm_clk_get(dev, "devfreq_clk");

	if (IS_ERR(d->clk))

		return PTR_ERR(d->clk);



	if (!of_find_property(dev->of_node, PROP_TBL, &len))

		return -EINVAL;

	if (!of_find_property(dev->of_node, PROP_TBL, &len)) {

		if (dev_pm_opp_get_opp_count(dev) <= 0)

			return -EPROBE_DEFER;

		return 0;

	}



	d->freq_in_khz = true;

	len /= sizeof(*data);

	data = devm_kzalloc(dev, len * sizeof(*data), GFP_KERNEL);

	if (!data)

		return -ENOMEM;



	p = &d->profile;

	p->freq_table = devm_kzalloc(dev, len * sizeof(*p->freq_table),

				     GFP_KERNEL);

	if (!p->freq_table)
@@ -134,6 +126,32 @@ static int devfreq_clock_probe(struct platform_device *pdev) 
		return -EINVAL;

	}



	return 0;

}



static int devfreq_clock_probe(struct platform_device *pdev)

{

	struct device *dev = &pdev->dev;

	struct dev_data *d;

	struct devfreq_dev_profile *p;

	u32 poll;

	const char *gov_name;

	int ret;



	d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL);

	if (!d)

		return -ENOMEM;

	platform_set_drvdata(pdev, d);



	d->clk = devm_clk_get(dev, "devfreq_clk");

	if (IS_ERR(d->clk))

		return PTR_ERR(d->clk);



	ret = parse_freq_table(dev, d);

	if (ret < 0)

		return ret;



	p = &d->profile;

	p->target = dev_target;

	p->get_cur_freq = dev_get_cur_freq;

	ret = dev_get_cur_freq(dev, &p->initial_freq);
@@ -147,11 +165,23 @@ static int devfreq_clock_probe(struct platform_device *pdev) 
	if (of_property_read_string(dev->of_node, "governor", &gov_name))

		gov_name = "performance";



	if (of_property_read_bool(dev->of_node, "qcom,prepare-clk")) {

		ret = clk_prepare(d->clk);

		if (ret < 0)

			return ret;

	}



	d->df = devfreq_add_device(dev, p, gov_name, NULL);

	if (IS_ERR(d->df))

		return PTR_ERR_OR_ZERO(d->df);

	if (IS_ERR(d->df)) {

		ret = PTR_ERR_OR_ZERO(d->df);

		goto add_err;

	}



	return 0;

add_err:

	if (of_property_read_bool(dev->of_node, "qcom,prepare-clk"))

		clk_unprepare(d->clk);

	return ret;

}



static int devfreq_clock_remove(struct platform_device *pdev)