Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 54567a93 authored by Rohit Gupta's avatar Rohit Gupta
Browse files

PM / devfreq : Introduce a memory-latency governor 



Use performance counters to detect the memory latency sensitivity
of CPU workloads and vote for higher DDR frequency if required.

Change-Id: Ie77a3523bc5713fc0315bd0abc3913f485a96e0e
Signed-off-by: default avatarRohit Gupta <rohgup@codeaurora.org>
Suggested-by: default avatarSaravana Kannan <skannan@codeaurora.org>
parent f4374974

Documentation/devicetree/bindings/devfreq/arm-memlat-mon.txt

0 → 100644
+16 −0
Original line number Diff line number Diff line 
ARM CPU memory latency monitor device



arm-memlat-mon is a device that represents the use of the PMU in ARM cores

to measure the parameters for latency driven memory access patterns.



Required properties:

- compatible:			Must be "qcom,arm-memlat-mon"

- qcom,firstcpu:		First CPU of the cluster to be monitored

- qcom,target-dev:		The DT device that corresponds to this master port



Example:

	qcom,arm-memlat-mon {

		compatible = "qcom,arm-memlat-mon";

		qcom,firstcpu = <&CPU0>;

		qcom,target-dev = <&memlat0>;

	};

arch/arm64/Kconfig

+2 −0
Original line number Diff line number Diff line
@@ -220,6 +220,8 @@ config ARCH_MSM 
	select ARCH_HAS_OPP

	select THERMAL_WRITABLE_TRIPS

	select CPU_FREQ_MSM

	select ARM_MEMLAT_MON

	select DEVFREQ_GOV_MEMLAT

	help

	  This enables support for the ARMv8 based Qualcomm chipsets.

drivers/devfreq/Kconfig

+19 −0
Original line number Diff line number Diff line
@@ -106,6 +106,15 @@ config ARMBW_HWMON 
	  capability to raise an IRQ when the counter overflows, which can be

	  used to get an IRQ when the count exceeds a certain value



config ARM_MEMLAT_MON

	tristate "ARM CPU Memory Latency monitor hardware"

	depends on ARCH_MSM

	help

	  The PMU present on these ARM cores allow for the use of counters to

	  monitor the memory latency characteristics of an ARM CPU workload.

	  This driver uses these counters to implement the APIs needed by

	  the mem_latency devfreq governor.



config MSMCCI_HWMON

	tristate "MSM CCI Cache monitor hardware"

	depends on ARCH_MSM
@@ -152,6 +161,16 @@ config DEVFREQ_GOV_SPDM_HYP 
	  Sets the frequency using a "on-demand" algorithm.

	  This governor is unlikely to be useful for other devices.



config DEVFREQ_GOV_MEMLAT

	tristate "HW monitor based governor for device BW"

	depends on ARM_MEMLAT_MON

	help

	  HW monitor based governor for device to DDR bandwidth voting.

	  This governor sets the CPU BW vote based on stats obtained from memalat

	  monitor if it determines that a workload is memory latency bound. Since

	  this uses target specific counters it can conflict with existing profiling

	  tools.



comment "DEVFREQ Drivers"



config ARM_EXYNOS4_BUS_DEVFREQ

drivers/devfreq/Makefile

+2 −0
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ obj-$(CONFIG_DEVFREQ_GOV_CPUFREQ) += governor_cpufreq.o 
obj-$(CONFIG_DEVFREQ_GOV_MSM_ADRENO_TZ)	+= governor_msm_adreno_tz.o

obj-$(CONFIG_MSM_BIMC_BWMON)		+= bimc-bwmon.o

obj-$(CONFIG_ARMBW_HWMON)		+= armbw-pm.o

obj-$(CONFIG_ARM_MEMLAT_MON)		+= arm-memlat-mon.o

obj-$(CONFIG_MSMCCI_HWMON)		+= msmcci-hwmon.o

obj-$(CONFIG_MSM_M4M_HWMON)		+= m4m-hwmon.o

obj-$(CONFIG_DEVFREQ_GOV_MSM_BW_HWMON)	+= governor_bw_hwmon.o
@@ -17,6 +18,7 @@ obj-$(CONFIG_DEVFREQ_GOV_MSM_CACHE_HWMON) += governor_cache_hwmon.o 
obj-$(CONFIG_DEVFREQ_GOV_MSM_GPUBW_MON)        += governor_gpubw_mon.o

obj-$(CONFIG_DEVFREQ_GOV_MSM_GPUBW_MON)	+= governor_bw_vbif.o

obj-$(CONFIG_DEVFREQ_GOV_SPDM_HYP) 	+= governor_spdm_bw_hyp.o

obj-$(CONFIG_DEVFREQ_GOV_MEMLAT)	+= governor_memlat.o



# DEVFREQ Drivers

obj-$(CONFIG_ARM_EXYNOS4_BUS_DEVFREQ)	+= exynos/

drivers/devfreq/arm-memlat-mon.c

0 → 100644
+316 −0
Original line number Diff line number Diff line 
/*

 * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 and

 * only version 2 as published by the Free Software Foundation.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 */



#define pr_fmt(fmt) "arm-memlat-mon: " fmt



#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/init.h>

#include <linux/io.h>

#include <linux/delay.h>

#include <linux/err.h>

#include <linux/errno.h>

#include <linux/interrupt.h>

#include <linux/platform_device.h>

#include <linux/of.h>

#include <linux/of_irq.h>

#include <linux/slab.h>

#include <linux/irq.h>

#include <linux/cpu_pm.h>

#include <linux/cpu.h>

#include "governor.h"

#include "governor_memlat.h"

#include <linux/perf_event.h>



enum ev_index {

	INST_IDX,

	L2DM_IDX,

	CYC_IDX,

	NUM_EVENTS

};

#define INST_EV		0x08

#define L2DM_EV		0x17

#define CYC_EV		0x11



struct event_data {

	struct perf_event *pevent;

	unsigned long prev_count;

	bool overflow;

};



struct memlat_hwmon_data {

	struct event_data events[NUM_EVENTS];

	ktime_t prev_ts;

};

static DEFINE_PER_CPU(struct memlat_hwmon_data, pm_data);



struct cpu_grp_info {

	cpumask_t cpus;

	struct memlat_hwmon hw;

};



static unsigned long compute_freq(struct memlat_hwmon_data *hw_data,

						unsigned long cyc_cnt)

{

	ktime_t ts;

	unsigned int diff;

	unsigned long freq = 0;



	ts = ktime_get();

	diff = ktime_to_us(ktime_sub(ts, hw_data->prev_ts));

	if (!diff)

		diff = 1;

	hw_data->prev_ts = ts;

	freq = cyc_cnt;

	do_div(freq, diff);



	return freq;

}



#define MAX_COUNT_LIM 0xFFFFFFFFFFFFFFFF

static inline unsigned long read_event(struct event_data *event)

{

	unsigned long ev_count;

	u64 total, enabled, running;



	total = perf_event_read_value(event->pevent, &enabled, &running);

	if (total >= event->prev_count)

		ev_count = total - event->prev_count;

	else

		ev_count = (MAX_COUNT_LIM - event->prev_count) + total;



	event->prev_count = total;



	return ev_count;

}



static void read_perf_counters(int cpu, struct cpu_grp_info *cpu_grp)

{

	int cpu_idx;

	struct memlat_hwmon_data *hw_data = &per_cpu(pm_data, cpu);

	struct memlat_hwmon *hw = &cpu_grp->hw;

	unsigned long cyc_cnt;



	cpu_idx = cpu - cpumask_first(&cpu_grp->cpus);



	hw->core_stats[cpu_idx].inst_count =

			read_event(&hw_data->events[INST_IDX]);



	hw->core_stats[cpu_idx].mem_count =

			read_event(&hw_data->events[L2DM_IDX]);



	cyc_cnt = read_event(&hw_data->events[CYC_IDX]);

	hw->core_stats[cpu_idx].freq = compute_freq(hw_data, cyc_cnt);

}



static unsigned long get_cnt(struct memlat_hwmon *hw)

{

	int cpu;

	struct cpu_grp_info *cpu_grp = container_of(hw,

					struct cpu_grp_info, hw);



	for_each_cpu(cpu, &cpu_grp->cpus)

		read_perf_counters(cpu, cpu_grp);



	return 0;

}



static void delete_events(struct memlat_hwmon_data *hw_data)

{

	perf_event_release_kernel(hw_data->events[INST_IDX].pevent);

	perf_event_release_kernel(hw_data->events[L2DM_IDX].pevent);

	perf_event_release_kernel(hw_data->events[CYC_IDX].pevent);

}



static void stop_hwmon(struct memlat_hwmon *hw)

{

	int cpu;

	struct memlat_hwmon_data *hw_data;

	struct cpu_grp_info *cpu_grp = container_of(hw,

					struct cpu_grp_info, hw);



	get_online_cpus();

	for_each_cpu(cpu, &cpu_grp->cpus) {

		hw_data = &per_cpu(pm_data, cpu);

		delete_events(hw_data);

	}

	put_online_cpus();



}



struct perf_event *create_event(int cpu, unsigned long event)

{

	struct perf_event *pevent;

	struct perf_event_attr *attr;



	attr = kzalloc(sizeof(struct perf_event_attr), GFP_KERNEL);

	if (!attr)

		return ERR_PTR(-ENOMEM);



	attr->config = event;

	attr->type = PERF_TYPE_RAW;

	attr->size = sizeof(struct perf_event_attr);

	attr->pinned = 1;

	attr->exclude_idle = 1;



	pevent = perf_event_create_kernel_counter(attr, cpu, NULL, NULL, NULL);



	return pevent;

}



static int set_events(struct memlat_hwmon_data *hw_data, int cpu)

{

	struct perf_event *pevent;

	int err;



	pevent = create_event(cpu, INST_EV);

	if (IS_ERR(pevent))

		goto err_out;

	hw_data->events[INST_IDX].pevent = pevent;

	perf_event_enable(hw_data->events[INST_IDX].pevent);



	pevent = create_event(cpu, L2DM_EV);

	if (IS_ERR(pevent))

		goto err_out;

	hw_data->events[L2DM_IDX].pevent = pevent;

	perf_event_enable(hw_data->events[L2DM_IDX].pevent);



	pevent = create_event(cpu, CYC_EV);

	if (IS_ERR(pevent))

		goto err_out;

	hw_data->events[CYC_IDX].pevent = pevent;

	perf_event_enable(hw_data->events[CYC_IDX].pevent);



	return 0;



err_out:

	err = PTR_ERR(pevent);

	return err;

}



static int start_hwmon(struct memlat_hwmon *hw)

{

	int cpu, ret = 0;

	struct memlat_hwmon_data *hw_data;

	struct cpu_grp_info *cpu_grp = container_of(hw,

					struct cpu_grp_info, hw);



	get_online_cpus();

	for_each_cpu(cpu, &cpu_grp->cpus) {

		hw_data = &per_cpu(pm_data, cpu);

		ret = set_events(hw_data, cpu);

		if (ret)

			goto err;

	}

	pr_info("Group %d events created\n", cpumask_first(&cpu_grp->cpus));



err:

	put_online_cpus();

	return ret;

}



static int get_mask_from_dev_handle(struct platform_device *pdev,

					cpumask_t *mask)

{

	struct device_node *dev_phandle;

	int cpu = 0;



	dev_phandle = of_parse_phandle(pdev->dev.of_node, "qcom,firstcpu", 0);

	if (!dev_phandle)

		return -ENOENT;



	for_each_possible_cpu(cpu) {

		if (arch_find_n_match_cpu_physical_id(dev_phandle, cpu, NULL)) {

			cpumask_copy(mask, cpu_coregroup_mask(cpu));

			return 0;

		}

	}



	return -ENOENT;

}



static int arm_memlat_mon_driver_probe(struct platform_device *pdev)

{

	struct device *dev = &pdev->dev;

	struct memlat_hwmon *hw;

	struct cpu_grp_info *cpu_grp;

	int cpu, ret;



	cpu_grp = devm_kzalloc(dev, sizeof(*cpu_grp), GFP_KERNEL);

	if (!cpu_grp)

		return -ENOMEM;

	hw = &cpu_grp->hw;



	hw->dev = dev;

	hw->of_node = of_parse_phandle(dev->of_node, "qcom,target-dev", 0);

	if (!hw->of_node) {

		dev_err(dev, "Couldn't find a target device\n");

		goto err_out;

	}



	if (get_mask_from_dev_handle(pdev, &cpu_grp->cpus)) {

		dev_err(dev, "CPU list is empty\n");

		goto err_out;

	}



	hw->num_cores = cpumask_weight(&cpu_grp->cpus);

	hw->core_stats = devm_kzalloc(dev, hw->num_cores *

				sizeof(*(hw->core_stats)), GFP_KERNEL);

	if (!hw->core_stats)

		goto err_out;



	for_each_cpu(cpu, &cpu_grp->cpus)

		hw->core_stats[cpu - cpumask_first(&cpu_grp->cpus)].id = cpu;



	hw->start_hwmon = &start_hwmon;

	hw->stop_hwmon = &stop_hwmon;

	hw->get_cnt = &get_cnt;



	ret = register_memlat(dev, hw);

	if (ret) {

		pr_err("Mem Latency Gov registration failed\n");

		goto err_out;

	}



	return 0;



err_out:

	kfree(cpu_grp);

	return -EINVAL;

}



static struct of_device_id match_table[] = {

	{ .compatible = "qcom,arm-memlat-mon" },

	{}

};



static struct platform_driver arm_memlat_mon_driver = {

	.probe = arm_memlat_mon_driver_probe,

	.driver = {

		.name = "arm-memlat-mon",

		.of_match_table = match_table,

		.owner = THIS_MODULE,

	},

};



static int __init arm_memlat_mon_init(void)

{

	return platform_driver_register(&arm_memlat_mon_driver);

}

module_init(arm_memlat_mon_init);



static void __exit arm_memlat_mon_exit(void)

{

	platform_driver_unregister(&arm_memlat_mon_driver);

}

module_exit(arm_memlat_mon_exit);