kernel/drivers/platform/x86/intel/uncore-frequency.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Intel Uncore Frequency Setting
 * Copyright (c) 2019, Intel Corporation.
 * All rights reserved.
 *
 * Provide interface to set MSR 620 at a granularity of per die. On CPU online,
 * one control CPU is identified per die to read/write limit. This control CPU
 * is changed, if the CPU state is changed to offline. When the last CPU is
 * offline in a die then remove the sysfs object for that die.
 * The majority of actual code is related to sysfs create and read/write
 * attributes.
 *
 * Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
 */

#include <linux/cpu.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/suspend.h>
#include <asm/cpu_device_id.h>
#include <asm/intel-family.h>

#define MSR_UNCORE_RATIO_LIMIT			0x620
#define UNCORE_FREQ_KHZ_MULTIPLIER		100000

/**
 * struct uncore_data -	Encapsulate all uncore data
 * @stored_uncore_data:	Last user changed MSR 620 value, which will be restored
 *			on system resume.
 * @initial_min_freq_khz: Sampled minimum uncore frequency at driver init
 * @initial_max_freq_khz: Sampled maximum uncore frequency at driver init
 * @control_cpu:	Designated CPU for a die to read/write
 * @valid:		Mark the data valid/invalid
 *
 * This structure is used to encapsulate all data related to uncore sysfs
 * settings for a die/package.
 */
struct uncore_data {
	struct kobject kobj;
	struct completion kobj_unregister;
	u64 stored_uncore_data;
	u32 initial_min_freq_khz;
	u32 initial_max_freq_khz;
	int control_cpu;
	bool valid;
};

#define to_uncore_data(a) container_of(a, struct uncore_data, kobj)

/* Max instances for uncore data, one for each die */
static int uncore_max_entries __read_mostly;
/* Storage for uncore data for all instances */
static struct uncore_data *uncore_instances;
/* Root of the all uncore sysfs kobjs */
static struct kobject *uncore_root_kobj;
/* Stores the CPU mask of the target CPUs to use during uncore read/write */
static cpumask_t uncore_cpu_mask;
/* CPU online callback register instance */
static enum cpuhp_state uncore_hp_state __read_mostly;
/* Mutex to control all mutual exclusions */
static DEFINE_MUTEX(uncore_lock);

struct uncore_attr {
	struct attribute attr;
	ssize_t (*show)(struct kobject *kobj,
			struct attribute *attr, char *buf);
	ssize_t (*store)(struct kobject *kobj,
			 struct attribute *attr, const char *c, ssize_t count);
};

#define define_one_uncore_ro(_name) \
static struct uncore_attr _name = \
__ATTR(_name, 0444, show_##_name, NULL)

#define define_one_uncore_rw(_name) \
static struct uncore_attr _name = \
__ATTR(_name, 0644, show_##_name, store_##_name)

#define show_uncore_data(member_name)					\
	static ssize_t show_##member_name(struct kobject *kobj,         \
					  struct attribute *attr,	\
					  char *buf)			\
	{                                                               \
		struct uncore_data *data = to_uncore_data(kobj);	\
		return scnprintf(buf, PAGE_SIZE, "%u\n",		\
				 data->member_name);			\
	}								\
	define_one_uncore_ro(member_name)

show_uncore_data(initial_min_freq_khz);
show_uncore_data(initial_max_freq_khz);

/* Common function to read MSR 0x620 and read min/max */
static int uncore_read_ratio(struct uncore_data *data, unsigned int *min,
			     unsigned int *max)
{
	u64 cap;
	int ret;

	if (data->control_cpu < 0)
		return -ENXIO;

	ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
	if (ret)
		return ret;

	*max = (cap & 0x7F) * UNCORE_FREQ_KHZ_MULTIPLIER;
	*min = ((cap & GENMASK(14, 8)) >> 8) * UNCORE_FREQ_KHZ_MULTIPLIER;

	return 0;
}

/* Common function to set min/max ratios to be used by sysfs callbacks */
static int uncore_write_ratio(struct uncore_data *data, unsigned int input,
			      int set_max)
{
	int ret;
	u64 cap;

	mutex_lock(&uncore_lock);

	if (data->control_cpu < 0) {
		ret = -ENXIO;
		goto finish_write;
	}

	input /= UNCORE_FREQ_KHZ_MULTIPLIER;
	if (!input || input > 0x7F) {
		ret = -EINVAL;
		goto finish_write;
	}

	ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
	if (ret)
		goto finish_write;

	if (set_max) {
		cap &= ~0x7F;
		cap |= input;
	} else  {
		cap &= ~GENMASK(14, 8);
		cap |= (input << 8);
	}

	ret = wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap);
	if (ret)
		goto finish_write;

	data->stored_uncore_data = cap;

finish_write:
	mutex_unlock(&uncore_lock);

	return ret;
}

static ssize_t store_min_max_freq_khz(struct kobject *kobj,
				      struct attribute *attr,
				      const char *buf, ssize_t count,
				      int min_max)
{
	struct uncore_data *data = to_uncore_data(kobj);
	unsigned int input;

	if (kstrtouint(buf, 10, &input))
		return -EINVAL;

	uncore_write_ratio(data, input, min_max);

	return count;
}

static ssize_t show_min_max_freq_khz(struct kobject *kobj,
				     struct attribute *attr,
				     char *buf, int min_max)
{
	struct uncore_data *data = to_uncore_data(kobj);
	unsigned int min, max;
	int ret;

	mutex_lock(&uncore_lock);
	ret = uncore_read_ratio(data, &min, &max);
	mutex_unlock(&uncore_lock);
	if (ret)
		return ret;

	if (min_max)
		return sprintf(buf, "%u\n", max);

	return sprintf(buf, "%u\n", min);
}

#define store_uncore_min_max(name, min_max)				\
	static ssize_t store_##name(struct kobject *kobj,		\
				    struct attribute *attr,		\
				    const char *buf, ssize_t count)	\
	{                                                               \
									\
		return store_min_max_freq_khz(kobj, attr, buf, count,	\
					      min_max);			\
	}

#define show_uncore_min_max(name, min_max)				\
	static ssize_t show_##name(struct kobject *kobj,		\
				   struct attribute *attr, char *buf)	\
	{                                                               \
									\
		return show_min_max_freq_khz(kobj, attr, buf, min_max); \
	}

store_uncore_min_max(min_freq_khz, 0);
store_uncore_min_max(max_freq_khz, 1);

show_uncore_min_max(min_freq_khz, 0);
show_uncore_min_max(max_freq_khz, 1);

define_one_uncore_rw(min_freq_khz);
define_one_uncore_rw(max_freq_khz);

static struct attribute *uncore_attrs[] = {
	&initial_min_freq_khz.attr,
	&initial_max_freq_khz.attr,
	&max_freq_khz.attr,
	&min_freq_khz.attr,
	NULL
};

static void uncore_sysfs_entry_release(struct kobject *kobj)
{
	struct uncore_data *data = to_uncore_data(kobj);

	complete(&data->kobj_unregister);
}

static struct kobj_type uncore_ktype = {
	.release = uncore_sysfs_entry_release,
	.sysfs_ops = &kobj_sysfs_ops,
	.default_attrs = uncore_attrs,
};

/* Caller provides protection */
static struct uncore_data *uncore_get_instance(unsigned int cpu)
{
	int id = topology_logical_die_id(cpu);

	if (id >= 0 && id < uncore_max_entries)
		return &uncore_instances[id];

	return NULL;
}

static void uncore_add_die_entry(int cpu)
{
	struct uncore_data *data;

	mutex_lock(&uncore_lock);
	data = uncore_get_instance(cpu);
	if (!data) {
		mutex_unlock(&uncore_lock);
		return;
	}

	if (data->valid) {
		/* control cpu changed */
		data->control_cpu = cpu;
	} else {
		char str[64];
		int ret;

		memset(data, 0, sizeof(*data));
		sprintf(str, "package_%02d_die_%02d",
			topology_physical_package_id(cpu),
			topology_die_id(cpu));

		uncore_read_ratio(data, &data->initial_min_freq_khz,
				  &data->initial_max_freq_khz);

		init_completion(&data->kobj_unregister);

		ret = kobject_init_and_add(&data->kobj, &uncore_ktype,
					   uncore_root_kobj, str);
		if (!ret) {
			data->control_cpu = cpu;
			data->valid = true;
		}
	}
	mutex_unlock(&uncore_lock);
}

/* Last CPU in this die is offline, make control cpu invalid */
static void uncore_remove_die_entry(int cpu)
{
	struct uncore_data *data;

	mutex_lock(&uncore_lock);
	data = uncore_get_instance(cpu);
	if (data)
		data->control_cpu = -1;
	mutex_unlock(&uncore_lock);
}

static int uncore_event_cpu_online(unsigned int cpu)
{
	int target;

	/* Check if there is an online cpu in the package for uncore MSR */
	target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
	if (target < nr_cpu_ids)
		return 0;

	/* Use this CPU on this die as a control CPU */
	cpumask_set_cpu(cpu, &uncore_cpu_mask);
	uncore_add_die_entry(cpu);

	return 0;
}

static int uncore_event_cpu_offline(unsigned int cpu)
{
	int target;

	/* Check if existing cpu is used for uncore MSRs */
	if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
		return 0;

	/* Find a new cpu to set uncore MSR */
	target = cpumask_any_but(topology_die_cpumask(cpu), cpu);

	if (target < nr_cpu_ids) {
		cpumask_set_cpu(target, &uncore_cpu_mask);
		uncore_add_die_entry(target);
	} else {
		uncore_remove_die_entry(cpu);
	}

	return 0;
}

static int uncore_pm_notify(struct notifier_block *nb, unsigned long mode,
			    void *_unused)
{
	int cpu;

	switch (mode) {
	case PM_POST_HIBERNATION:
	case PM_POST_RESTORE:
	case PM_POST_SUSPEND:
		for_each_cpu(cpu, &uncore_cpu_mask) {
			struct uncore_data *data;
			int ret;

			data = uncore_get_instance(cpu);
			if (!data || !data->valid || !data->stored_uncore_data)
				continue;

			ret = wrmsrl_on_cpu(cpu, MSR_UNCORE_RATIO_LIMIT,
					    data->stored_uncore_data);
			if (ret)
				return ret;
		}
		break;
	default:
		break;
	}
	return 0;
}

static struct notifier_block uncore_pm_nb = {
	.notifier_call = uncore_pm_notify,
};

static const struct x86_cpu_id intel_uncore_cpu_ids[] = {
	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G,	NULL),
	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X,	NULL),
	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D,	NULL),
	X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X,	NULL),
	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X,	NULL),
	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D,	NULL),
	X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, NULL),
	{}
};

static int __init intel_uncore_init(void)
{
	const struct x86_cpu_id *id;
	int ret;

	id = x86_match_cpu(intel_uncore_cpu_ids);
	if (!id)
		return -ENODEV;

	uncore_max_entries = topology_max_packages() *
					topology_max_die_per_package();
	uncore_instances = kcalloc(uncore_max_entries,
				   sizeof(*uncore_instances), GFP_KERNEL);
	if (!uncore_instances)
		return -ENOMEM;

	uncore_root_kobj = kobject_create_and_add("intel_uncore_frequency",
						  &cpu_subsys.dev_root->kobj);
	if (!uncore_root_kobj) {
		ret = -ENOMEM;
		goto err_free;
	}

	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
				"platform/x86/uncore-freq:online",
				uncore_event_cpu_online,
				uncore_event_cpu_offline);
	if (ret < 0)
		goto err_rem_kobj;

	uncore_hp_state = ret;

	ret = register_pm_notifier(&uncore_pm_nb);
	if (ret)
		goto err_rem_state;

	return 0;

err_rem_state:
	cpuhp_remove_state(uncore_hp_state);
err_rem_kobj:
	kobject_put(uncore_root_kobj);
err_free:
	kfree(uncore_instances);

	return ret;
}
module_init(intel_uncore_init)

static void __exit intel_uncore_exit(void)
{
	int i;

	unregister_pm_notifier(&uncore_pm_nb);
	cpuhp_remove_state(uncore_hp_state);
	for (i = 0; i < uncore_max_entries; ++i) {
		if (uncore_instances[i].valid) {
			kobject_put(&uncore_instances[i].kobj);
			wait_for_completion(&uncore_instances[i].kobj_unregister);
		}
	}
	kobject_put(uncore_root_kobj);
	kfree(uncore_instances);
}
module_exit(intel_uncore_exit)

MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Intel Uncore Frequency Limits Driver");