x86/resctrl: Allow overflow/limbo handlers to be scheduled on any-but CPU

When a CPU is taken offline resctrl may need to move the overflow or limbo
handlers to run on a different CPU.

Once the offline callbacks have been split, cqm_setup_limbo_handler() will be
called while the CPU that is going offline is still present in the CPU mask.

Pass the CPU to exclude to cqm_setup_limbo_handler() and
mbm_setup_overflow_handler(). These functions can use a variant of
cpumask_any_but() when selecting the CPU. -1 is used to indicate no CPUs need
excluding.

Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: Shaopeng Tan <tan.shaopeng@fujitsu.com>
Reviewed-by: Babu Moger <babu.moger@amd.com>
Reviewed-by: Reinette Chatre <reinette.chatre@intel.com>
Tested-by: Shaopeng Tan <tan.shaopeng@fujitsu.com>
Tested-by: Peter Newman <peternewman@google.com>
Tested-by: Babu Moger <babu.moger@amd.com>
Tested-by: Carl Worth <carl@os.amperecomputing.com> # arm64
Link: https://lore.kernel.org/r/20240213184438.16675-22-james.morse@arm.com
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>

diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c
index 4627d447bc3dc936a486395722514130e23235af..55322ba629da3e2ffbb7d415c2887e71ecdcd178 100644 (file)
--- a/arch/x86/kernel/cpu/resctrl/core.c
+++ b/arch/x86/kernel/cpu/resctrl/core.c
@@ -584,12 +584,16 @@ static void domain_remove_cpu(int cpu, struct rdt_resource *r)
        if (r == &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl) {
                if (is_mbm_enabled() && cpu == d->mbm_work_cpu) {
                        cancel_delayed_work(&d->mbm_over);
-                       mbm_setup_overflow_handler(d, 0);
+                       /*
+                        * temporary: exclude_cpu=-1 as this CPU has already
+                        * been removed by cpumask_clear_cpu()d
+                        */
+                       mbm_setup_overflow_handler(d, 0, RESCTRL_PICK_ANY_CPU);
                }
                if (is_llc_occupancy_enabled() && cpu == d->cqm_work_cpu &&
                    has_busy_rmid(d)) {
                        cancel_delayed_work(&d->cqm_limbo);
-                       cqm_setup_limbo_handler(d, 0);
+                       cqm_setup_limbo_handler(d, 0, RESCTRL_PICK_ANY_CPU);
                }
        }
 }

diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
index 52fa0e14cb86cb789b0b80c19fad1282794a6459..20b02d6f02c198075f7c1f525a7dbb5c35b1a438 100644 (file)
--- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
+++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
@@ -552,7 +552,7 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r,
                return;
        }
 
-       cpu = cpumask_any_housekeeping(&d->cpu_mask);
+       cpu = cpumask_any_housekeeping(&d->cpu_mask, RESCTRL_PICK_ANY_CPU);
 
        /*
         * cpumask_any_housekeeping() prefers housekeeping CPUs, but

diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h
index 3ee855c374471b822a0c0fbd447123622e8a0ebd..c99f26ebe7a6537a7cd43274701ac4f489648081 100644 (file)
--- a/arch/x86/kernel/cpu/resctrl/internal.h
+++ b/arch/x86/kernel/cpu/resctrl/internal.h
@@ -60,19 +60,36 @@
  * cpumask_any_housekeeping() - Choose any CPU in @mask, preferring those that
  *                             aren't marked nohz_full
  * @mask:      The mask to pick a CPU from.
+ * @exclude_cpu:The CPU to avoid picking.
  *
- * Returns a CPU in @mask. If there are housekeeping CPUs that don't use
- * nohz_full, these are preferred.
+ * Returns a CPU from @mask, but not @exclude_cpu. If there are housekeeping
+ * CPUs that don't use nohz_full, these are preferred. Pass
+ * RESCTRL_PICK_ANY_CPU to avoid excluding any CPUs.
+ *
+ * When a CPU is excluded, returns >= nr_cpu_ids if no CPUs are available.
  */
-static inline unsigned int cpumask_any_housekeeping(const struct cpumask *mask)
+static inline unsigned int
+cpumask_any_housekeeping(const struct cpumask *mask, int exclude_cpu)
 {
        unsigned int cpu, hk_cpu;
 
-       cpu = cpumask_any(mask);
-       if (!tick_nohz_full_cpu(cpu))
+       if (exclude_cpu == RESCTRL_PICK_ANY_CPU)
+               cpu = cpumask_any(mask);
+       else
+               cpu = cpumask_any_but(mask, exclude_cpu);
+
+       if (!IS_ENABLED(CONFIG_NO_HZ_FULL))
                return cpu;
 
+       /* If the CPU picked isn't marked nohz_full nothing more needs doing. */
+       if (cpu < nr_cpu_ids && !tick_nohz_full_cpu(cpu))
+               return cpu;
+
+       /* Try to find a CPU that isn't nohz_full to use in preference */
        hk_cpu = cpumask_nth_andnot(0, mask, tick_nohz_full_mask);
+       if (hk_cpu == exclude_cpu)
+               hk_cpu = cpumask_nth_andnot(1, mask, tick_nohz_full_mask);
+
        if (hk_cpu < nr_cpu_ids)
                cpu = hk_cpu;
 
@@ -573,11 +590,13 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r,
                    struct rdt_domain *d, struct rdtgroup *rdtgrp,
                    int evtid, int first);
 void mbm_setup_overflow_handler(struct rdt_domain *dom,
-                               unsigned long delay_ms);
+                               unsigned long delay_ms,
+                               int exclude_cpu);
 void mbm_handle_overflow(struct work_struct *work);
 void __init intel_rdt_mbm_apply_quirk(void);
 bool is_mba_sc(struct rdt_resource *r);
-void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms);
+void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms,
+                            int exclude_cpu);
 void cqm_handle_limbo(struct work_struct *work);
 bool has_busy_rmid(struct rdt_domain *d);
 void __check_limbo(struct rdt_domain *d, bool force_free);

diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c
index 92d7ba674003be73e8eeba2aa020b4c13555b827..67edd4c440f06ea31d5a17e2c676bf33b0578c93 100644 (file)
--- a/arch/x86/kernel/cpu/resctrl/monitor.c
+++ b/arch/x86/kernel/cpu/resctrl/monitor.c
@@ -481,7 +481,8 @@ static void add_rmid_to_limbo(struct rmid_entry *entry)
                 * setup up the limbo worker.
                 */
                if (!has_busy_rmid(d))
-                       cqm_setup_limbo_handler(d, CQM_LIMBOCHECK_INTERVAL);
+                       cqm_setup_limbo_handler(d, CQM_LIMBOCHECK_INTERVAL,
+                                               RESCTRL_PICK_ANY_CPU);
                set_bit(idx, d->rmid_busy_llc);
                entry->busy++;
        }
@@ -784,7 +785,8 @@ void cqm_handle_limbo(struct work_struct *work)
        __check_limbo(d, false);
 
        if (has_busy_rmid(d)) {
-               d->cqm_work_cpu = cpumask_any_housekeeping(&d->cpu_mask);
+               d->cqm_work_cpu = cpumask_any_housekeeping(&d->cpu_mask,
+                                                          RESCTRL_PICK_ANY_CPU);
                schedule_delayed_work_on(d->cqm_work_cpu, &d->cqm_limbo,
                                         delay);
        }
@@ -792,15 +794,25 @@ void cqm_handle_limbo(struct work_struct *work)
        mutex_unlock(&rdtgroup_mutex);
 }
 
-void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms)
+/**
+ * cqm_setup_limbo_handler() - Schedule the limbo handler to run for this
+ *                             domain.
+ * @dom:           The domain the limbo handler should run for.
+ * @delay_ms:      How far in the future the handler should run.
+ * @exclude_cpu:   Which CPU the handler should not run on,
+ *                RESCTRL_PICK_ANY_CPU to pick any CPU.
+ */
+void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms,
+                            int exclude_cpu)
 {
        unsigned long delay = msecs_to_jiffies(delay_ms);
        int cpu;
 
-       cpu = cpumask_any_housekeeping(&dom->cpu_mask);
+       cpu = cpumask_any_housekeeping(&dom->cpu_mask, exclude_cpu);
        dom->cqm_work_cpu = cpu;
 
-       schedule_delayed_work_on(cpu, &dom->cqm_limbo, delay);
+       if (cpu < nr_cpu_ids)
+               schedule_delayed_work_on(cpu, &dom->cqm_limbo, delay);
 }
 
 void mbm_handle_overflow(struct work_struct *work)
@@ -838,14 +850,24 @@ void mbm_handle_overflow(struct work_struct *work)
         * Re-check for housekeeping CPUs. This allows the overflow handler to
         * move off a nohz_full CPU quickly.
         */
-       d->mbm_work_cpu = cpumask_any_housekeeping(&d->cpu_mask);
+       d->mbm_work_cpu = cpumask_any_housekeeping(&d->cpu_mask,
+                                                  RESCTRL_PICK_ANY_CPU);
        schedule_delayed_work_on(d->mbm_work_cpu, &d->mbm_over, delay);
 
 out_unlock:
        mutex_unlock(&rdtgroup_mutex);
 }
 
-void mbm_setup_overflow_handler(struct rdt_domain *dom, unsigned long delay_ms)
+/**
+ * mbm_setup_overflow_handler() - Schedule the overflow handler to run for this
+ *                                domain.
+ * @dom:           The domain the overflow handler should run for.
+ * @delay_ms:      How far in the future the handler should run.
+ * @exclude_cpu:   Which CPU the handler should not run on,
+ *                RESCTRL_PICK_ANY_CPU to pick any CPU.
+ */
+void mbm_setup_overflow_handler(struct rdt_domain *dom, unsigned long delay_ms,
+                               int exclude_cpu)
 {
        unsigned long delay = msecs_to_jiffies(delay_ms);
        int cpu;
@@ -856,9 +878,11 @@ void mbm_setup_overflow_handler(struct rdt_domain *dom, unsigned long delay_ms)
         */
        if (!resctrl_mounted || !resctrl_arch_mon_capable())
                return;
-       cpu = cpumask_any_housekeeping(&dom->cpu_mask);
+       cpu = cpumask_any_housekeeping(&dom->cpu_mask, exclude_cpu);
        dom->mbm_work_cpu = cpu;
-       schedule_delayed_work_on(cpu, &dom->mbm_over, delay);
+
+       if (cpu < nr_cpu_ids)
+               schedule_delayed_work_on(cpu, &dom->mbm_over, delay);
 }
 
 static int dom_data_init(struct rdt_resource *r)

diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
index 38d3b19a3aca1ff8e7c11fd41da2637058f07658..f5688c79d94f4a89b0c986fd5a5ffc16f88527ba 100644 (file)
--- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c
+++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
@@ -2678,7 +2678,8 @@ static int rdt_get_tree(struct fs_context *fc)
        if (is_mbm_enabled()) {
                r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;
                list_for_each_entry(dom, &r->domains, list)
-                       mbm_setup_overflow_handler(dom, MBM_OVERFLOW_INTERVAL);
+                       mbm_setup_overflow_handler(dom, MBM_OVERFLOW_INTERVAL,
+                                                  RESCTRL_PICK_ANY_CPU);
        }
 
        goto out;
@@ -3989,7 +3990,8 @@ int resctrl_online_domain(struct rdt_resource *r, struct rdt_domain *d)
 
        if (is_mbm_enabled()) {
                INIT_DELAYED_WORK(&d->mbm_over, mbm_handle_overflow);
-               mbm_setup_overflow_handler(d, MBM_OVERFLOW_INTERVAL);
+               mbm_setup_overflow_handler(d, MBM_OVERFLOW_INTERVAL,
+                                          RESCTRL_PICK_ANY_CPU);
        }
 
        if (is_llc_occupancy_enabled())

diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h
index 4c4bad3c34e48715c77398100ed5ae809aa2878a..ccbbbe5d18d34e6d1ef8946732761666d318508a 100644 (file)
--- a/include/linux/resctrl.h
+++ b/include/linux/resctrl.h
@@ -10,6 +10,8 @@
 #define RESCTRL_RESERVED_CLOSID                0
 #define RESCTRL_RESERVED_RMID          0
 
+#define RESCTRL_PICK_ANY_CPU           -1
+
 #ifdef CONFIG_PROC_CPU_RESCTRL
 
 int proc_resctrl_show(struct seq_file *m,