XArray: set the marks correctly when splitting an entry

If we created a new node to replace an entry which had search marks set,
we were setting the search mark on every entry in that node.  That works
fine when we're splitting to order 0, but when splitting to a larger
order, we must not set the search marks on the sibling entries.

Link: https://lkml.kernel.org/r/20240501153120.4094530-1-willy@infradead.org
Fixes: c010d47f107f ("mm: thp: split huge page to any lower order pages")
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Reported-by: Luis Chamberlain <mcgrof@kernel.org>
Link: https://lore.kernel.org/r/ZjFGCOYk3FK_zVy3@bombadil.infradead.org
Tested-by: Luis Chamberlain <mcgrof@kernel.org>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/lib/test_xarray.c b/lib/test_xarray.c
index 5ab35190aae37d4bcccebe8c85d61ed635fab417..928fc20337e65834a9f0aa414b306a08d6891f4e 100644 (file)
--- a/lib/test_xarray.c
+++ b/lib/test_xarray.c
@@ -1788,9 +1788,11 @@ static void check_split_1(struct xarray *xa, unsigned long index,
                                unsigned int order, unsigned int new_order)
 {
        XA_STATE_ORDER(xas, xa, index, new_order);
-       unsigned int i;
+       unsigned int i, found;
+       void *entry;
 
        xa_store_order(xa, index, order, xa, GFP_KERNEL);
+       xa_set_mark(xa, index, XA_MARK_1);
 
        xas_split_alloc(&xas, xa, order, GFP_KERNEL);
        xas_lock(&xas);
@@ -1807,6 +1809,16 @@ static void check_split_1(struct xarray *xa, unsigned long index,
        xa_set_mark(xa, index, XA_MARK_0);
        XA_BUG_ON(xa, !xa_get_mark(xa, index, XA_MARK_0));
 
+       xas_set_order(&xas, index, 0);
+       found = 0;
+       rcu_read_lock();
+       xas_for_each_marked(&xas, entry, ULONG_MAX, XA_MARK_1) {
+               found++;
+               XA_BUG_ON(xa, xa_is_internal(entry));
+       }
+       rcu_read_unlock();
+       XA_BUG_ON(xa, found != 1 << (order - new_order));
+
        xa_destroy(xa);
 }
 

diff --git a/lib/xarray.c b/lib/xarray.c
index 39f07bfc4dccacd4da23f76bc0c553815728bee5..5e7d6334d70d7d1e16291a15fd2f5a45a83f81b9 100644 (file)
--- a/lib/xarray.c
+++ b/lib/xarray.c
@@ -969,8 +969,22 @@ static unsigned int node_get_marks(struct xa_node *node, unsigned int offset)
        return marks;
 }
 
+static inline void node_mark_slots(struct xa_node *node, unsigned int sibs,
+               xa_mark_t mark)
+{
+       int i;
+
+       if (sibs == 0)
+               node_mark_all(node, mark);
+       else {
+               for (i = 0; i < XA_CHUNK_SIZE; i += sibs + 1)
+                       node_set_mark(node, i, mark);
+       }
+}
+
 static void node_set_marks(struct xa_node *node, unsigned int offset,
-                       struct xa_node *child, unsigned int marks)
+                       struct xa_node *child, unsigned int sibs,
+                       unsigned int marks)
 {
        xa_mark_t mark = XA_MARK_0;
 
@@ -978,7 +992,7 @@ static void node_set_marks(struct xa_node *node, unsigned int offset,
                if (marks & (1 << (__force unsigned int)mark)) {
                        node_set_mark(node, offset, mark);
                        if (child)
-                               node_mark_all(child, mark);
+                               node_mark_slots(child, sibs, mark);
                }
                if (mark == XA_MARK_MAX)
                        break;
@@ -1077,7 +1091,8 @@ void xas_split(struct xa_state *xas, void *entry, unsigned int order)
                        child->nr_values = xa_is_value(entry) ?
                                        XA_CHUNK_SIZE : 0;
                        RCU_INIT_POINTER(child->parent, node);
-                       node_set_marks(node, offset, child, marks);
+                       node_set_marks(node, offset, child, xas->xa_sibs,
+                                       marks);
                        rcu_assign_pointer(node->slots[offset],
                                        xa_mk_node(child));
                        if (xa_is_value(curr))
@@ -1086,7 +1101,7 @@ void xas_split(struct xa_state *xas, void *entry, unsigned int order)
                } else {
                        unsigned int canon = offset - xas->xa_sibs;
 
-                       node_set_marks(node, canon, NULL, marks);
+                       node_set_marks(node, canon, NULL, 0, marks);
                        rcu_assign_pointer(node->slots[canon], entry);
                        while (offset > canon)
                                rcu_assign_pointer(node->slots[offset--],