udp: add local "peek offset enabled" flag

We want to re-organize the struct sock layout. The sk_peek_off
field location is problematic, as most protocols want it in the
RX read area, while UDP wants it on a cacheline different from
sk_receive_queue.

Create a local (inside udp_sock) copy of the 'peek offset is enabled'
flag and place it inside the same cacheline of reader_queue.

Check such flag before reading sk_peek_off. This will save potential
false sharing and cache misses in the fast-path.

Tested under UDP flood with small packets. The struct sock layout
update causes a 4% performance drop, and this patch restores completely
the original tput.

Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Link: https://lore.kernel.org/r/67ab679c15fbf49fa05b3ffe05d91c47ab84f147.1708426665.git.pabeni@redhat.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/linux/udp.h b/include/linux/udp.h
index d04188714dca14da25aa57083488cd28c34c41ba..3748e82b627b7044508db66adbf77c54a8e3d612 100644 (file)
--- a/include/linux/udp.h
+++ b/include/linux/udp.h
@@ -92,6 +92,9 @@ struct udp_sock {
 
        /* This fields follows rcvbuf value, and is touched by udp_recvmsg */
        int             forward_threshold;
+
+       /* Cache friendly copy of sk->sk_peek_off >= 0 */
+       bool            peeking_with_offset;
 };
 
 #define udp_test_bit(nr, sk)                   \
@@ -109,6 +112,13 @@ struct udp_sock {
 
 #define udp_sk(ptr) container_of_const(ptr, struct udp_sock, inet.sk)
 
+static inline int udp_set_peek_off(struct sock *sk, int val)
+{
+       sk_set_peek_off(sk, val);
+       WRITE_ONCE(udp_sk(sk)->peeking_with_offset, val >= 0);
+       return 0;
+}
+
 static inline void udp_set_no_check6_tx(struct sock *sk, bool val)
 {
        udp_assign_bit(NO_CHECK6_TX, sk, val);

diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index ad278009e4698c03dfcc7a06e79af39345a41180..5daebdcbca326aa1fc042e1e1ff1e82a18bd283d 100644 (file)
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1103,7 +1103,7 @@ const struct proto_ops inet_dgram_ops = {
        .recvmsg           = inet_recvmsg,
        .mmap              = sock_no_mmap,
        .splice_eof        = inet_splice_eof,
-       .set_peek_off      = sk_set_peek_off,
+       .set_peek_off      = udp_set_peek_off,
 #ifdef CONFIG_COMPAT
        .compat_ioctl      = inet_compat_ioctl,
 #endif

diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index f631b0a21af4c7a520212c94ed0580f86d269ed2..38cce7cc51f65ce6a4ead15974dd5fcecd3881c7 100644 (file)
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1589,7 +1589,7 @@ int udp_init_sock(struct sock *sk)
 
 void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
 {
-       if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) {
+       if (unlikely(READ_ONCE(udp_sk(sk)->peeking_with_offset))) {
                bool slow = lock_sock_fast(sk);
 
                sk_peek_offset_bwd(sk, len);

diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c
index 959bfd9f6344f11241dd20246f92bd1d47ff565e..b90d46533cdcc1ffb61ca483e6f67ab358ede55c 100644 (file)
--- a/net/ipv6/af_inet6.c
+++ b/net/ipv6/af_inet6.c
@@ -736,7 +736,7 @@ const struct proto_ops inet6_dgram_ops = {
        .recvmsg           = inet6_recvmsg,             /* retpoline's sake */
        .read_skb          = udp_read_skb,
        .mmap              = sock_no_mmap,
-       .set_peek_off      = sk_set_peek_off,
+       .set_peek_off      = udp_set_peek_off,
 #ifdef CONFIG_COMPAT
        .compat_ioctl      = inet6_compat_ioctl,
 #endif