target/linux/generic/backport-4.9/024-8-tcp-tsq-move-tsq_flags-close-to-sk_wmem_alloc.patch

   1 From 7aa5470c2c09265902b5e4289afa82e4e7c2987e Mon Sep 17 00:00:00 2001
   2 From: Eric Dumazet <edumazet@google.com>
   3 Date: Sat, 3 Dec 2016 11:14:57 -0800
   4 Subject: [PATCH 08/10] tcp: tsq: move tsq_flags close to sk_wmem_alloc
   5
   6 tsq_flags being in the same cache line than sk_wmem_alloc
   7 makes a lot of sense. Both fields are changed from tcp_wfree()
   8 and more generally by various TSQ related functions.
   9
  10 Prior patch made room in struct sock and added sk_tsq_flags,
  11 this patch deletes tsq_flags from struct tcp_sock.
  12
  13 Signed-off-by: Eric Dumazet <edumazet@google.com>
  14 Signed-off-by: David S. Miller <davem@davemloft.net>
  15 ---
  16  include/linux/tcp.h   |  1 -
  17  net/ipv4/tcp.c        |  4 ++--
  18  net/ipv4/tcp_ipv4.c   |  2 +-
  19  net/ipv4/tcp_output.c | 24 +++++++++++-------------
  20  net/ipv4/tcp_timer.c  |  4 ++--
  21  net/ipv6/tcp_ipv6.c   |  2 +-
  22  6 files changed, 17 insertions(+), 20 deletions(-)
  23
  24 --- a/include/linux/tcp.h
  25 +++ b/include/linux/tcp.h
  26 @@ -192,7 +192,6 @@ struct tcp_sock {
  27         u32     tsoffset;       /* timestamp offset */
  28
  29         struct list_head tsq_node; /* anchor in tsq_tasklet.head list */
  30 -       unsigned long   tsq_flags;
  31
  32         /* Data for direct copy to user */
  33         struct {
  34 --- a/net/ipv4/tcp.c
  35 +++ b/net/ipv4/tcp.c
  36 @@ -665,9 +665,9 @@ static void tcp_push(struct sock *sk, in
  37         if (tcp_should_autocork(sk, skb, size_goal)) {
  38
  39                 /* avoid atomic op if TSQ_THROTTLED bit is already set */
  40 -               if (!test_bit(TSQ_THROTTLED, &tp->tsq_flags)) {
  41 +               if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) {
  42                         NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
  43 -                       set_bit(TSQ_THROTTLED, &tp->tsq_flags);
  44 +                       set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
  45                 }
  46                 /* It is possible TX completion already happened
  47                  * before we set TSQ_THROTTLED.
  48 --- a/net/ipv4/tcp_ipv4.c
  49 +++ b/net/ipv4/tcp_ipv4.c
  50 @@ -446,7 +446,7 @@ void tcp_v4_err(struct sk_buff *icmp_skb
  51                         if (!sock_owned_by_user(sk)) {
  52                                 tcp_v4_mtu_reduced(sk);
  53                         } else {
  54 -                               if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &tp->tsq_flags))
  55 +                               if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
  56                                         sock_hold(sk);
  57                         }
  58                         goto out;
  59 --- a/net/ipv4/tcp_output.c
  60 +++ b/net/ipv4/tcp_output.c
  61 @@ -772,14 +772,15 @@ static void tcp_tasklet_func(unsigned lo
  62         list_for_each_safe(q, n, &list) {
  63                 tp = list_entry(q, struct tcp_sock, tsq_node);
  64                 list_del(&tp->tsq_node);
  65 -               clear_bit(TSQ_QUEUED, &tp->tsq_flags);
  66
  67                 sk = (struct sock *)tp;
  68 +               clear_bit(TSQ_QUEUED, &sk->sk_tsq_flags);
  69 +
  70                 if (!sk->sk_lock.owned &&
  71 -                   test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags)) {
  72 +                   test_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags)) {
  73                         bh_lock_sock(sk);
  74                         if (!sock_owned_by_user(sk)) {
  75 -                               clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
  76 +                               clear_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags);
  77                                 tcp_tsq_handler(sk);
  78                         }
  79                         bh_unlock_sock(sk);
  80 @@ -802,16 +803,15 @@ static void tcp_tasklet_func(unsigned lo
  81   */
  82  void tcp_release_cb(struct sock *sk)
  83  {
  84 -       struct tcp_sock *tp = tcp_sk(sk);
  85         unsigned long flags, nflags;
  86
  87         /* perform an atomic operation only if at least one flag is set */
  88         do {
  89 -               flags = tp->tsq_flags;
  90 +               flags = sk->sk_tsq_flags;
  91                 if (!(flags & TCP_DEFERRED_ALL))
  92                         return;
  93                 nflags = flags & ~TCP_DEFERRED_ALL;
  94 -       } while (cmpxchg(&tp->tsq_flags, flags, nflags) != flags);
  95 +       } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags);
  96
  97         if (flags & TCPF_TSQ_DEFERRED)
  98                 tcp_tsq_handler(sk);
  99 @@ -883,7 +883,7 @@ void tcp_wfree(struct sk_buff *skb)
 100         if (wmem >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current)
 101                 goto out;
 102
 103 -       for (oval = READ_ONCE(tp->tsq_flags);; oval = nval) {
 104 +       for (oval = READ_ONCE(sk->sk_tsq_flags);; oval = nval) {
 105                 struct tsq_tasklet *tsq;
 106                 bool empty;
 107
 108 @@ -891,7 +891,7 @@ void tcp_wfree(struct sk_buff *skb)
 109                         goto out;
 110
 111                 nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED;
 112 -               nval = cmpxchg(&tp->tsq_flags, oval, nval);
 113 +               nval = cmpxchg(&sk->sk_tsq_flags, oval, nval);
 114                 if (nval != oval)
 115                         continue;
 116
 117 @@ -2136,7 +2136,7 @@ static bool tcp_small_queue_check(struct
 118                     skb->prev == sk->sk_write_queue.next)
 119                         return false;
 120
 121 -               set_bit(TSQ_THROTTLED, &tcp_sk(sk)->tsq_flags);
 122 +               set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
 123                 /* It is possible TX completion already happened
 124                  * before we set TSQ_THROTTLED, so we must
 125                  * test again the condition.
 126 @@ -2234,8 +2234,8 @@ static bool tcp_write_xmit(struct sock *
 127                     unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
 128                         break;
 129
 130 -               if (test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags))
 131 -                       clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
 132 +               if (test_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags))
 133 +                       clear_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags);
 134                 if (tcp_small_queue_check(sk, skb, 0))
 135                         break;
 136
 137 @@ -3546,8 +3546,6 @@ void __tcp_send_ack(struct sock *sk, u32
 138         /* We do not want pure acks influencing TCP Small Queues or fq/pacing
 139          * too much.
 140          * SKB_TRUESIZE(max(1 .. 66, MAX_TCP_HEADER)) is unfortunately ~784
 141 -        * We also avoid tcp_wfree() overhead (cache line miss accessing
 142 -        * tp->tsq_flags) by using regular sock_wfree()
 143          */
 144         skb_set_tcp_pure_ack(buff);
 145
 146 --- a/net/ipv4/tcp_timer.c
 147 +++ b/net/ipv4/tcp_timer.c
 148 @@ -326,7 +326,7 @@ static void tcp_delack_timer(unsigned lo
 149                 inet_csk(sk)->icsk_ack.blocked = 1;
 150                 __NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOCKED);
 151                 /* deleguate our work to tcp_release_cb() */
 152 -               if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, &tcp_sk(sk)->tsq_flags))
 153 +               if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, &sk->sk_tsq_flags))
 154                         sock_hold(sk);
 155         }
 156         bh_unlock_sock(sk);
 157 @@ -609,7 +609,7 @@ static void tcp_write_timer(unsigned lon
 158                 tcp_write_timer_handler(sk);
 159         } else {
 160                 /* delegate our work to tcp_release_cb() */
 161 -               if (!test_and_set_bit(TCP_WRITE_TIMER_DEFERRED, &tcp_sk(sk)->tsq_flags))
 162 +               if (!test_and_set_bit(TCP_WRITE_TIMER_DEFERRED, &sk->sk_tsq_flags))
 163                         sock_hold(sk);
 164         }
 165         bh_unlock_sock(sk);
 166 --- a/net/ipv6/tcp_ipv6.c
 167 +++ b/net/ipv6/tcp_ipv6.c
 168 @@ -404,7 +404,7 @@ static void tcp_v6_err(struct sk_buff *s
 169                 if (!sock_owned_by_user(sk))
 170                         tcp_v6_mtu_reduced(sk);
 171                 else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED,
 172 -                                          &tp->tsq_flags))
 173 +                                          &sk->sk_tsq_flags))
 174                         sock_hold(sk);
 175                 goto out;
 176         }