target/linux/generic/backport-4.9/024-3-tcp-tsq-add-shortcut-in-tcp_tasklet_func.patch

   1 From b223feb9de2a65c533ff95c08e834fa732906ea5 Mon Sep 17 00:00:00 2001
   2 From: Eric Dumazet <edumazet@google.com>
   3 Date: Sat, 3 Dec 2016 11:14:52 -0800
   4 Subject: [PATCH 03/10] tcp: tsq: add shortcut in tcp_tasklet_func()
   5
   6 Under high stress, I've seen tcp_tasklet_func() consuming
   7 ~700 usec, handling ~150 tcp sockets.
   8
   9 By setting TCP_TSQ_DEFERRED in tcp_wfree(), we give a chance
  10 for other cpus/threads entering tcp_write_xmit() to grab it,
  11 allowing tcp_tasklet_func() to skip sockets that already did
  12 an xmit cycle.
  13
  14 In the future, we might give to ACK processing an increased
  15 budget to reduce even more tcp_tasklet_func() amount of work.
  16
  17 Signed-off-by: Eric Dumazet <edumazet@google.com>
  18 Signed-off-by: David S. Miller <davem@davemloft.net>
  19 ---
  20  net/ipv4/tcp_output.c | 22 ++++++++++++----------
  21  1 file changed, 12 insertions(+), 10 deletions(-)
  22
  23 --- a/net/ipv4/tcp_output.c
  24 +++ b/net/ipv4/tcp_output.c
  25 @@ -772,19 +772,19 @@ static void tcp_tasklet_func(unsigned lo
  26         list_for_each_safe(q, n, &list) {
  27                 tp = list_entry(q, struct tcp_sock, tsq_node);
  28                 list_del(&tp->tsq_node);
  29 +               clear_bit(TSQ_QUEUED, &tp->tsq_flags);
  30
  31                 sk = (struct sock *)tp;
  32 -               bh_lock_sock(sk);
  33 -
  34 -               if (!sock_owned_by_user(sk)) {
  35 -                       tcp_tsq_handler(sk);
  36 -               } else {
  37 -                       /* defer the work to tcp_release_cb() */
  38 -                       set_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
  39 +               if (!sk->sk_lock.owned &&
  40 +                   test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags)) {
  41 +                       bh_lock_sock(sk);
  42 +                       if (!sock_owned_by_user(sk)) {
  43 +                               clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
  44 +                               tcp_tsq_handler(sk);
  45 +                       }
  46 +                       bh_unlock_sock(sk);
  47                 }
  48 -               bh_unlock_sock(sk);
  49
  50 -               clear_bit(TSQ_QUEUED, &tp->tsq_flags);
  51                 sk_free(sk);
  52         }
  53  }
  54 @@ -889,7 +889,7 @@ void tcp_wfree(struct sk_buff *skb)
  55                 if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED))
  56                         goto out;
  57
  58 -               nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED;
  59 +               nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED;
  60                 nval = cmpxchg(&tp->tsq_flags, oval, nval);
  61                 if (nval != oval)
  62                         continue;
  63 @@ -2222,6 +2222,8 @@ static bool tcp_write_xmit(struct sock *
  64                     unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
  65                         break;
  66
  67 +               if (test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags))
  68 +                       clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
  69                 if (tcp_small_queue_check(sk, skb, 0))
  70                         break;
  71