1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright 2018 Google Inc.
4 * Author: Eric Dumazet (edumazet@google.com)
6 * Reference program demonstrating tcp mmap() usage,
7 * and SO_RCVLOWAT hints for receiver.
9 * Note : NIC with header split is needed to use mmap() on TCP :
10 * Each incoming frame must be a multiple of PAGE_SIZE bytes of TCP payload.
12 * How to use on loopback interface :
14 * ifconfig lo mtu 61512 # 15*4096 + 40 (ipv6 header) + 32 (TCP with TS option header)
18 * Or leave default lo mtu, but use -M option to set TCP_MAXSEG option to (4096 + 12)
19 * (4096 : page size on x86, 12: TCP TS option length)
20 * tcp_mmap -s -z -M $((4096+12)) &
21 * tcp_mmap -H ::1 -z -M $((4096+12))
23 * Note: -z option on sender uses MSG_ZEROCOPY, which forces a copy when packets go through loopback interface.
24 * We might use sendfile() instead, but really this test program is about mmap(), for receivers ;)
26 * $ ./tcp_mmap -s & # Without mmap()
27 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
28 * received 32768 MB (0 % mmap'ed) in 14.1157 s, 19.4732 Gbit
29 * cpu usage user:0.057 sys:7.815, 240.234 usec per MB, 65531 c-switches
30 * received 32768 MB (0 % mmap'ed) in 14.6833 s, 18.7204 Gbit
31 * cpu usage user:0.043 sys:8.103, 248.596 usec per MB, 65524 c-switches
32 * received 32768 MB (0 % mmap'ed) in 11.143 s, 24.6682 Gbit
33 * cpu usage user:0.044 sys:6.576, 202.026 usec per MB, 65519 c-switches
34 * received 32768 MB (0 % mmap'ed) in 14.9056 s, 18.4413 Gbit
35 * cpu usage user:0.036 sys:8.193, 251.129 usec per MB, 65530 c-switches
36 * $ kill %1 # kill tcp_mmap server
38 * $ ./tcp_mmap -s -z & # With mmap()
39 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
40 * received 32768 MB (99.9939 % mmap'ed) in 6.73792 s, 40.7956 Gbit
41 * cpu usage user:0.045 sys:2.827, 87.6465 usec per MB, 65532 c-switches
42 * received 32768 MB (99.9939 % mmap'ed) in 7.26732 s, 37.8238 Gbit
43 * cpu usage user:0.037 sys:3.087, 95.3369 usec per MB, 65532 c-switches
44 * received 32768 MB (99.9939 % mmap'ed) in 7.61661 s, 36.0893 Gbit
45 * cpu usage user:0.046 sys:3.559, 110.016 usec per MB, 65529 c-switches
46 * received 32768 MB (99.9939 % mmap'ed) in 7.43764 s, 36.9577 Gbit
47 * cpu usage user:0.035 sys:3.467, 106.873 usec per MB, 65530 c-switches
51 #include <sys/types.h>
54 #include <sys/socket.h>
56 #include <sys/resource.h>
64 #include <netinet/in.h>
65 #include <arpa/inet.h>
67 #include <linux/tcp.h>
71 #define MSG_ZEROCOPY 0x4000000
74 #define FILE_SZ (1UL << 35)
75 static int cfg_family = AF_INET6;
76 static socklen_t cfg_alen = sizeof(struct sockaddr_in6);
77 static int cfg_port = 8787;
79 static int rcvbuf; /* Default: autotuning. Can be set with -r <integer> option */
80 static int sndbuf; /* Default: autotuning. Can be set with -w <integer> option */
81 static int zflg; /* zero copy option. (MSG_ZEROCOPY for sender, mmap() for receiver */
82 static int xflg; /* hash received data (simple xor) (-h option) */
83 static int keepflag; /* -k option: receiver shall keep all received file in memory (no munmap() calls) */
85 static int chunk_size = 512*1024;
89 static inline void prefetch(const void *x)
91 #if defined(__x86_64__)
92 asm volatile("prefetcht0 %P0" : : "m" (*(const char *)x));
96 void hash_zone(void *zone, unsigned int length)
98 unsigned long temp = htotal;
100 while (length >= 8*sizeof(long)) {
101 prefetch(zone + 384);
102 temp ^= *(unsigned long *)zone;
103 temp ^= *(unsigned long *)(zone + sizeof(long));
104 temp ^= *(unsigned long *)(zone + 2*sizeof(long));
105 temp ^= *(unsigned long *)(zone + 3*sizeof(long));
106 temp ^= *(unsigned long *)(zone + 4*sizeof(long));
107 temp ^= *(unsigned long *)(zone + 5*sizeof(long));
108 temp ^= *(unsigned long *)(zone + 6*sizeof(long));
109 temp ^= *(unsigned long *)(zone + 7*sizeof(long));
110 zone += 8*sizeof(long);
111 length -= 8*sizeof(long);
113 while (length >= 1) {
114 temp ^= *(unsigned char *)zone;
121 void *child_thread(void *arg)
123 unsigned long total_mmap = 0, total = 0;
124 struct tcp_zerocopy_receive zc;
125 unsigned long delta_usec;
126 int flags = MAP_SHARED;
127 struct timeval t0, t1;
134 fd = (int)(unsigned long)arg;
136 gettimeofday(&t0, NULL);
138 fcntl(fd, F_SETFL, O_NDELAY);
139 buffer = malloc(chunk_size);
145 addr = mmap(NULL, chunk_size, PROT_READ, flags, fd, 0);
146 if (addr == (void *)-1)
150 struct pollfd pfd = { .fd = fd, .events = POLLIN, };
153 poll(&pfd, 1, 10000);
155 socklen_t zc_len = sizeof(zc);
158 zc.address = (__u64)addr;
159 zc.length = chunk_size;
160 zc.recv_skip_hint = 0;
161 res = getsockopt(fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE,
167 assert(zc.length <= chunk_size);
168 total_mmap += zc.length;
170 hash_zone(addr, zc.length);
173 if (zc.recv_skip_hint) {
174 assert(zc.recv_skip_hint <= chunk_size);
175 lu = read(fd, buffer, zc.recv_skip_hint);
178 hash_zone(buffer, lu);
185 while (sub < chunk_size) {
186 lu = read(fd, buffer + sub, chunk_size - sub);
192 hash_zone(buffer + sub, lu);
198 gettimeofday(&t1, NULL);
199 delta_usec = (t1.tv_sec - t0.tv_sec) * 1000000 + t1.tv_usec - t0.tv_usec;
203 throughput = total * 8.0 / (double)delta_usec / 1000.0;
204 getrusage(RUSAGE_THREAD, &ru);
205 if (total > 1024*1024) {
206 unsigned long total_usec;
207 unsigned long mb = total >> 20;
208 total_usec = 1000000*ru.ru_utime.tv_sec + ru.ru_utime.tv_usec +
209 1000000*ru.ru_stime.tv_sec + ru.ru_stime.tv_usec;
210 printf("received %lg MB (%lg %% mmap'ed) in %lg s, %lg Gbit\n"
211 " cpu usage user:%lg sys:%lg, %lg usec per MB, %lu c-switches\n",
212 total / (1024.0 * 1024.0),
213 100.0*total_mmap/total,
214 (double)delta_usec / 1000000.0,
216 (double)ru.ru_utime.tv_sec + (double)ru.ru_utime.tv_usec / 1000000.0,
217 (double)ru.ru_stime.tv_sec + (double)ru.ru_stime.tv_usec / 1000000.0,
218 (double)total_usec/mb,
225 munmap(addr, chunk_size);
229 static void apply_rcvsnd_buf(int fd)
231 if (rcvbuf && setsockopt(fd, SOL_SOCKET,
232 SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)) == -1) {
233 perror("setsockopt SO_RCVBUF");
236 if (sndbuf && setsockopt(fd, SOL_SOCKET,
237 SO_SNDBUF, &sndbuf, sizeof(sndbuf)) == -1) {
238 perror("setsockopt SO_SNDBUF");
243 static void setup_sockaddr(int domain, const char *str_addr,
244 struct sockaddr_storage *sockaddr)
246 struct sockaddr_in6 *addr6 = (void *) sockaddr;
247 struct sockaddr_in *addr4 = (void *) sockaddr;
251 memset(addr4, 0, sizeof(*addr4));
252 addr4->sin_family = AF_INET;
253 addr4->sin_port = htons(cfg_port);
255 inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)
256 error(1, 0, "ipv4 parse error: %s", str_addr);
259 memset(addr6, 0, sizeof(*addr6));
260 addr6->sin6_family = AF_INET6;
261 addr6->sin6_port = htons(cfg_port);
263 inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)
264 error(1, 0, "ipv6 parse error: %s", str_addr);
267 error(1, 0, "illegal domain");
271 static void do_accept(int fdlisten)
273 if (setsockopt(fdlisten, SOL_SOCKET, SO_RCVLOWAT,
274 &chunk_size, sizeof(chunk_size)) == -1) {
275 perror("setsockopt SO_RCVLOWAT");
278 apply_rcvsnd_buf(fdlisten);
281 struct sockaddr_in addr;
282 socklen_t addrlen = sizeof(addr);
286 fd = accept(fdlisten, (struct sockaddr *)&addr, &addrlen);
291 res = pthread_create(&th, NULL, child_thread,
292 (void *)(unsigned long)fd);
295 perror("pthread_create");
301 int main(int argc, char *argv[])
303 struct sockaddr_storage listenaddr, addr;
304 unsigned int max_pacing_rate = 0;
305 unsigned long total = 0;
312 while ((c = getopt(argc, argv, "46p:svr:w:H:zxkP:M:")) != -1) {
315 cfg_family = PF_INET;
316 cfg_alen = sizeof(struct sockaddr_in);
319 cfg_family = PF_INET6;
320 cfg_alen = sizeof(struct sockaddr_in6);
323 cfg_port = atoi(optarg);
328 case 's': /* server : listen for incoming connections */
332 rcvbuf = atoi(optarg);
335 sndbuf = atoi(optarg);
350 max_pacing_rate = atoi(optarg) ;
357 int fdlisten = socket(cfg_family, SOCK_STREAM, 0);
359 if (fdlisten == -1) {
363 apply_rcvsnd_buf(fdlisten);
364 setsockopt(fdlisten, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
366 setup_sockaddr(cfg_family, host, &listenaddr);
369 setsockopt(fdlisten, IPPROTO_TCP, TCP_MAXSEG,
370 &mss, sizeof(mss)) == -1) {
371 perror("setsockopt TCP_MAXSEG");
374 if (bind(fdlisten, (const struct sockaddr *)&listenaddr, cfg_alen) == -1) {
378 if (listen(fdlisten, 128) == -1) {
384 buffer = mmap(NULL, chunk_size, PROT_READ | PROT_WRITE,
385 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
386 if (buffer == (char *)-1) {
391 fd = socket(cfg_family, SOCK_STREAM, 0);
396 apply_rcvsnd_buf(fd);
398 setup_sockaddr(cfg_family, host, &addr);
401 setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss)) == -1) {
402 perror("setsockopt TCP_MAXSEG");
405 if (connect(fd, (const struct sockaddr *)&addr, cfg_alen) == -1) {
409 if (max_pacing_rate &&
410 setsockopt(fd, SOL_SOCKET, SO_MAX_PACING_RATE,
411 &max_pacing_rate, sizeof(max_pacing_rate)) == -1)
412 perror("setsockopt SO_MAX_PACING_RATE");
414 if (zflg && setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY,
415 &on, sizeof(on)) == -1) {
416 perror("setsockopt SO_ZEROCOPY, (-z option disabled)");
419 while (total < FILE_SZ) {
420 long wr = FILE_SZ - total;
424 /* Note : we just want to fill the pipe with 0 bytes */
425 wr = send(fd, buffer, wr, zflg ? MSG_ZEROCOPY : 0);
431 munmap(buffer, chunk_size);