optimize hot paths of getc with manual shrink-wrapping

with these changes, in a program that has not created any threads
besides the main thread and that has not called f[try]lockfile, getc
performs indistinguishably from getc_unlocked. this was measured on
several i386 and x86_64 models, and should hold on other archs too
simply by the properties of the code generation.

the case where the caller already holds the lock (via flockfile) is
improved significantly as well (40-60% reduction in time on machines
tested) and the case where locking is needed is improved somewhat
(roughly 10%).

the key technique used here is forcing the non-hot path out-of-line
and enabling it to be a tail call. a static noinline function
(conditional on __GNUC__) is used rather than the extern hiddens used
elsewhere for this purpose, so that the compiler can choose
non-default calling conventions, making it possible to tail-call to a
callee that takes more arguments than the caller on archs where
arguments are passed on the stack or must have space reserved on the
stack for spilling the. the tid could just be reloaded via the thread
pointer in locking_getc, but that would be ridiculously expensive on
some archs where thread pointer load requires a trap or syscall.

diff --git a/src/stdio/fgetc.c b/src/stdio/fgetc.c
index e122416406d8c8351b29b9bde71573dd355f894d..2578afccd28ac8f75e8bb470e99ca74015cb75f2 100644 (file)
--- a/src/stdio/fgetc.c
+++ b/src/stdio/fgetc.c
@@ -1,11 +1,7 @@
-#include "stdio_impl.h"
+#include <stdio.h>
+#include "getc.h"
 
 int fgetc(FILE *f)
 {
-       int c;
-       if (f->lock < 0 || !__lockfile(f))
-               return getc_unlocked(f);
-       c = getc_unlocked(f);
-       __unlockfile(f);
-       return c;
+       return do_getc(f);
 }

diff --git a/src/stdio/getc.c b/src/stdio/getc.c
index b3f351d1d7e45e5f4ef9ad91c868981e6aaf1929..8409fc2343fc9060da320ccefe93b394b7801a2b 100644 (file)
--- a/src/stdio/getc.c
+++ b/src/stdio/getc.c
@@ -1,13 +1,9 @@
-#include "stdio_impl.h"
+#include <stdio.h>
+#include "getc.h"
 
 int getc(FILE *f)
 {
-       int c;
-       if (f->lock < 0 || !__lockfile(f))
-               return getc_unlocked(f);
-       c = getc_unlocked(f);
-       __unlockfile(f);
-       return c;
+       return do_getc(f);
 }
 
 weak_alias(getc, _IO_getc);

diff --git a/src/stdio/getc.h b/src/stdio/getc.h
new file mode 100644 (file)
index 0000000..0657ab6
--- /dev/null
+++ b/src/stdio/getc.h
@@ -0,0 +1,22 @@
+#include "stdio_impl.h"
+#include "pthread_impl.h"
+
+#ifdef __GNUC__
+__attribute__((__noinline__))
+#endif
+static int locking_getc(FILE *f, int tid)
+{
+       if (a_cas(&f->lock, 0, tid)) __lockfile(f);
+       int c = getc_unlocked(f);
+       if (a_swap(&f->lock, 0) & MAYBE_WAITERS)
+               __wake(&f->lock, 1, 1);
+       return c;
+}
+
+static inline int do_getc(FILE *f)
+{
+       int tid, l = f->lock;
+       if (l < 0 || (l & ~MAYBE_WAITERS) == (tid=__pthread_self()->tid))
+               return getc_unlocked(f);
+       return locking_getc(f, tid);
+}

diff --git a/src/stdio/getchar.c b/src/stdio/getchar.c
index c10126581bf95d532e974fe30f6d2646a0ef8db6..df395ca9548dc3591469e0676c334c3e362aba2d 100644 (file)
--- a/src/stdio/getchar.c
+++ b/src/stdio/getchar.c
@@ -1,6 +1,7 @@
 #include <stdio.h>
+#include "getc.h"
 
 int getchar(void)
 {
-       return fgetc(stdin);
+       return do_getc(stdin);
 }