1 /* vi: set sw=4 ts=4: */
3 * Unicode support routines.
5 * Copyright (C) 2009 Denys Vlasenko
7 * Licensed under GPL version 2, see file LICENSE in this tarball for details.
12 /* If it's not #defined as a constant in unicode.h... */
13 #ifndef unicode_status
14 uint8_t unicode_status;
17 /* This file is compiled only if FEATURE_ASSUME_UNICODE is on.
18 * We check other options and decide whether to use libc support
19 * via locale, or use our own logic:
22 #if ENABLE_LOCALE_SUPPORT
24 /* Unicode support using libc locale support. */
26 void FAST_FUNC init_unicode(void)
28 /* In unicode, this is a one character string */
29 static const char unicode_0x394[] = { 0xce, 0x94, 0 };
31 if (unicode_status != UNICODE_UNKNOWN)
34 unicode_status = unicode_strlen(unicode_0x394) == 1 ? UNICODE_ON : UNICODE_OFF;
39 /* Homegrown Unicode support. It knows only C and Unicode locales. */
41 # if ENABLE_FEATURE_CHECK_UNICODE_IN_ENV
42 void FAST_FUNC init_unicode(void)
46 if (unicode_status != UNICODE_UNKNOWN)
49 unicode_status = UNICODE_OFF;
50 lang = getenv("LANG");
51 if (!lang || !(strstr(lang, ".utf") || strstr(lang, ".UTF")))
53 unicode_status = UNICODE_ON;
57 static size_t wcrtomb_internal(char *s, wchar_t wc)
67 /* RFC 3629 says that Unicode ends at 10FFFF,
68 * but we cover entire 32 bits */
70 /* 4000000-FFFFFFFF -> 111111tt 10tttttt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */
71 /* 200000-3FFFFFF -> 111110tt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */
72 /* 10000-1FFFFF -> 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx */
73 /* 800-FFFF -> 1110yyyy 10yyyyxx 10xxxxxx */
74 /* 80-7FF -> 110yyyxx 10xxxxxx */
76 /* How many bytes do we need? */
78 /* (0x80000000+ would result in n = 7, limiting n to 6) */
79 while (v >= 0x800 && n < 6) {
83 /* Fill bytes n-1..1 */
86 s[i] = (wc & 0x3f) | 0x80;
90 s[0] = wc | (uint8_t)(0x3f00 >> n);
93 size_t FAST_FUNC wcrtomb(char *s, wchar_t wc, mbstate_t *ps UNUSED_PARAM)
95 if (unicode_status != UNICODE_ON) {
100 return wcrtomb_internal(s, wc);
102 size_t FAST_FUNC wcstombs(char *dest, const wchar_t *src, size_t n)
106 if (unicode_status != UNICODE_ON) {
117 while (n >= MB_CUR_MAX) {
119 size_t len = wcrtomb_internal(dest, wc);
127 char tbuf[MB_CUR_MAX];
129 size_t len = wcrtomb_internal(tbuf, wc);
133 memcpy(dest, tbuf, len);
142 #define ERROR_WCHAR (~(wchar_t)0)
144 static const char *mbstowc_internal(wchar_t *res, const char *src)
147 unsigned c = (unsigned char) *src++;
154 /* 80-7FF -> 110yyyxx 10xxxxxx */
155 /* 800-FFFF -> 1110yyyy 10yyyyxx 10xxxxxx */
156 /* 10000-1FFFFF -> 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx */
157 /* 200000-3FFFFFF -> 111110tt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */
158 /* 4000000-FFFFFFFF -> 111111tt 10tttttt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */
163 } while ((c & 0x80) && bytes < 6);
165 /* A bare "continuation" byte. Say, 80 */
169 c = (uint8_t)(c) >> bytes;
172 unsigned ch = (unsigned char) *src;
173 if ((ch & 0xc0) != 0x80) {
174 /* Missing "continuation" byte. Example: e0 80 */
178 c = (c << 6) + (ch & 0x3f);
183 /* Need to check that c isn't produced by overlong encoding */
184 /* Example: 11000000 10000000 converts to NUL */
185 /* 11110000 10000000 10000100 10000000 converts to 0x100 */
186 /* correct encoding: 11000100 10000000 */
187 if (c <= 0x7f) { /* crude check */
195 size_t FAST_FUNC mbstowcs(wchar_t *dest, const char *src, size_t n)
199 if (unicode_status != UNICODE_ON) {
201 unsigned char c = *src++;
214 src = mbstowc_internal(&wc, src);
215 if (wc == ERROR_WCHAR) /* error */
219 if (wc == 0) /* end-of-string */
227 int FAST_FUNC iswspace(wint_t wc)
229 return (unsigned)wc <= 0x7f && isspace(wc);
232 int FAST_FUNC iswalnum(wint_t wc)
234 return (unsigned)wc <= 0x7f && isalnum(wc);
237 int FAST_FUNC iswpunct(wint_t wc)
239 return (unsigned)wc <= 0x7f && ispunct(wc);
242 #include "unicode_wcwidth.c"
244 # if ENABLE_UNICODE_BIDI_SUPPORT
245 int FAST_FUNC unicode_isrtl(wint_t wc)
248 * http://www.unicode.org/Public/5.2.0/ucd/extracted/DerivedBidiClass.txt
249 * Bidi_Class=Left_To_Right | Bidi_Class=Arabic_Letter
251 static const struct interval rtl_b[] = {
252 # define BIG_(a,b) { a, b },
292 /* Probably not necessary
309 static const uint16_t rtl_p[] = {
311 # define PAIR(a,b) (a << 2) | (b-a),
312 /* Exact copy-n-paste of the above: */
351 /* Probably not necessary
368 if (in_interval_table(wc, rtl_b, ARRAY_SIZE(rtl_b) - 1))
370 if (in_uint16_table(wc, rtl_p, ARRAY_SIZE(rtl_p) - 1))
374 # endif /* UNICODE_BIDI_SUPPORT */
376 #endif /* Homegrown Unicode support */
379 /* The rest is mostly same for libc and for "homegrown" support */
381 size_t FAST_FUNC unicode_strlen(const char *string)
383 size_t width = mbstowcs(NULL, string, INT_MAX);
384 if (width == (size_t)-1L)
385 return strlen(string);
389 static char* FAST_FUNC unicode_conv_to_printable2(uni_stat_t *stats, const char *src, unsigned width, int flags)
396 if (unicode_status != UNICODE_ON) {
398 if (flags & UNI_FLAG_PAD) {
399 d = dst = xmalloc(width + 1);
400 while ((int)--width >= 0) {
401 unsigned char c = *src;
405 while ((int)--width >= 0);
408 *d++ = (c >= ' ' && c < 0x7f) ? c : '?';
413 d = dst = xstrndup(src, width);
415 unsigned char c = *d;
416 if (c < ' ' || c >= 0x7f)
422 stats->byte_count = stats->unicode_count = (d - dst);
427 uni_count = uni_width = 0;
433 #if ENABLE_LOCALE_SUPPORT
435 mbstate_t mbst = { 0 };
436 ssize_t rc = mbsrtowcs(&wc, &src, 1, &mbst);
437 /* If invalid sequence is seen: -1 is returned,
438 * src points to the invalid sequence, errno = EILSEQ.
439 * Else number of wchars (excluding terminating L'\0')
440 * written to dest is returned.
441 * If len (here: 1) non-L'\0' wchars stored at dest,
442 * src points to the next char to be converted.
443 * If string is completely converted: src = NULL.
445 if (rc == 0) /* end-of-string */
447 if (rc < 0) { /* error */
455 src = mbstowc_internal(&wc, src);
456 /* src is advanced to next mb char
457 * wc == ERROR_WCHAR: invalid sequence is seen
460 if (wc == ERROR_WCHAR) /* error */
462 if (wc == 0) /* end-of-string */
465 if (CONFIG_LAST_SUPPORTED_WCHAR && wc > CONFIG_LAST_SUPPORTED_WCHAR)
468 if ((ENABLE_UNICODE_COMBINING_WCHARS && w < 0) /* non-printable wchar */
469 || (!ENABLE_UNICODE_COMBINING_WCHARS && w <= 0)
470 || (!ENABLE_UNICODE_WIDE_WCHARS && w > 1)
473 wc = CONFIG_SUBST_WCHAR;
477 /* Note: if width == 0, we still may add more chars,
478 * they may be zero-width or combining ones */
479 if ((int)width < 0) {
480 /* can't add this wc, string would become longer than width */
487 dst = xrealloc(dst, dst_len + MB_CUR_MAX);
488 #if ENABLE_LOCALE_SUPPORT
490 mbstate_t mbst = { 0 };
491 dst_len += wcrtomb(&dst[dst_len], wc, &mbst);
494 dst_len += wcrtomb_internal(&dst[dst_len], wc);
498 /* Pad to remaining width */
499 if (flags & UNI_FLAG_PAD) {
500 dst = xrealloc(dst, dst_len + width + 1);
503 while ((int)--width >= 0) {
504 dst[dst_len++] = ' ';
509 stats->byte_count = dst_len;
510 stats->unicode_count = uni_count;
511 stats->unicode_width = uni_width;
516 char* FAST_FUNC unicode_conv_to_printable(uni_stat_t *stats, const char *src)
518 return unicode_conv_to_printable2(stats, src, INT_MAX, 0);
520 char* FAST_FUNC unicode_conv_to_printable_maxwidth(uni_stat_t *stats, const char *src, unsigned maxwidth)
522 return unicode_conv_to_printable2(stats, src, maxwidth, 0);
524 char* FAST_FUNC unicode_conv_to_printable_fixedwidth(uni_stat_t *stats, const char *src, unsigned width)
526 return unicode_conv_to_printable2(stats, src, width, UNI_FLAG_PAD);
530 unsigned FAST_FUNC unicode_padding_to_width(unsigned width, const char *src)
532 if (unicode_status != UNICODE_ON) {
533 return width - strnlen(src, width);
540 #if ENABLE_LOCALE_SUPPORT
542 mbstate_t mbst = { 0 };
543 ssize_t rc = mbsrtowcs(&wc, &src, 1, &mbst);
544 if (rc <= 0) /* error, or end-of-string */
548 src = mbstowc_internal(&wc, src);
549 if (wc == ERROR_WCHAR || wc == 0) /* error, or end-of-string */
553 if (w < 0) /* non-printable wchar */
556 if ((int)width <= 0) /* string is longer than width */