Android: fix cyrillic characters, update iconv lib (#9117)

diff --git a/build/android/Makefile b/build/android/Makefile
index c5a21c290b72c692b9a96f534273d6f36fa13ca9..9ec237a75483ad7eb186364cc772b9397aac5f03 100644 (file)
--- a/build/android/Makefile
+++ b/build/android/Makefile
@@ -112,7 +112,7 @@ FREETYPE_TIMESTAMP = $(FREETYPE_DIR)timestamp
 FREETYPE_TIMESTAMP_INT = $(ANDR_ROOT)/deps/freetype_timestamp
 FREETYPE_URL_GIT = https://github.com/cdave1/freetype2-android
 
-ICONV_VERSION = 1.14
+ICONV_VERSION = 1.16
 ICONV_DIR = $(ANDR_ROOT)/deps/libiconv/
 ICONV_LIB = $(ICONV_DIR)/lib/.libs/libiconv.so
 ICONV_TIMESTAMP = $(ICONV_DIR)timestamp
@@ -445,9 +445,6 @@ iconv_download :
                tar -xzf libiconv-${ICONV_VERSION}.tar.gz || exit 1;               \
                rm libiconv-${ICONV_VERSION}.tar.gz;                               \
                ln -s libiconv-${ICONV_VERSION} libiconv;                          \
-               cd ${ICONV_DIR};                                                   \
-               patch -p1 < ${ANDR_ROOT}/patches/libiconv_android.patch;           \
-               patch -p1 < ${ANDR_ROOT}/patches/libiconv_stdio.patch;             \
        fi
 
 iconv : $(ICONV_LIB)

diff --git a/build/android/patches/libiconv_android.patch b/build/android/patches/libiconv_android.patch
deleted file mode 100644 (file)
index 4eca0a4..0000000
--- a/build/android/patches/libiconv_android.patch
+++ /dev/null
@@ -1,39 +0,0 @@
---- a/libcharset/lib/localcharset.c    2015-06-10 11:55:25.933870724 +0200
-+++ b/libcharset/lib/localcharset.c    2015-06-10 11:55:39.578063493 +0200
-@@ -47,7 +47,7 @@
- 
- #if !defined WIN32_NATIVE
- # include <unistd.h>
--# if HAVE_LANGINFO_CODESET
-+# if HAVE_LANGINFO_CODESET && !(defined __ANDROID__)
- #  include <langinfo.h>
- # else
- #  if 0 /* see comment below */
-@@ -124,7 +124,7 @@ get_charset_aliases (void)
-   cp = charset_aliases;
-   if (cp == NULL)
-     {
--#if !(defined DARWIN7 || defined VMS || defined WIN32_NATIVE || defined __CYGWIN__)
-+#if !(defined DARWIN7 || defined VMS || defined WIN32_NATIVE || defined __CYGWIN__ || defined __ANDROID__)
-       const char *dir;
-       const char *base = "charset.alias";
-       char *file_name;
-@@ -338,6 +338,9 @@ get_charset_aliases (void)
-            "CP54936" "\0" "GB18030" "\0"
-            "CP65001" "\0" "UTF-8" "\0";
- # endif
-+# if defined __ANDROID__
-+      cp = "*" "\0" "UTF-8" "\0";
-+# endif
- #endif
- 
-       charset_aliases = cp;
-@@ -361,7 +364,7 @@ locale_charset (void)
-   const char *codeset;
-   const char *aliases;
- 
--#if !(defined WIN32_NATIVE || defined OS2)
-+#if !(defined WIN32_NATIVE || defined OS2 || defined __ANDROID__)
- 
- # if HAVE_LANGINFO_CODESET
- 

diff --git a/build/android/patches/libiconv_stdio.patch b/build/android/patches/libiconv_stdio.patch
deleted file mode 100644 (file)
index 9fa50f7..0000000
--- a/build/android/patches/libiconv_stdio.patch
+++ /dev/null
@@ -1,13 +0,0 @@
---- a/srclib/stdio.in.h        2011-08-07 15:42:06.000000000 +0200
-+++ b/srclib/stdio.in.h        2015-06-10 09:27:58.129056262 +0200
-@@ -695,8 +696,9 @@ _GL_CXXALIASWARN (gets);
- /* It is very rare that the developer ever has full control of stdin,
-    so any use of gets warrants an unconditional warning.  Assume it is
-    always declared, since it is required by C89.  */
--_GL_WARN_ON_USE (gets, "gets is a security hole - use fgets instead");
-+/*_GL_WARN_ON_USE (gets, "gets is a security hole - use fgets instead");*/
-+#define gets(a) fgets( a, sizeof(*(a)), stdin)
- #endif
-
-
-#if @GNULIB_OBSTACK_PRINTF@ || @GNULIB_OBSTACK_PRINTF_POSIX@

diff --git a/src/util/string.cpp b/src/util/string.cpp
index caaef9b30a7fcaa8091b357f6b6e1b888e1d8dad..2134fbd15cbc3cf997671884cec0ad82acef25c3 100644 (file)
--- a/src/util/string.cpp
+++ b/src/util/string.cpp
@@ -79,6 +79,13 @@ bool convert(const char *to, const char *from, char *outbuf,
        return true;
 }
 
+#ifdef __ANDROID__
+// Android need manual caring to support the full character set possible with wchar_t
+const char *DEFAULT_ENCODING = "UTF-32LE";
+#else
+const char *DEFAULT_ENCODING = "WCHAR_T";
+#endif
+
 std::wstring utf8_to_wide(const std::string &input)
 {
        size_t inbuf_size = input.length() + 1;
@@ -90,7 +97,12 @@ std::wstring utf8_to_wide(const std::string &input)
        char *outbuf = new char[outbuf_size];
        memset(outbuf, 0, outbuf_size);
 
-       if (!convert("WCHAR_T", "UTF-8", outbuf, outbuf_size, inbuf, inbuf_size)) {
+#ifdef __ANDROID__
+       // Android need manual caring to support the full character set possible with wchar_t
+       SANITY_CHECK(sizeof(wchar_t) == 4);
+#endif
+
+       if (!convert(DEFAULT_ENCODING, "UTF-8", outbuf, outbuf_size, inbuf, inbuf_size)) {
                infostream << "Couldn't convert UTF-8 string 0x" << hex_encode(input)
                        << " into wstring" << std::endl;
                delete[] inbuf;
@@ -105,13 +117,6 @@ std::wstring utf8_to_wide(const std::string &input)
        return out;
 }
 
-#ifdef __ANDROID__
-// TODO: this is an ugly fix for wide_to_utf8 somehow not working on android
-std::string wide_to_utf8(const std::wstring &input)
-{
-       return wide_to_narrow(input);
-}
-#else
 std::string wide_to_utf8(const std::wstring &input)
 {
        size_t inbuf_size = (input.length() + 1) * sizeof(wchar_t);
@@ -123,7 +128,7 @@ std::string wide_to_utf8(const std::wstring &input)
        char *outbuf = new char[outbuf_size];
        memset(outbuf, 0, outbuf_size);
 
-       if (!convert("UTF-8", "WCHAR_T", outbuf, outbuf_size, inbuf, inbuf_size)) {
+       if (!convert("UTF-8", DEFAULT_ENCODING, outbuf, outbuf_size, inbuf, inbuf_size)) {
                infostream << "Couldn't convert wstring 0x" << hex_encode(inbuf, inbuf_size)
                        << " into UTF-8 string" << std::endl;
                delete[] inbuf;
@@ -138,7 +143,6 @@ std::string wide_to_utf8(const std::wstring &input)
        return out;
 }
 
-#endif
 #else // _WIN32
 
 std::wstring utf8_to_wide(const std::string &input)
@@ -183,7 +187,7 @@ wchar_t *utf8_to_wide_c(const char *str)
 // The returned string is allocated using new
 wchar_t *narrow_to_wide_c(const char *str)
 {
-       wchar_t *nstr = NULL;
+       wchar_t *nstr = nullptr;
 #if defined(_WIN32)
        int nResult = MultiByteToWideChar(CP_UTF8, 0, (LPCSTR) str, -1, 0, 0);
        if (nResult == 0) {
@@ -204,67 +208,8 @@ wchar_t *narrow_to_wide_c(const char *str)
        return nstr;
 }
 
-
-#ifdef __ANDROID__
-
-const wchar_t* wide_chars =
-       L" !\"#$%&'()*+,-./0123456789:;<=>?@"
-       L"ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`"
-       L"abcdefghijklmnopqrstuvwxyz{|}~";
-
-int wctomb(char *s, wchar_t wc)
-{
-       for (unsigned int j = 0; j < (sizeof(wide_chars)/sizeof(wchar_t));j++) {
-               if (wc == wide_chars[j]) {
-                       *s = (char) (j+32);
-                       return 1;
-               }
-               else if (wc == L'\n') {
-                       *s = '\n';
-                       return 1;
-               }
-       }
-       return -1;
-}
-
-int mbtowc(wchar_t *pwc, const char *s, size_t n)
-{
-       std::wstring intermediate = narrow_to_wide(s);
-
-       if (intermediate.length() > 0) {
-               *pwc = intermediate[0];
-               return 1;
-       }
-       else {
-               return -1;
-       }
-}
-
 std::wstring narrow_to_wide(const std::string &mbs) {
        size_t wcl = mbs.size();
-
-       std::wstring retval = L"";
-
-       for (unsigned int i = 0; i < wcl; i++) {
-               if (((unsigned char) mbs[i] >31) &&
-                ((unsigned char) mbs[i] < 127)) {
-
-                       retval += wide_chars[(unsigned char) mbs[i] -32];
-               }
-               //handle newline
-               else if (mbs[i] == '\n') {
-                       retval += L'\n';
-               }
-       }
-
-       return retval;
-}
-
-#else // not Android
-
-std::wstring narrow_to_wide(const std::string &mbs)
-{
-       size_t wcl = mbs.size();
        Buffer<wchar_t> wcs(wcl + 1);
        size_t len = mbstowcs(*wcs, mbs.c_str(), wcl);
        if (len == (size_t)(-1))
@@ -273,37 +218,6 @@ std::wstring narrow_to_wide(const std::string &mbs)
        return *wcs;
 }
 
-#endif
-
-#ifdef __ANDROID__
-
-std::string wide_to_narrow(const std::wstring &wcs) {
-       size_t mbl = wcs.size()*4;
-
-       std::string retval = "";
-       for (unsigned int i = 0; i < wcs.size(); i++) {
-               wchar_t char1 = (wchar_t) wcs[i];
-
-               if (char1 == L'\n') {
-                       retval += '\n';
-                       continue;
-               }
-
-               for (unsigned int j = 0; j < wcslen(wide_chars);j++) {
-                       wchar_t char2 = (wchar_t) wide_chars[j];
-
-                       if (char1 == char2) {
-                               char toadd = (j+32);
-                               retval += toadd;
-                               break;
-                       }
-               }
-       }
-
-       return retval;
-}
-
-#else // not Android
 
 std::string wide_to_narrow(const std::wstring &wcs)
 {
@@ -317,7 +231,6 @@ std::string wide_to_narrow(const std::wstring &wcs)
        return *mbs;
 }
 
-#endif
 
 std::string urlencode(const std::string &str)
 {
@@ -361,10 +274,10 @@ u32 readFlagString(std::string str, const FlagDesc *flagdesc, u32 *flagmask)
        u32 mask = 0;
        char *s = &str[0];
        char *flagstr;
-       char *strpos = NULL;
+       char *strpos = nullptr;
 
        while ((flagstr = strtok_r(s, ",", &strpos))) {
-               s = NULL;
+               s = nullptr;
 
                while (*flagstr == ' ' || *flagstr == '\t')
                        flagstr++;
@@ -436,7 +349,7 @@ char *mystrtok_r(char *s, const char *sep, char **lasts)
                s++;
 
        if (!*s)
-               return NULL;
+               return nullptr;
 
        t = s;
        while (*t) {