2 Unix SMB/CIFS implementation.
3 minimal iconv implementation
4 Copyright (C) Andrew Tridgell 2001
5 Copyright (C) Jelmer Vernooij 2002
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with this program. If not, see <http://www.gnu.org/licenses/>.
22 #include "../lib/util/dlinklist.h"
23 #include "system/iconv.h"
24 #include "system/filesys.h"
30 #ifdef static_decl_charset
37 * @brief Samba wrapper/stub for iconv character set conversion.
39 * iconv is the XPG2 interface for converting between character
40 * encodings. This file provides a Samba wrapper around it, and also
41 * a simple reimplementation that is used if the system does not
44 * Samba only works with encodings that are supersets of ASCII: ascii
45 * characters like whitespace can be tested for directly, multibyte
46 * sequences start with a byte with the high bit set, and strings are
47 * terminated by a nul byte.
49 * Note that the only function provided by iconv is conversion between
50 * characters. It doesn't directly support operations like
51 * uppercasing or comparison. We have to convert to UTF-16LE and
54 * @sa Samba Developers Guide
57 static size_t ascii_pull (void *,const char **, size_t *, char **, size_t *);
58 static size_t ascii_push (void *,const char **, size_t *, char **, size_t *);
59 static size_t latin1_push(void *,const char **, size_t *, char **, size_t *);
60 static size_t utf8_pull (void *,const char **, size_t *, char **, size_t *);
61 static size_t utf8_push (void *,const char **, size_t *, char **, size_t *);
62 static size_t utf16_munged_pull(void *,const char **, size_t *, char **, size_t *);
63 static size_t ucs2hex_pull(void *,const char **, size_t *, char **, size_t *);
64 static size_t ucs2hex_push(void *,const char **, size_t *, char **, size_t *);
65 static size_t iconv_copy (void *,const char **, size_t *, char **, size_t *);
66 static size_t iconv_swab (void *,const char **, size_t *, char **, size_t *);
68 static const struct charset_functions builtin_functions[] = {
69 /* windows is closest to UTF-16 */
70 {"UCS-2LE", iconv_copy, iconv_copy},
71 {"UTF-16LE", iconv_copy, iconv_copy},
72 {"UCS-2BE", iconv_swab, iconv_swab},
73 {"UTF-16BE", iconv_swab, iconv_swab},
75 /* we include the UTF-8 alias to cope with differing locale settings */
76 {"UTF8", utf8_pull, utf8_push},
77 {"UTF-8", utf8_pull, utf8_push},
79 /* this handles the munging needed for String2Key */
80 {"UTF16_MUNGED", utf16_munged_pull, iconv_copy},
82 {"ASCII", ascii_pull, ascii_push},
83 {"646", ascii_pull, ascii_push},
84 {"ISO-8859-1", ascii_pull, latin1_push},
85 {"UCS2-HEX", ucs2hex_pull, ucs2hex_push}
88 static struct charset_functions *charsets = NULL;
90 static struct charset_functions *find_charset_functions(const char *name)
92 struct charset_functions *c;
94 /* Check whether we already have this charset... */
95 for (c = charsets; c != NULL; c = c->next) {
96 if(strcasecmp(c->name, name) == 0) {
105 bool smb_register_charset(const struct charset_functions *funcs_in)
107 struct charset_functions *funcs;
109 DEBUG(5, ("Attempting to register new charset %s\n", funcs_in->name));
110 /* Check whether we already have this charset... */
111 if (find_charset_functions(funcs_in->name)) {
112 DEBUG(0, ("Duplicate charset %s, not registering\n", funcs_in->name));
116 funcs = talloc(NULL, struct charset_functions);
118 DEBUG(0, ("Out of memory duplicating charset %s\n", funcs_in->name));
123 funcs->next = funcs->prev = NULL;
124 DEBUG(5, ("Registered charset %s\n", funcs->name));
125 DLIST_ADD(charsets, funcs);
129 static void lazy_initialize_iconv(void)
131 #ifdef static_init_charset
132 static bool initialized = false;
141 #ifdef HAVE_NATIVE_ICONV
142 /* if there was an error then reset the internal state,
143 this ensures that we don't have a shift state remaining for
144 character sets like SJIS */
145 static size_t sys_iconv(void *cd,
146 const char **inbuf, size_t *inbytesleft,
147 char **outbuf, size_t *outbytesleft)
149 size_t ret = iconv((iconv_t)cd,
150 discard_const_p(char *, inbuf), inbytesleft,
151 outbuf, outbytesleft);
152 if (ret == (size_t)-1) iconv(cd, NULL, NULL, NULL, NULL);
158 * This is a simple portable iconv() implementaion.
160 * It only knows about a very small number of character sets - just
161 * enough that Samba works on systems that don't have iconv.
163 _PUBLIC_ size_t smb_iconv(smb_iconv_t cd,
164 const char **inbuf, size_t *inbytesleft,
165 char **outbuf, size_t *outbytesleft)
167 /* in many cases we can go direct */
169 return cd->direct(cd->cd_direct,
170 inbuf, inbytesleft, outbuf, outbytesleft);
173 /* otherwise we have to do it chunks at a time */
175 #ifndef SMB_ICONV_BUFSIZE
176 #define SMB_ICONV_BUFSIZE 2048
182 #if _SAMBA_BUILD_ == 3
183 mem_ctx = talloc_tos();
187 cvtbuf = talloc_array(mem_ctx, char, SMB_ICONV_BUFSIZE);
193 while (*inbytesleft > 0) {
194 char *bufp1 = cvtbuf;
195 const char *bufp2 = cvtbuf;
197 bufsize = SMB_ICONV_BUFSIZE;
199 if (cd->pull(cd->cd_pull,
200 inbuf, inbytesleft, &bufp1, &bufsize) == -1
206 bufsize = SMB_ICONV_BUFSIZE - bufsize;
208 if (cd->push(cd->cd_push,
210 outbuf, outbytesleft) == -1) {
221 static bool is_utf16(const char *name)
223 return strcasecmp(name, "UCS-2LE") == 0 ||
224 strcasecmp(name, "UTF-16LE") == 0;
227 static int smb_iconv_t_destructor(smb_iconv_t hwd)
229 #ifdef HAVE_NATIVE_ICONV
230 if (hwd->cd_pull != NULL && hwd->cd_pull != (iconv_t)-1)
231 iconv_close(hwd->cd_pull);
232 if (hwd->cd_push != NULL && hwd->cd_push != (iconv_t)-1)
233 iconv_close(hwd->cd_push);
234 if (hwd->cd_direct != NULL && hwd->cd_direct != (iconv_t)-1)
235 iconv_close(hwd->cd_direct);
241 _PUBLIC_ smb_iconv_t smb_iconv_open_ex(TALLOC_CTX *mem_ctx, const char *tocode,
242 const char *fromcode, bool native_iconv)
245 const struct charset_functions *from=NULL, *to=NULL;
248 lazy_initialize_iconv();
250 ret = (smb_iconv_t)talloc_named(mem_ctx,
252 "iconv(%s,%s)", tocode, fromcode);
255 return (smb_iconv_t)-1;
257 memset(ret, 0, sizeof(*ret));
258 talloc_set_destructor(ret, smb_iconv_t_destructor);
260 /* check for the simplest null conversion */
261 if (strcmp(fromcode, tocode) == 0) {
262 ret->direct = iconv_copy;
266 for (i=0;i<ARRAY_SIZE(builtin_functions);i++) {
267 if (strcasecmp(fromcode, builtin_functions[i].name) == 0) {
268 from = &builtin_functions[i];
270 if (strcasecmp(tocode, builtin_functions[i].name) == 0) {
271 to = &builtin_functions[i];
276 for (from=charsets; from; from=from->next) {
277 if (strcasecmp(from->name, fromcode) == 0) break;
282 for (to=charsets; to; to=to->next) {
283 if (strcasecmp(to->name, tocode) == 0) break;
287 #ifdef HAVE_NATIVE_ICONV
288 if ((!from || !to) && !native_iconv) {
292 ret->pull = sys_iconv;
293 ret->cd_pull = iconv_open("UTF-16LE", fromcode);
294 if (ret->cd_pull == (iconv_t)-1)
295 ret->cd_pull = iconv_open("UCS-2LE", fromcode);
296 if (ret->cd_pull == (iconv_t)-1) goto failed;
300 ret->push = sys_iconv;
301 ret->cd_push = iconv_open(tocode, "UTF-16LE");
302 if (ret->cd_push == (iconv_t)-1)
303 ret->cd_push = iconv_open(tocode, "UCS-2LE");
304 if (ret->cd_push == (iconv_t)-1) goto failed;
312 /* check for conversion to/from ucs2 */
313 if (is_utf16(fromcode) && to) {
314 ret->direct = to->push;
317 if (is_utf16(tocode) && from) {
318 ret->direct = from->pull;
322 #ifdef HAVE_NATIVE_ICONV
323 if (is_utf16(fromcode)) {
324 ret->direct = sys_iconv;
325 ret->cd_direct = ret->cd_push;
329 if (is_utf16(tocode)) {
330 ret->direct = sys_iconv;
331 ret->cd_direct = ret->cd_pull;
337 /* the general case has to go via a buffer */
338 if (!ret->pull) ret->pull = from->pull;
339 if (!ret->push) ret->push = to->push;
345 return (smb_iconv_t)-1;
349 simple iconv_open() wrapper
351 _PUBLIC_ smb_iconv_t smb_iconv_open(const char *tocode, const char *fromcode)
353 return smb_iconv_open_ex(NULL, tocode, fromcode, true);
357 simple iconv_close() wrapper
359 _PUBLIC_ int smb_iconv_close(smb_iconv_t cd)
366 /**********************************************************************
367 the following functions implement the builtin character sets in Samba
368 and also the "test" character sets that are designed to test
369 multi-byte character set support for english users
370 ***********************************************************************/
371 static size_t ascii_pull(void *cd, const char **inbuf, size_t *inbytesleft,
372 char **outbuf, size_t *outbytesleft)
374 while (*inbytesleft >= 1 && *outbytesleft >= 2) {
375 (*outbuf)[0] = (*inbuf)[0];
378 (*outbytesleft) -= 2;
383 if (*inbytesleft > 0) {
391 static size_t ascii_push(void *cd, const char **inbuf, size_t *inbytesleft,
392 char **outbuf, size_t *outbytesleft)
396 while (*inbytesleft >= 2 && *outbytesleft >= 1) {
397 (*outbuf)[0] = (*inbuf)[0] & 0x7F;
398 if ((*inbuf)[1]) ir_count++;
400 (*outbytesleft) -= 1;
405 if (*inbytesleft == 1) {
410 if (*inbytesleft > 1) {
418 static size_t latin1_push(void *cd, const char **inbuf, size_t *inbytesleft,
419 char **outbuf, size_t *outbytesleft)
423 while (*inbytesleft >= 2 && *outbytesleft >= 1) {
424 (*outbuf)[0] = (*inbuf)[0];
425 if ((*inbuf)[1]) ir_count++;
427 (*outbytesleft) -= 1;
432 if (*inbytesleft == 1) {
437 if (*inbytesleft > 1) {
445 static size_t ucs2hex_pull(void *cd, const char **inbuf, size_t *inbytesleft,
446 char **outbuf, size_t *outbytesleft)
448 while (*inbytesleft >= 1 && *outbytesleft >= 2) {
451 if ((*inbuf)[0] != '@') {
452 /* seven bit ascii case */
453 (*outbuf)[0] = (*inbuf)[0];
456 (*outbytesleft) -= 2;
461 /* it's a hex character */
462 if (*inbytesleft < 5) {
467 if (sscanf(&(*inbuf)[1], "%04x", &v) != 1) {
472 (*outbuf)[0] = v&0xff;
475 (*outbytesleft) -= 2;
480 if (*inbytesleft > 0) {
488 static size_t ucs2hex_push(void *cd, const char **inbuf, size_t *inbytesleft,
489 char **outbuf, size_t *outbytesleft)
491 while (*inbytesleft >= 2 && *outbytesleft >= 1) {
494 if ((*inbuf)[1] == 0 &&
495 ((*inbuf)[0] & 0x80) == 0 &&
496 (*inbuf)[0] != '@') {
497 (*outbuf)[0] = (*inbuf)[0];
499 (*outbytesleft) -= 1;
504 if (*outbytesleft < 5) {
508 snprintf(buf, 6, "@%04x", SVAL(*inbuf, 0));
509 memcpy(*outbuf, buf, 5);
511 (*outbytesleft) -= 5;
516 if (*inbytesleft == 1) {
521 if (*inbytesleft > 1) {
529 static size_t iconv_swab(void *cd, const char **inbuf, size_t *inbytesleft,
530 char **outbuf, size_t *outbytesleft)
534 n = MIN(*inbytesleft, *outbytesleft);
536 swab(*inbuf, *outbuf, (n&~1));
542 (*outbytesleft) -= n;
546 if (*inbytesleft > 0) {
555 static size_t iconv_copy(void *cd, const char **inbuf, size_t *inbytesleft,
556 char **outbuf, size_t *outbytesleft)
560 n = MIN(*inbytesleft, *outbytesleft);
562 memmove(*outbuf, *inbuf, n);
565 (*outbytesleft) -= n;
569 if (*inbytesleft > 0) {
578 this takes a UTF8 sequence and produces a UTF16 sequence
580 static size_t utf8_pull(void *cd, const char **inbuf, size_t *inbytesleft,
581 char **outbuf, size_t *outbytesleft)
583 size_t in_left=*inbytesleft, out_left=*outbytesleft;
584 const uint8_t *c = (const uint8_t *)*inbuf;
585 uint8_t *uc = (uint8_t *)*outbuf;
587 while (in_left >= 1 && out_left >= 2) {
588 if ((c[0] & 0x80) == 0) {
598 if ((c[0] & 0xe0) == 0xc0) {
600 (c[1] & 0xc0) != 0x80) {
604 uc[1] = (c[0]>>2) & 0x7;
605 uc[0] = (c[0]<<6) | (c[1]&0x3f);
613 if ((c[0] & 0xf0) == 0xe0) {
615 (c[1] & 0xc0) != 0x80 ||
616 (c[2] & 0xc0) != 0x80) {
620 uc[1] = ((c[0]&0xF)<<4) | ((c[1]>>2)&0xF);
621 uc[0] = (c[1]<<6) | (c[2]&0x3f);
629 if ((c[0] & 0xf8) == 0xf0) {
630 unsigned int codepoint;
632 (c[1] & 0xc0) != 0x80 ||
633 (c[2] & 0xc0) != 0x80 ||
634 (c[3] & 0xc0) != 0x80) {
643 if (codepoint < 0x10000) {
644 /* accept UTF-8 characters that are not
645 minimally packed, but pack the result */
646 uc[0] = (codepoint & 0xFF);
647 uc[1] = (codepoint >> 8);
655 codepoint -= 0x10000;
662 uc[0] = (codepoint>>10) & 0xFF;
663 uc[1] = (codepoint>>18) | 0xd8;
664 uc[2] = codepoint & 0xFF;
665 uc[3] = ((codepoint>>8) & 0x3) | 0xdc;
673 /* we don't handle 5 byte sequences */
683 *inbytesleft = in_left;
684 *outbytesleft = out_left;
685 *inbuf = (const char *)c;
686 *outbuf = (char *)uc;
690 *inbytesleft = in_left;
691 *outbytesleft = out_left;
692 *inbuf = (const char *)c;
693 *outbuf = (char *)uc;
699 this takes a UTF16 sequence and produces a UTF8 sequence
701 static size_t utf8_push(void *cd, const char **inbuf, size_t *inbytesleft,
702 char **outbuf, size_t *outbytesleft)
704 size_t in_left=*inbytesleft, out_left=*outbytesleft;
705 uint8_t *c = (uint8_t *)*outbuf;
706 const uint8_t *uc = (const uint8_t *)*inbuf;
708 while (in_left >= 2 && out_left >= 1) {
709 unsigned int codepoint;
711 if (uc[1] == 0 && !(uc[0] & 0x80)) {
721 if ((uc[1]&0xf8) == 0) {
722 /* next simplest case */
727 c[0] = 0xc0 | (uc[0]>>6) | (uc[1]<<2);
728 c[1] = 0x80 | (uc[0] & 0x3f);
736 if ((uc[1] & 0xfc) == 0xdc) {
737 /* its the second part of a 4 byte sequence. Illegal */
746 if ((uc[1] & 0xfc) != 0xd8) {
747 codepoint = uc[0] | (uc[1]<<8);
752 c[0] = 0xe0 | (codepoint >> 12);
753 c[1] = 0x80 | ((codepoint >> 6) & 0x3f);
754 c[2] = 0x80 | (codepoint & 0x3f);
763 /* its the first part of a 4 byte sequence */
768 if ((uc[3] & 0xfc) != 0xdc) {
772 codepoint = 0x10000 + (uc[2] | ((uc[3] & 0x3)<<8) |
773 (uc[0]<<10) | ((uc[1] & 0x3)<<18));
779 c[0] = 0xf0 | (codepoint >> 18);
780 c[1] = 0x80 | ((codepoint >> 12) & 0x3f);
781 c[2] = 0x80 | ((codepoint >> 6) & 0x3f);
782 c[3] = 0x80 | (codepoint & 0x3f);
800 *inbytesleft = in_left;
801 *outbytesleft = out_left;
802 *inbuf = (const char *)uc;
808 *inbytesleft = in_left;
809 *outbytesleft = out_left;
810 *inbuf = (const char *)uc;
817 this takes a UTF16 munged sequence, modifies it according to the
818 string2key rules, and produces a UTF16 sequence
822 1) any 0x0000 characters are mapped to 0x0001
824 2) convert any instance of 0xD800 - 0xDBFF (high surrogate)
825 without an immediately following 0xDC00 - 0x0xDFFF (low surrogate) to
826 U+FFFD (OBJECT REPLACEMENT CHARACTER).
828 3) the same for any low surrogate that was not preceded by a high surrogate.
831 static size_t utf16_munged_pull(void *cd, const char **inbuf, size_t *inbytesleft,
832 char **outbuf, size_t *outbytesleft)
834 size_t in_left=*inbytesleft, out_left=*outbytesleft;
835 uint8_t *c = (uint8_t *)*outbuf;
836 const uint8_t *uc = (const uint8_t *)*inbuf;
838 while (in_left >= 2 && out_left >= 2) {
839 unsigned int codepoint = uc[0] | (uc[1]<<8);
841 if (codepoint == 0) {
845 if ((codepoint & 0xfc00) == 0xd800) {
846 /* a high surrogate */
847 unsigned int codepoint2;
852 codepoint2 = uc[2] | (uc[3]<<8);
853 if ((codepoint2 & 0xfc00) != 0xdc00) {
854 /* high surrogate not followed by low
855 surrogate: convert to 0xfffd */
871 if ((codepoint & 0xfc00) == 0xdc00) {
872 /* low surrogate not preceded by high
873 surrogate: convert to 0xfffd */
878 c[0] = codepoint & 0xFF;
879 c[1] = (codepoint>>8) & 0xFF;
898 *inbytesleft = in_left;
899 *outbytesleft = out_left;
900 *inbuf = (const char *)uc;
906 *inbytesleft = in_left;
907 *outbytesleft = out_left;
908 *inbuf = (const char *)uc;