2 Unix SMB/CIFS implementation.
3 minimal iconv implementation
4 Copyright (C) Andrew Tridgell 2001
5 Copyright (C) Jelmer Vernooij 2002
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with this program. If not, see <http://www.gnu.org/licenses/>.
22 #include "../lib/util/dlinklist.h"
23 #include "system/iconv.h"
24 #include "system/filesys.h"
30 #ifdef static_decl_charset
37 * @brief Samba wrapper/stub for iconv character set conversion.
39 * iconv is the XPG2 interface for converting between character
40 * encodings. This file provides a Samba wrapper around it, and also
41 * a simple reimplementation that is used if the system does not
44 * Samba only works with encodings that are supersets of ASCII: ascii
45 * characters like whitespace can be tested for directly, multibyte
46 * sequences start with a byte with the high bit set, and strings are
47 * terminated by a nul byte.
49 * Note that the only function provided by iconv is conversion between
50 * characters. It doesn't directly support operations like
51 * uppercasing or comparison. We have to convert to UTF-16LE and
54 * @sa Samba Developers Guide
57 static size_t ascii_pull (void *,const char **, size_t *, char **, size_t *);
58 static size_t ascii_push (void *,const char **, size_t *, char **, size_t *);
59 static size_t latin1_push(void *,const char **, size_t *, char **, size_t *);
60 static size_t utf8_pull (void *,const char **, size_t *, char **, size_t *);
61 static size_t utf8_push (void *,const char **, size_t *, char **, size_t *);
62 static size_t utf16_munged_pull(void *,const char **, size_t *, char **, size_t *);
63 static size_t ucs2hex_pull(void *,const char **, size_t *, char **, size_t *);
64 static size_t ucs2hex_push(void *,const char **, size_t *, char **, size_t *);
65 static size_t iconv_copy (void *,const char **, size_t *, char **, size_t *);
66 static size_t iconv_swab (void *,const char **, size_t *, char **, size_t *);
68 static const struct charset_functions builtin_functions[] = {
69 /* windows is closest to UTF-16 */
70 {"UCS-2LE", iconv_copy, iconv_copy},
71 {"UTF-16LE", iconv_copy, iconv_copy},
72 {"UCS-2BE", iconv_swab, iconv_swab},
73 {"UTF-16BE", iconv_swab, iconv_swab},
75 /* we include the UTF-8 alias to cope with differing locale settings */
76 {"UTF8", utf8_pull, utf8_push},
77 {"UTF-8", utf8_pull, utf8_push},
79 /* this handles the munging needed for String2Key */
80 {"UTF16_MUNGED", utf16_munged_pull, iconv_copy},
82 {"ASCII", ascii_pull, ascii_push},
83 {"646", ascii_pull, ascii_push},
84 {"ISO-8859-1", ascii_pull, latin1_push},
85 {"UCS2-HEX", ucs2hex_pull, ucs2hex_push}
88 static struct charset_functions *charsets = NULL;
90 static struct charset_functions *find_charset_functions(const char *name)
92 struct charset_functions *c;
94 /* Check whether we already have this charset... */
95 for (c = charsets; c != NULL; c = c->next) {
96 if(strcasecmp(c->name, name) == 0) {
105 bool smb_register_charset(const struct charset_functions *funcs_in)
107 struct charset_functions *funcs;
109 DEBUG(5, ("Attempting to register new charset %s\n", funcs_in->name));
110 /* Check whether we already have this charset... */
111 if (find_charset_functions(funcs_in->name)) {
112 DEBUG(0, ("Duplicate charset %s, not registering\n", funcs_in->name));
116 funcs = talloc(NULL, struct charset_functions);
118 DEBUG(0, ("Out of memory duplicating charset %s\n", funcs_in->name));
123 funcs->next = funcs->prev = NULL;
124 DEBUG(5, ("Registered charset %s\n", funcs->name));
125 DLIST_ADD(charsets, funcs);
129 static void lazy_initialize_iconv(void)
131 #ifdef static_init_charset
132 static bool initialized = false;
141 #ifdef HAVE_NATIVE_ICONV
142 /* if there was an error then reset the internal state,
143 this ensures that we don't have a shift state remaining for
144 character sets like SJIS */
145 static size_t sys_iconv(void *cd,
146 const char **inbuf, size_t *inbytesleft,
147 char **outbuf, size_t *outbytesleft)
149 size_t ret = iconv((iconv_t)cd,
150 discard_const_p(char *, inbuf), inbytesleft,
151 outbuf, outbytesleft);
152 if (ret == (size_t)-1) iconv(cd, NULL, NULL, NULL, NULL);
158 * This is a simple portable iconv() implementaion.
160 * It only knows about a very small number of character sets - just
161 * enough that Samba works on systems that don't have iconv.
163 _PUBLIC_ size_t smb_iconv(smb_iconv_t cd,
164 const char **inbuf, size_t *inbytesleft,
165 char **outbuf, size_t *outbytesleft)
167 /* in many cases we can go direct */
169 return cd->direct(cd->cd_direct,
170 inbuf, inbytesleft, outbuf, outbytesleft);
173 /* otherwise we have to do it chunks at a time */
175 #ifndef SMB_ICONV_BUFSIZE
176 #define SMB_ICONV_BUFSIZE 2048
179 char *cvtbuf = talloc_array(cd, char, SMB_ICONV_BUFSIZE);
185 while (*inbytesleft > 0) {
186 char *bufp1 = cvtbuf;
187 const char *bufp2 = cvtbuf;
189 bufsize = SMB_ICONV_BUFSIZE;
191 if (cd->pull(cd->cd_pull,
192 inbuf, inbytesleft, &bufp1, &bufsize) == -1
198 bufsize = SMB_ICONV_BUFSIZE - bufsize;
200 if (cd->push(cd->cd_push,
202 outbuf, outbytesleft) == -1) {
213 static bool is_utf16(const char *name)
215 return strcasecmp(name, "UCS-2LE") == 0 ||
216 strcasecmp(name, "UTF-16LE") == 0;
219 static int smb_iconv_t_destructor(smb_iconv_t hwd)
221 #ifdef HAVE_NATIVE_ICONV
222 if (hwd->cd_pull != NULL && hwd->cd_pull != (iconv_t)-1)
223 iconv_close(hwd->cd_pull);
224 if (hwd->cd_push != NULL && hwd->cd_push != (iconv_t)-1)
225 iconv_close(hwd->cd_push);
226 if (hwd->cd_direct != NULL && hwd->cd_direct != (iconv_t)-1)
227 iconv_close(hwd->cd_direct);
233 _PUBLIC_ smb_iconv_t smb_iconv_open_ex(TALLOC_CTX *mem_ctx, const char *tocode,
234 const char *fromcode, bool native_iconv)
237 const struct charset_functions *from=NULL, *to=NULL;
240 lazy_initialize_iconv();
242 ret = (smb_iconv_t)talloc_named(mem_ctx,
244 "iconv(%s,%s)", tocode, fromcode);
247 return (smb_iconv_t)-1;
249 memset(ret, 0, sizeof(*ret));
250 talloc_set_destructor(ret, smb_iconv_t_destructor);
252 /* check for the simplest null conversion */
253 if (strcmp(fromcode, tocode) == 0) {
254 ret->direct = iconv_copy;
258 for (i=0;i<ARRAY_SIZE(builtin_functions);i++) {
259 if (strcasecmp(fromcode, builtin_functions[i].name) == 0) {
260 from = &builtin_functions[i];
262 if (strcasecmp(tocode, builtin_functions[i].name) == 0) {
263 to = &builtin_functions[i];
268 for (from=charsets; from; from=from->next) {
269 if (strcasecmp(from->name, fromcode) == 0) break;
274 for (to=charsets; to; to=to->next) {
275 if (strcasecmp(to->name, tocode) == 0) break;
279 #ifdef HAVE_NATIVE_ICONV
280 if ((!from || !to) && !native_iconv) {
284 ret->pull = sys_iconv;
285 ret->cd_pull = iconv_open("UTF-16LE", fromcode);
286 if (ret->cd_pull == (iconv_t)-1)
287 ret->cd_pull = iconv_open("UCS-2LE", fromcode);
288 if (ret->cd_pull == (iconv_t)-1) goto failed;
292 ret->push = sys_iconv;
293 ret->cd_push = iconv_open(tocode, "UTF-16LE");
294 if (ret->cd_push == (iconv_t)-1)
295 ret->cd_push = iconv_open(tocode, "UCS-2LE");
296 if (ret->cd_push == (iconv_t)-1) goto failed;
304 /* check for conversion to/from ucs2 */
305 if (is_utf16(fromcode) && to) {
306 ret->direct = to->push;
309 if (is_utf16(tocode) && from) {
310 ret->direct = from->pull;
314 #ifdef HAVE_NATIVE_ICONV
315 if (is_utf16(fromcode)) {
316 ret->direct = sys_iconv;
317 ret->cd_direct = ret->cd_push;
321 if (is_utf16(tocode)) {
322 ret->direct = sys_iconv;
323 ret->cd_direct = ret->cd_pull;
329 /* the general case has to go via a buffer */
330 if (!ret->pull) ret->pull = from->pull;
331 if (!ret->push) ret->push = to->push;
337 return (smb_iconv_t)-1;
341 simple iconv_open() wrapper
343 _PUBLIC_ smb_iconv_t smb_iconv_open(const char *tocode, const char *fromcode)
345 return smb_iconv_open_ex(NULL, tocode, fromcode, true);
349 simple iconv_close() wrapper
351 _PUBLIC_ int smb_iconv_close(smb_iconv_t cd)
358 /**********************************************************************
359 the following functions implement the builtin character sets in Samba
360 and also the "test" character sets that are designed to test
361 multi-byte character set support for english users
362 ***********************************************************************/
363 static size_t ascii_pull(void *cd, const char **inbuf, size_t *inbytesleft,
364 char **outbuf, size_t *outbytesleft)
366 while (*inbytesleft >= 1 && *outbytesleft >= 2) {
367 (*outbuf)[0] = (*inbuf)[0];
370 (*outbytesleft) -= 2;
375 if (*inbytesleft > 0) {
383 static size_t ascii_push(void *cd, const char **inbuf, size_t *inbytesleft,
384 char **outbuf, size_t *outbytesleft)
388 while (*inbytesleft >= 2 && *outbytesleft >= 1) {
389 (*outbuf)[0] = (*inbuf)[0] & 0x7F;
390 if ((*inbuf)[1]) ir_count++;
392 (*outbytesleft) -= 1;
397 if (*inbytesleft == 1) {
402 if (*inbytesleft > 1) {
410 static size_t latin1_push(void *cd, const char **inbuf, size_t *inbytesleft,
411 char **outbuf, size_t *outbytesleft)
415 while (*inbytesleft >= 2 && *outbytesleft >= 1) {
416 (*outbuf)[0] = (*inbuf)[0];
417 if ((*inbuf)[1]) ir_count++;
419 (*outbytesleft) -= 1;
424 if (*inbytesleft == 1) {
429 if (*inbytesleft > 1) {
437 static size_t ucs2hex_pull(void *cd, const char **inbuf, size_t *inbytesleft,
438 char **outbuf, size_t *outbytesleft)
440 while (*inbytesleft >= 1 && *outbytesleft >= 2) {
443 if ((*inbuf)[0] != '@') {
444 /* seven bit ascii case */
445 (*outbuf)[0] = (*inbuf)[0];
448 (*outbytesleft) -= 2;
453 /* it's a hex character */
454 if (*inbytesleft < 5) {
459 if (sscanf(&(*inbuf)[1], "%04x", &v) != 1) {
464 (*outbuf)[0] = v&0xff;
467 (*outbytesleft) -= 2;
472 if (*inbytesleft > 0) {
480 static size_t ucs2hex_push(void *cd, const char **inbuf, size_t *inbytesleft,
481 char **outbuf, size_t *outbytesleft)
483 while (*inbytesleft >= 2 && *outbytesleft >= 1) {
486 if ((*inbuf)[1] == 0 &&
487 ((*inbuf)[0] & 0x80) == 0 &&
488 (*inbuf)[0] != '@') {
489 (*outbuf)[0] = (*inbuf)[0];
491 (*outbytesleft) -= 1;
496 if (*outbytesleft < 5) {
500 snprintf(buf, 6, "@%04x", SVAL(*inbuf, 0));
501 memcpy(*outbuf, buf, 5);
503 (*outbytesleft) -= 5;
508 if (*inbytesleft == 1) {
513 if (*inbytesleft > 1) {
521 static size_t iconv_swab(void *cd, const char **inbuf, size_t *inbytesleft,
522 char **outbuf, size_t *outbytesleft)
526 n = MIN(*inbytesleft, *outbytesleft);
528 swab(*inbuf, *outbuf, (n&~1));
534 (*outbytesleft) -= n;
538 if (*inbytesleft > 0) {
547 static size_t iconv_copy(void *cd, const char **inbuf, size_t *inbytesleft,
548 char **outbuf, size_t *outbytesleft)
552 n = MIN(*inbytesleft, *outbytesleft);
554 memmove(*outbuf, *inbuf, n);
557 (*outbytesleft) -= n;
561 if (*inbytesleft > 0) {
570 this takes a UTF8 sequence and produces a UTF16 sequence
572 static size_t utf8_pull(void *cd, const char **inbuf, size_t *inbytesleft,
573 char **outbuf, size_t *outbytesleft)
575 size_t in_left=*inbytesleft, out_left=*outbytesleft;
576 const uint8_t *c = (const uint8_t *)*inbuf;
577 uint8_t *uc = (uint8_t *)*outbuf;
579 while (in_left >= 1 && out_left >= 2) {
580 if ((c[0] & 0x80) == 0) {
590 if ((c[0] & 0xe0) == 0xc0) {
592 (c[1] & 0xc0) != 0x80) {
596 uc[1] = (c[0]>>2) & 0x7;
597 uc[0] = (c[0]<<6) | (c[1]&0x3f);
605 if ((c[0] & 0xf0) == 0xe0) {
607 (c[1] & 0xc0) != 0x80 ||
608 (c[2] & 0xc0) != 0x80) {
612 uc[1] = ((c[0]&0xF)<<4) | ((c[1]>>2)&0xF);
613 uc[0] = (c[1]<<6) | (c[2]&0x3f);
621 if ((c[0] & 0xf8) == 0xf0) {
622 unsigned int codepoint;
624 (c[1] & 0xc0) != 0x80 ||
625 (c[2] & 0xc0) != 0x80 ||
626 (c[3] & 0xc0) != 0x80) {
635 if (codepoint < 0x10000) {
636 /* accept UTF-8 characters that are not
637 minimally packed, but pack the result */
638 uc[0] = (codepoint & 0xFF);
639 uc[1] = (codepoint >> 8);
647 codepoint -= 0x10000;
654 uc[0] = (codepoint>>10) & 0xFF;
655 uc[1] = (codepoint>>18) | 0xd8;
656 uc[2] = codepoint & 0xFF;
657 uc[3] = ((codepoint>>8) & 0x3) | 0xdc;
665 /* we don't handle 5 byte sequences */
675 *inbytesleft = in_left;
676 *outbytesleft = out_left;
677 *inbuf = (const char *)c;
678 *outbuf = (char *)uc;
682 *inbytesleft = in_left;
683 *outbytesleft = out_left;
684 *inbuf = (const char *)c;
685 *outbuf = (char *)uc;
691 this takes a UTF16 sequence and produces a UTF8 sequence
693 static size_t utf8_push(void *cd, const char **inbuf, size_t *inbytesleft,
694 char **outbuf, size_t *outbytesleft)
696 size_t in_left=*inbytesleft, out_left=*outbytesleft;
697 uint8_t *c = (uint8_t *)*outbuf;
698 const uint8_t *uc = (const uint8_t *)*inbuf;
700 while (in_left >= 2 && out_left >= 1) {
701 unsigned int codepoint;
703 if (uc[1] == 0 && !(uc[0] & 0x80)) {
713 if ((uc[1]&0xf8) == 0) {
714 /* next simplest case */
719 c[0] = 0xc0 | (uc[0]>>6) | (uc[1]<<2);
720 c[1] = 0x80 | (uc[0] & 0x3f);
728 if ((uc[1] & 0xfc) == 0xdc) {
729 /* its the second part of a 4 byte sequence. Illegal */
738 if ((uc[1] & 0xfc) != 0xd8) {
739 codepoint = uc[0] | (uc[1]<<8);
744 c[0] = 0xe0 | (codepoint >> 12);
745 c[1] = 0x80 | ((codepoint >> 6) & 0x3f);
746 c[2] = 0x80 | (codepoint & 0x3f);
755 /* its the first part of a 4 byte sequence */
760 if ((uc[3] & 0xfc) != 0xdc) {
764 codepoint = 0x10000 + (uc[2] | ((uc[3] & 0x3)<<8) |
765 (uc[0]<<10) | ((uc[1] & 0x3)<<18));
771 c[0] = 0xf0 | (codepoint >> 18);
772 c[1] = 0x80 | ((codepoint >> 12) & 0x3f);
773 c[2] = 0x80 | ((codepoint >> 6) & 0x3f);
774 c[3] = 0x80 | (codepoint & 0x3f);
792 *inbytesleft = in_left;
793 *outbytesleft = out_left;
794 *inbuf = (const char *)uc;
800 *inbytesleft = in_left;
801 *outbytesleft = out_left;
802 *inbuf = (const char *)uc;
809 this takes a UTF16 munged sequence, modifies it according to the
810 string2key rules, and produces a UTF16 sequence
814 1) any 0x0000 characters are mapped to 0x0001
816 2) convert any instance of 0xD800 - 0xDBFF (high surrogate)
817 without an immediately following 0xDC00 - 0x0xDFFF (low surrogate) to
818 U+FFFD (OBJECT REPLACEMENT CHARACTER).
820 3) the same for any low surrogate that was not preceded by a high surrogate.
823 static size_t utf16_munged_pull(void *cd, const char **inbuf, size_t *inbytesleft,
824 char **outbuf, size_t *outbytesleft)
826 size_t in_left=*inbytesleft, out_left=*outbytesleft;
827 uint8_t *c = (uint8_t *)*outbuf;
828 const uint8_t *uc = (const uint8_t *)*inbuf;
830 while (in_left >= 2 && out_left >= 2) {
831 unsigned int codepoint = uc[0] | (uc[1]<<8);
833 if (codepoint == 0) {
837 if ((codepoint & 0xfc00) == 0xd800) {
838 /* a high surrogate */
839 unsigned int codepoint2;
844 codepoint2 = uc[2] | (uc[3]<<8);
845 if ((codepoint2 & 0xfc00) != 0xdc00) {
846 /* high surrogate not followed by low
847 surrogate: convert to 0xfffd */
863 if ((codepoint & 0xfc00) == 0xdc00) {
864 /* low surrogate not preceded by high
865 surrogate: convert to 0xfffd */
870 c[0] = codepoint & 0xFF;
871 c[1] = (codepoint>>8) & 0xFF;
890 *inbytesleft = in_left;
891 *outbytesleft = out_left;
892 *inbuf = (const char *)uc;
898 *inbytesleft = in_left;
899 *outbytesleft = out_left;
900 *inbuf = (const char *)uc;