VirtualBox

source: vbox/trunk/src/VBox/Runtime/common/string/utf-8.cpp@ 31213

最後變更 在這個檔案從31213是 31213,由 vboxsync 提交於 15 年 前

Runtime/string: Utf-8 to Latin-1 translation should be exact or fail
  • 屬性 svn:eol-style 設為 native
  • 屬性 svn:keywords 設為 Id
檔案大小: 53.7 KB
行 
1/* $Id: utf-8.cpp 31213 2010-07-29 13:52:23Z vboxsync $ */
2/** @file
3 * IPRT - UTF-8 Decoding.
4 */
5
6/*
7 * Copyright (C) 2006-2010 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.alldomusa.eu.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*******************************************************************************
29* Header Files *
30*******************************************************************************/
31#include <iprt/string.h>
32#include "internal/iprt.h"
33
34#include <iprt/uni.h>
35#include <iprt/alloc.h>
36#include <iprt/assert.h>
37#include <iprt/err.h>
38#include "internal/string.h"
39
40
41
42/**
43 * Get get length in code points of a UTF-8 encoded string.
44 * The string is validated while doing this.
45 *
46 * @returns IPRT status code.
47 * @param psz Pointer to the UTF-8 string.
48 * @param cch The max length of the string. (btw cch = cb)
49 * Use RTSTR_MAX if all of the string is to be examined.
50 * @param pcuc Where to store the length in unicode code points.
51 * @param pcchActual Where to store the actual size of the UTF-8 string
52 * on success (cch = cb again). Optional.
53 */
54int rtUtf8Length(const char *psz, size_t cch, size_t *pcuc, size_t *pcchActual)
55{
56 const unsigned char *puch = (const unsigned char *)psz;
57 size_t cCodePoints = 0;
58 while (cch > 0)
59 {
60 const unsigned char uch = *puch;
61 if (!uch)
62 break;
63 if (uch & RT_BIT(7))
64 {
65 /* figure sequence length and validate the first byte */
66 unsigned cb;
67 if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
68 cb = 2;
69 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
70 cb = 3;
71 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)))
72 cb = 4;
73 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3)))
74 cb = 5;
75 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2) | RT_BIT(1))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2)))
76 cb = 6;
77 else
78 {
79 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
80 return VERR_INVALID_UTF8_ENCODING;
81 }
82
83 /* check length */
84 if (cb > cch)
85 {
86 RTStrAssertMsgFailed(("Invalid UTF-8 length: cb=%d cch=%d (%.*Rhxs)\n", cb, cch, RT_MIN(cch, 10), puch));
87 return VERR_INVALID_UTF8_ENCODING;
88 }
89
90 /* validate the rest */
91 switch (cb)
92 {
93 case 6:
94 RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
95 case 5:
96 RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
97 case 4:
98 RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
99 case 3:
100 RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
101 case 2:
102 RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
103 break;
104 }
105
106 /* validate the code point. */
107 RTUNICP uc;
108 switch (cb)
109 {
110 case 6:
111 uc = (puch[5] & 0x3f)
112 | ((RTUNICP)(puch[4] & 0x3f) << 6)
113 | ((RTUNICP)(puch[3] & 0x3f) << 12)
114 | ((RTUNICP)(puch[2] & 0x3f) << 18)
115 | ((RTUNICP)(puch[1] & 0x3f) << 24)
116 | ((RTUNICP)(uch & 0x01) << 30);
117 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
118 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
119 break;
120 case 5:
121 uc = (puch[4] & 0x3f)
122 | ((RTUNICP)(puch[3] & 0x3f) << 6)
123 | ((RTUNICP)(puch[2] & 0x3f) << 12)
124 | ((RTUNICP)(puch[1] & 0x3f) << 18)
125 | ((RTUNICP)(uch & 0x03) << 24);
126 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
127 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
128 break;
129 case 4:
130 uc = (puch[3] & 0x3f)
131 | ((RTUNICP)(puch[2] & 0x3f) << 6)
132 | ((RTUNICP)(puch[1] & 0x3f) << 12)
133 | ((RTUNICP)(uch & 0x07) << 18);
134 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
135 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
136 break;
137 case 3:
138 uc = (puch[2] & 0x3f)
139 | ((RTUNICP)(puch[1] & 0x3f) << 6)
140 | ((RTUNICP)(uch & 0x0f) << 12);
141 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
142 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
143 uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
144 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
145 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
146 break;
147 case 2:
148 uc = (puch[1] & 0x3f)
149 | ((RTUNICP)(uch & 0x1f) << 6);
150 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
151 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
152 break;
153 }
154
155 /* advance */
156 cch -= cb;
157 puch += cb;
158 }
159 else
160 {
161 /* one ASCII byte */
162 puch++;
163 cch--;
164 }
165 cCodePoints++;
166 }
167
168 /* done */
169 *pcuc = cCodePoints;
170 if (pcchActual)
171 *pcchActual = puch - (unsigned char const *)psz;
172 return VINF_SUCCESS;
173}
174
175
176/**
177 * Decodes and UTF-8 string into an array of unicode code point.
178 *
179 * Since we know the input is valid, we do *not* perform encoding or length checks.
180 *
181 * @returns iprt status code.
182 * @param psz The UTF-8 string to recode. This is a valid encoding.
183 * @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
184 * The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
185 * @param paCps Where to store the code points array.
186 * @param cCps The number of RTUNICP items the paCps buffer can hold, excluding the terminator ('\\0').
187 */
188static int rtUtf8Decode(const char *psz, size_t cch, PRTUNICP paCps, size_t cCps)
189{
190 int rc = VINF_SUCCESS;
191 const unsigned char *puch = (const unsigned char *)psz;
192 PRTUNICP pCp = paCps;
193 while (cch > 0)
194 {
195 /* read the next char and check for terminator. */
196 const unsigned char uch = *puch;
197 if (!uch)
198 break;
199
200 /* check for output overflow */
201 if (RT_UNLIKELY(cCps < 1))
202 {
203 rc = VERR_BUFFER_OVERFLOW;
204 break;
205 }
206 cCps--;
207
208 /* decode and recode the code point */
209 if (!(uch & RT_BIT(7)))
210 {
211 *pCp++ = uch;
212 puch++;
213 cch--;
214 }
215#ifdef RT_STRICT
216 else if (!(uch & RT_BIT(6)))
217 AssertMsgFailed(("Internal error!\n"));
218#endif
219 else if (!(uch & RT_BIT(5)))
220 {
221 *pCp++ = (puch[1] & 0x3f)
222 | ((uint16_t)(uch & 0x1f) << 6);
223 puch += 2;
224 cch -= 2;
225 }
226 else if (!(uch & RT_BIT(4)))
227 {
228 *pCp++ = (puch[2] & 0x3f)
229 | ((uint16_t)(puch[1] & 0x3f) << 6)
230 | ((uint16_t)(uch & 0x0f) << 12);
231 puch += 3;
232 cch -= 3;
233 }
234 else if (!(uch & RT_BIT(3)))
235 {
236 *pCp++ = (puch[3] & 0x3f)
237 | ((RTUNICP)(puch[2] & 0x3f) << 6)
238 | ((RTUNICP)(puch[1] & 0x3f) << 12)
239 | ((RTUNICP)(uch & 0x07) << 18);
240 puch += 4;
241 cch -= 4;
242 }
243 else if (!(uch & RT_BIT(2)))
244 {
245 *pCp++ = (puch[4] & 0x3f)
246 | ((RTUNICP)(puch[3] & 0x3f) << 6)
247 | ((RTUNICP)(puch[2] & 0x3f) << 12)
248 | ((RTUNICP)(puch[1] & 0x3f) << 18)
249 | ((RTUNICP)(uch & 0x03) << 24);
250 puch += 5;
251 cch -= 6;
252 }
253 else
254 {
255 Assert(!(uch & RT_BIT(1)));
256 *pCp++ = (puch[5] & 0x3f)
257 | ((RTUNICP)(puch[4] & 0x3f) << 6)
258 | ((RTUNICP)(puch[3] & 0x3f) << 12)
259 | ((RTUNICP)(puch[2] & 0x3f) << 18)
260 | ((RTUNICP)(puch[1] & 0x3f) << 24)
261 | ((RTUNICP)(uch & 0x01) << 30);
262 puch += 6;
263 cch -= 6;
264 }
265 }
266
267 /* done */
268 *pCp = 0;
269 return rc;
270}
271
272
273RTDECL(size_t) RTStrUniLen(const char *psz)
274{
275 size_t cCodePoints;
276 int rc = rtUtf8Length(psz, RTSTR_MAX, &cCodePoints, NULL);
277 return RT_SUCCESS(rc) ? cCodePoints : 0;
278}
279RT_EXPORT_SYMBOL(RTStrUniLen);
280
281
282RTDECL(int) RTStrUniLenEx(const char *psz, size_t cch, size_t *pcCps)
283{
284 size_t cCodePoints;
285 int rc = rtUtf8Length(psz, cch, &cCodePoints, NULL);
286 if (pcCps)
287 *pcCps = RT_SUCCESS(rc) ? cCodePoints : 0;
288 return rc;
289}
290RT_EXPORT_SYMBOL(RTStrUniLenEx);
291
292
293RTDECL(int) RTStrValidateEncoding(const char *psz)
294{
295 return RTStrValidateEncodingEx(psz, RTSTR_MAX, 0);
296}
297RT_EXPORT_SYMBOL(RTStrValidateEncoding);
298
299
300RTDECL(int) RTStrValidateEncodingEx(const char *psz, size_t cch, uint32_t fFlags)
301{
302 AssertReturn(!(fFlags & ~(RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED)), VERR_INVALID_PARAMETER);
303 AssertPtr(psz);
304
305 /*
306 * Use rtUtf8Length for the job.
307 */
308 size_t cchActual;
309 size_t cCpsIgnored;
310 int rc = rtUtf8Length(psz, cch, &cCpsIgnored, &cchActual);
311 if (RT_SUCCESS(rc))
312 {
313 if ( (fFlags & RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED)
314 && cchActual >= cch)
315 rc = VERR_BUFFER_OVERFLOW;
316 }
317 return rc;
318}
319RT_EXPORT_SYMBOL(RTStrValidateEncodingEx);
320
321
322RTDECL(bool) RTStrIsValidEncoding(const char *psz)
323{
324 int rc = RTStrValidateEncodingEx(psz, RTSTR_MAX, 0);
325 return RT_SUCCESS(rc);
326}
327RT_EXPORT_SYMBOL(RTStrIsValidEncoding);
328
329
330RTDECL(size_t) RTStrPurgeEncoding(char *psz)
331{
332 size_t cErrors = 0;
333 for (;;)
334 {
335 RTUNICP Cp;
336 int rc = RTStrGetCpEx((const char **)&psz, &Cp);
337 if (RT_SUCCESS(rc))
338 {
339 if (!Cp)
340 break;
341 }
342 else
343 {
344 psz[-1] = '?';
345 cErrors++;
346 }
347 }
348 return cErrors;
349}
350RT_EXPORT_SYMBOL(RTStrPurgeEncoding);
351
352
353RTDECL(int) RTStrToUni(const char *pszString, PRTUNICP *ppaCps)
354{
355 /*
356 * Validate input.
357 */
358 Assert(VALID_PTR(pszString));
359 Assert(VALID_PTR(ppaCps));
360 *ppaCps = NULL;
361
362 /*
363 * Validate the UTF-8 input and count its code points.
364 */
365 size_t cCps;
366 int rc = rtUtf8Length(pszString, RTSTR_MAX, &cCps, NULL);
367 if (RT_SUCCESS(rc))
368 {
369 /*
370 * Allocate buffer.
371 */
372 PRTUNICP paCps = (PRTUNICP)RTMemAlloc((cCps + 1) * sizeof(RTUNICP));
373 if (paCps)
374 {
375 /*
376 * Decode the string.
377 */
378 rc = rtUtf8Decode(pszString, RTSTR_MAX, paCps, cCps);
379 if (RT_SUCCESS(rc))
380 {
381 *ppaCps = paCps;
382 return rc;
383 }
384 RTMemFree(paCps);
385 }
386 else
387 rc = VERR_NO_CODE_POINT_MEMORY;
388 }
389 return rc;
390}
391RT_EXPORT_SYMBOL(RTStrToUni);
392
393
394RTDECL(int) RTStrToUniEx(const char *pszString, size_t cchString, PRTUNICP *ppaCps, size_t cCps, size_t *pcCps)
395{
396 /*
397 * Validate input.
398 */
399 Assert(VALID_PTR(pszString));
400 Assert(VALID_PTR(ppaCps));
401 Assert(!pcCps || VALID_PTR(pcCps));
402
403 /*
404 * Validate the UTF-8 input and count the code points.
405 */
406 size_t cCpsResult;
407 int rc = rtUtf8Length(pszString, cchString, &cCpsResult, NULL);
408 if (RT_SUCCESS(rc))
409 {
410 if (pcCps)
411 *pcCps = cCpsResult;
412
413 /*
414 * Check buffer size / Allocate buffer.
415 */
416 bool fShouldFree;
417 PRTUNICP paCpsResult;
418 if (cCps > 0 && *ppaCps)
419 {
420 fShouldFree = false;
421 if (cCps <= cCpsResult)
422 return VERR_BUFFER_OVERFLOW;
423 paCpsResult = *ppaCps;
424 }
425 else
426 {
427 *ppaCps = NULL;
428 fShouldFree = true;
429 cCps = RT_MAX(cCpsResult + 1, cCps);
430 paCpsResult = (PRTUNICP)RTMemAlloc(cCps * sizeof(RTUNICP));
431 }
432 if (paCpsResult)
433 {
434 /*
435 * Encode the UTF-16 string.
436 */
437 rc = rtUtf8Decode(pszString, cchString, paCpsResult, cCps - 1);
438 if (RT_SUCCESS(rc))
439 {
440 *ppaCps = paCpsResult;
441 return rc;
442 }
443 if (fShouldFree)
444 RTMemFree(paCpsResult);
445 }
446 else
447 rc = VERR_NO_CODE_POINT_MEMORY;
448 }
449 return rc;
450}
451RT_EXPORT_SYMBOL(RTStrToUniEx);
452
453
454/**
455 * Calculates the UTF-16 length of a string, validating the encoding while doing so.
456 *
457 * @returns IPRT status code.
458 * @param psz Pointer to the UTF-8 string.
459 * @param cch The max length of the string. (btw cch = cb)
460 * Use RTSTR_MAX if all of the string is to be examined.
461 * @param pcwc Where to store the length of the UTF-16 string as a number of RTUTF16 characters.
462 */
463static int rtUtf8CalcUtf16Length(const char *psz, size_t cch, size_t *pcwc)
464{
465 const unsigned char *puch = (const unsigned char *)psz;
466 size_t cwc = 0;
467 while (cch > 0)
468 {
469 const unsigned char uch = *puch;
470 if (!uch)
471 break;
472 if (!(uch & RT_BIT(7)))
473 {
474 /* one ASCII byte */
475 cwc++;
476 puch++;
477 cch--;
478 }
479 else
480 {
481 /* figure sequence length and validate the first byte */
482 unsigned cb;
483 if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
484 cb = 2;
485 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
486 cb = 3;
487 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)))
488 cb = 4;
489 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3)))
490 cb = 5;
491 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2) | RT_BIT(1))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2)))
492 cb = 6;
493 else
494 {
495 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
496 return VERR_INVALID_UTF8_ENCODING;
497 }
498
499 /* check length */
500 if (cb > cch)
501 {
502 RTStrAssertMsgFailed(("Invalid UTF-8 length: cb=%d cch=%d (%.*Rhxs)\n", cb, cch, RT_MIN(cch, 10), puch));
503 return VERR_INVALID_UTF8_ENCODING;
504 }
505
506 /* validate the rest */
507 switch (cb)
508 {
509 case 6:
510 RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
511 case 5:
512 RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
513 case 4:
514 RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
515 case 3:
516 RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
517 case 2:
518 RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
519 break;
520 }
521
522 /* validate the code point. */
523 RTUNICP uc;
524 switch (cb)
525 {
526 case 6:
527 uc = (puch[5] & 0x3f)
528 | ((RTUNICP)(puch[4] & 0x3f) << 6)
529 | ((RTUNICP)(puch[3] & 0x3f) << 12)
530 | ((RTUNICP)(puch[2] & 0x3f) << 18)
531 | ((RTUNICP)(puch[1] & 0x3f) << 24)
532 | ((RTUNICP)(uch & 0x01) << 30);
533 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
534 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
535 RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
536 return VERR_CANT_RECODE_AS_UTF16;
537 case 5:
538 uc = (puch[4] & 0x3f)
539 | ((RTUNICP)(puch[3] & 0x3f) << 6)
540 | ((RTUNICP)(puch[2] & 0x3f) << 12)
541 | ((RTUNICP)(puch[1] & 0x3f) << 18)
542 | ((RTUNICP)(uch & 0x03) << 24);
543 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
544 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
545 RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
546 return VERR_CANT_RECODE_AS_UTF16;
547 case 4:
548 uc = (puch[3] & 0x3f)
549 | ((RTUNICP)(puch[2] & 0x3f) << 6)
550 | ((RTUNICP)(puch[1] & 0x3f) << 12)
551 | ((RTUNICP)(uch & 0x07) << 18);
552 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
553 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
554 RTStrAssertMsgReturn(uc <= 0x0010ffff,
555 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CANT_RECODE_AS_UTF16);
556 cwc++;
557 break;
558 case 3:
559 uc = (puch[2] & 0x3f)
560 | ((RTUNICP)(puch[1] & 0x3f) << 6)
561 | ((RTUNICP)(uch & 0x0f) << 12);
562 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
563 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
564 uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
565 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
566 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
567 break;
568 case 2:
569 uc = (puch[1] & 0x3f)
570 | ((RTUNICP)(uch & 0x1f) << 6);
571 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
572 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
573 break;
574 }
575
576 /* advance */
577 cch -= cb;
578 puch += cb;
579 cwc++;
580 }
581 }
582
583 /* done */
584 *pcwc = cwc;
585 return VINF_SUCCESS;
586}
587
588
589/**
590 * Recodes a valid UTF-8 string as UTF-16.
591 *
592 * Since we know the input is valid, we do *not* perform encoding or length checks.
593 *
594 * @returns iprt status code.
595 * @param psz The UTF-8 string to recode. This is a valid encoding.
596 * @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
597 * The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
598 * @param pwsz Where to store the UTF-16 string.
599 * @param cwc The number of RTUTF16 items the pwsz buffer can hold, excluding the terminator ('\\0').
600 */
601static int rtUtf8RecodeAsUtf16(const char *psz, size_t cch, PRTUTF16 pwsz, size_t cwc)
602{
603 int rc = VINF_SUCCESS;
604 const unsigned char *puch = (const unsigned char *)psz;
605 PRTUTF16 pwc = pwsz;
606 while (cch > 0)
607 {
608 /* read the next char and check for terminator. */
609 const unsigned char uch = *puch;
610 if (!uch)
611 break;
612
613 /* check for output overflow */
614 if (RT_UNLIKELY(cwc < 1))
615 {
616 rc = VERR_BUFFER_OVERFLOW;
617 break;
618 }
619 cwc--;
620
621 /* decode and recode the code point */
622 if (!(uch & RT_BIT(7)))
623 {
624 *pwc++ = uch;
625 puch++;
626 cch--;
627 }
628 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
629 {
630 uint16_t uc = (puch[1] & 0x3f)
631 | ((uint16_t)(uch & 0x1f) << 6);
632 *pwc++ = uc;
633 puch += 2;
634 cch -= 2;
635 }
636 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
637 {
638 uint16_t uc = (puch[2] & 0x3f)
639 | ((uint16_t)(puch[1] & 0x3f) << 6)
640 | ((uint16_t)(uch & 0x0f) << 12);
641 *pwc++ = uc;
642 puch += 3;
643 cch -= 3;
644 }
645 else
646 {
647 /* generate surrugate pair */
648 Assert((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)));
649 RTUNICP uc = (puch[3] & 0x3f)
650 | ((RTUNICP)(puch[2] & 0x3f) << 6)
651 | ((RTUNICP)(puch[1] & 0x3f) << 12)
652 | ((RTUNICP)(uch & 0x07) << 18);
653 if (RT_UNLIKELY(cwc < 1))
654 {
655 rc = VERR_BUFFER_OVERFLOW;
656 break;
657 }
658 cwc--;
659
660 uc -= 0x10000;
661 *pwc++ = 0xd800 | (uc >> 10);
662 *pwc++ = 0xdc00 | (uc & 0x3ff);
663 puch += 4;
664 cch -= 4;
665 }
666 }
667
668 /* done */
669 *pwc = '\0';
670 return rc;
671}
672
673
674RTDECL(int) RTStrToUtf16Tag(const char *pszString, PRTUTF16 *ppwszString, const char *pszTag)
675{
676 /*
677 * Validate input.
678 */
679 Assert(VALID_PTR(ppwszString));
680 Assert(VALID_PTR(pszString));
681 *ppwszString = NULL;
682
683 /*
684 * Validate the UTF-8 input and calculate the length of the UTF-16 string.
685 */
686 size_t cwc;
687 int rc = rtUtf8CalcUtf16Length(pszString, RTSTR_MAX, &cwc);
688 if (RT_SUCCESS(rc))
689 {
690 /*
691 * Allocate buffer.
692 */
693 PRTUTF16 pwsz = (PRTUTF16)RTMemAllocTag((cwc + 1) * sizeof(RTUTF16), pszTag);
694 if (pwsz)
695 {
696 /*
697 * Encode the UTF-16 string.
698 */
699 rc = rtUtf8RecodeAsUtf16(pszString, RTSTR_MAX, pwsz, cwc);
700 if (RT_SUCCESS(rc))
701 {
702 *ppwszString = pwsz;
703 return rc;
704 }
705 RTMemFree(pwsz);
706 }
707 else
708 rc = VERR_NO_UTF16_MEMORY;
709 }
710 return rc;
711}
712RT_EXPORT_SYMBOL(RTStrToUtf16Tag);
713
714
715RTDECL(int) RTStrToUtf16ExTag(const char *pszString, size_t cchString,
716 PRTUTF16 *ppwsz, size_t cwc, size_t *pcwc, const char *pszTag)
717{
718 /*
719 * Validate input.
720 */
721 Assert(VALID_PTR(pszString));
722 Assert(VALID_PTR(ppwsz));
723 Assert(!pcwc || VALID_PTR(pcwc));
724
725 /*
726 * Validate the UTF-8 input and calculate the length of the UTF-16 string.
727 */
728 size_t cwcResult;
729 int rc = rtUtf8CalcUtf16Length(pszString, cchString, &cwcResult);
730 if (RT_SUCCESS(rc))
731 {
732 if (pcwc)
733 *pcwc = cwcResult;
734
735 /*
736 * Check buffer size / Allocate buffer.
737 */
738 bool fShouldFree;
739 PRTUTF16 pwszResult;
740 if (cwc > 0 && *ppwsz)
741 {
742 fShouldFree = false;
743 if (cwc <= cwcResult)
744 return VERR_BUFFER_OVERFLOW;
745 pwszResult = *ppwsz;
746 }
747 else
748 {
749 *ppwsz = NULL;
750 fShouldFree = true;
751 cwc = RT_MAX(cwcResult + 1, cwc);
752 pwszResult = (PRTUTF16)RTMemAllocTag(cwc * sizeof(RTUTF16), pszTag);
753 }
754 if (pwszResult)
755 {
756 /*
757 * Encode the UTF-16 string.
758 */
759 rc = rtUtf8RecodeAsUtf16(pszString, cchString, pwszResult, cwc - 1);
760 if (RT_SUCCESS(rc))
761 {
762 *ppwsz = pwszResult;
763 return rc;
764 }
765 if (fShouldFree)
766 RTMemFree(pwszResult);
767 }
768 else
769 rc = VERR_NO_UTF16_MEMORY;
770 }
771 return rc;
772}
773RT_EXPORT_SYMBOL(RTStrToUtf16ExTag);
774
775
776RTDECL(size_t) RTStrCalcUtf16Len(const char *psz)
777{
778 size_t cwc;
779 int rc = rtUtf8CalcUtf16Length(psz, RTSTR_MAX, &cwc);
780 return RT_SUCCESS(rc) ? cwc : 0;
781}
782RT_EXPORT_SYMBOL(RTStrCalcUtf16Len);
783
784
785RTDECL(int) RTStrCalcUtf16LenEx(const char *psz, size_t cch, size_t *pcwc)
786{
787 size_t cwc;
788 int rc = rtUtf8CalcUtf16Length(psz, cch, &cwc);
789 if (pcwc)
790 *pcwc = RT_SUCCESS(rc) ? cwc : ~(size_t)0;
791 return rc;
792}
793RT_EXPORT_SYMBOL(RTStrCalcUtf16LenEx);
794
795
796/**
797 * Calculates the length of the UTF-8 encoding of a Latin-1 string.
798 *
799 * @returns iprt status code.
800 * @param psz The Latin-1 string.
801 * @param cchIn The max length of the Latin-1 string to consider.
802 * @param pcch Where to store the length (excluding '\\0') of the UTF-8 string. (cch == cb, btw)
803 */
804static int rtLatin1CalcUtf8Length(const char *psz, size_t cchIn, size_t *pcch)
805{
806 size_t cch = 0;
807 while (cchIn > 0)
808 {
809 char ch = *psz++; cchIn--;
810 if (!ch)
811 break;
812 if (!(ch & 0x80))
813 cch++;
814 else
815 cch += 2;
816 }
817
818
819 /* done */
820 *pcch = cch;
821 return VINF_SUCCESS;
822}
823
824
825/**
826 * Recodes a Latin-1 string as UTF-8.
827 *
828 * @returns iprt status code.
829 * @param psz The Latin-1 string.
830 * @param cchIn The number of characters to process from psz. The recoding
831 * will stop when cch or '\\0' is reached.
832 * @param psz Where to store the UTF-8 string.
833 * @param cch The size of the UTF-8 buffer, excluding the terminator.
834 * @param pcch Where to store the number of octets actually encoded.
835 */
836static int rtLatin1RecodeAsUtf8(const char *pszIn, size_t cchIn, char *psz, size_t cch, size_t *pcch)
837{
838 unsigned char *puch = (unsigned char *)psz;
839 int rc = VINF_SUCCESS;
840 while (cchIn > 0)
841 {
842 char ch = *pszIn++; cchIn--;
843 if (!ch)
844 break;
845 if (!(ch & 0x80))
846 {
847 if (RT_UNLIKELY(cch < 1))
848 {
849 RTStrAssertMsgFailed(("Buffer overflow! 1\n"));
850 rc = VERR_BUFFER_OVERFLOW;
851 break;
852 }
853 cch--;
854 *puch++ = (unsigned char)ch;
855 }
856 else
857 {
858 if (RT_UNLIKELY(cch < 2))
859 {
860 RTStrAssertMsgFailed(("Buffer overflow! 2\n"));
861 rc = VERR_BUFFER_OVERFLOW;
862 break;
863 }
864 cch -= 2;
865 *puch++ = 0xc0 | (ch >> 6);
866 *puch++ = 0x80 | (ch & 0x3f);
867 }
868 }
869
870 /* done */
871 *puch = '\0';
872 *pcch = (char *)puch - psz;
873 return rc;
874}
875
876
877
878RTDECL(int) RTLatin1ToUtf8Tag(const char *pszString, char **ppszString, const char *pszTag)
879{
880 /*
881 * Validate input.
882 */
883 Assert(VALID_PTR(ppszString));
884 Assert(VALID_PTR(pszString));
885 *ppszString = NULL;
886
887 /*
888 * Calculate the length of the UTF-8 encoding of the Latin-1 string.
889 */
890 size_t cch;
891 int rc = rtLatin1CalcUtf8Length(pszString, RTSTR_MAX, &cch);
892 if (RT_SUCCESS(rc))
893 {
894 /*
895 * Allocate buffer and recode it.
896 */
897 char *pszResult = (char *)RTMemAllocTag(cch + 1, pszTag);
898 if (pszResult)
899 {
900 rc = rtLatin1RecodeAsUtf8(pszString, RTSTR_MAX, pszResult, cch, &cch);
901 if (RT_SUCCESS(rc))
902 {
903 *ppszString = pszResult;
904 return rc;
905 }
906
907 RTMemFree(pszResult);
908 }
909 else
910 rc = VERR_NO_STR_MEMORY;
911 }
912 return rc;
913}
914RT_EXPORT_SYMBOL(RTLatin1ToUtf8Tag);
915
916
917RTDECL(int) RTLatin1ToUtf8ExTag(const char *pszString, size_t cchString, char **ppsz, size_t cch, size_t *pcch, const char *pszTag)
918{
919 /*
920 * Validate input.
921 */
922 Assert(VALID_PTR(pszString));
923 Assert(VALID_PTR(ppsz));
924 Assert(!pcch || VALID_PTR(pcch));
925
926 /*
927 * Calculate the length of the UTF-8 encoding of the Latin-1 string.
928 */
929 size_t cchResult;
930 int rc = rtLatin1CalcUtf8Length(pszString, cchString, &cchResult);
931 if (RT_SUCCESS(rc))
932 {
933 if (pcch)
934 *pcch = cchResult;
935
936 /*
937 * Check buffer size / Allocate buffer and recode it.
938 */
939 bool fShouldFree;
940 char *pszResult;
941 if (cch > 0 && *ppsz)
942 {
943 fShouldFree = false;
944 if (RT_UNLIKELY(cch <= cchResult))
945 return VERR_BUFFER_OVERFLOW;
946 pszResult = *ppsz;
947 }
948 else
949 {
950 *ppsz = NULL;
951 fShouldFree = true;
952 cch = RT_MAX(cch, cchResult + 1);
953 pszResult = (char *)RTStrAllocTag(cch, pszTag);
954 }
955 if (pszResult)
956 {
957 rc = rtLatin1RecodeAsUtf8(pszString, cchString, pszResult, cch - 1, &cch);
958 if (RT_SUCCESS(rc))
959 {
960 *ppsz = pszResult;
961 return rc;
962 }
963
964 if (fShouldFree)
965 RTStrFree(pszResult);
966 }
967 else
968 rc = VERR_NO_STR_MEMORY;
969 }
970 return rc;
971}
972RT_EXPORT_SYMBOL(RTLatin1ToUtf8ExTag);
973
974
975RTDECL(size_t) RTLatin1CalcUtf8Len(const char *psz)
976{
977 size_t cch;
978 int rc = rtLatin1CalcUtf8Length(psz, RTSTR_MAX, &cch);
979 return RT_SUCCESS(rc) ? cch : 0;
980}
981RT_EXPORT_SYMBOL(RTLatin1CalcUtf8Len);
982
983
984RTDECL(int) RTLatin1CalcUtf8LenEx(const char *psz, size_t cchIn, size_t *pcch)
985{
986 size_t cch;
987 int rc = rtLatin1CalcUtf8Length(psz, cchIn, &cch);
988 if (pcch)
989 *pcch = RT_SUCCESS(rc) ? cch : ~(size_t)0;
990 return rc;
991}
992RT_EXPORT_SYMBOL(RTLatin1CalcUtf8LenEx);
993
994
995/**
996 * Calculates the Latin-1 length of a string, validating the encoding while doing so.
997 *
998 * @returns IPRT status code.
999 * @param psz Pointer to the UTF-8 string.
1000 * @param cch The max length of the string. (btw cch = cb)
1001 * Use RTSTR_MAX if all of the string is to be examined.
1002 * @param pcwc Where to store the length of the Latin-1 string in bytes.
1003 */
1004static int rtUtf8CalcLatin1Length(const char *psz, size_t cch, size_t *pcch)
1005{
1006 /* We re-encode to one byte per unicode code point. */
1007 return RTStrUniLenEx(psz, cch, pcch);
1008}
1009
1010
1011/**
1012 * Recodes a valid UTF-8 string as Latin-1.
1013 *
1014 * Since we know the input is valid, we do *not* perform encoding or length checks.
1015 *
1016 * @returns iprt status code.
1017 * @param psz The UTF-8 string to recode. This is a valid encoding.
1018 * @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
1019 * The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
1020 * @param pszOut Where to store the Latin-1 string.
1021 * @param cchOut The number of characters the pszOut buffer can hold, excluding the terminator ('\\0').
1022 */
1023static int rtUtf8RecodeAsLatin1(const char *psz, size_t cch, char *pszOut, size_t cchOut)
1024{
1025 int rc = VINF_SUCCESS;
1026 const unsigned char *puch = (const unsigned char *)psz;
1027 unsigned char *puchOut = (unsigned char *)pszOut;
1028 while (cch > 0)
1029 {
1030 /* read the next char and check for terminator. */
1031 const unsigned char uch = *puch;
1032 if (!uch)
1033 break;
1034
1035 /* check for output overflow */
1036 if (RT_UNLIKELY(cchOut < 1))
1037 {
1038 rc = VERR_BUFFER_OVERFLOW;
1039 break;
1040 }
1041 cchOut--;
1042
1043 /* decode and recode the code point */
1044 if (!(uch & RT_BIT(7)))
1045 {
1046 *puchOut++ = uch;
1047 puch++;
1048 cch--;
1049 }
1050 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
1051 {
1052 uint16_t uc = (puch[1] & 0x3f)
1053 | ((uint16_t)(uch & 0x1f) << 6);
1054 if (uc >= 0x100)
1055 {
1056 rc = VERR_NO_TRANSLATION;
1057 break;
1058 }
1059 *puchOut++ = uc;
1060 puch += 2;
1061 cch -= 2;
1062 }
1063 else
1064 {
1065 rc = VERR_NO_TRANSLATION;
1066 break;
1067 }
1068 }
1069
1070 /* done */
1071 *puchOut = '\0';
1072 return rc;
1073}
1074
1075
1076RTDECL(int) RTStrToLatin1Tag(const char *pszString, char **ppszString, const char *pszTag)
1077{
1078 /*
1079 * Validate input.
1080 */
1081 Assert(VALID_PTR(ppszString));
1082 Assert(VALID_PTR(pszString));
1083 *ppszString = NULL;
1084
1085 /*
1086 * Validate the UTF-8 input and calculate the length of the Latin-1 string.
1087 */
1088 size_t cch;
1089 int rc = rtUtf8CalcLatin1Length(pszString, RTSTR_MAX, &cch);
1090 if (RT_SUCCESS(rc))
1091 {
1092 /*
1093 * Allocate buffer.
1094 */
1095 char *psz = (char *)RTMemAllocTag(cch + 1, pszTag);
1096 if (psz)
1097 {
1098 /*
1099 * Encode the UTF-16 string.
1100 */
1101 rc = rtUtf8RecodeAsLatin1(pszString, RTSTR_MAX, psz, cch);
1102 if (RT_SUCCESS(rc))
1103 {
1104 *ppszString = psz;
1105 return rc;
1106 }
1107 RTMemFree(psz);
1108 }
1109 else
1110 rc = VERR_NO_STR_MEMORY;
1111 }
1112 return rc;
1113}
1114RT_EXPORT_SYMBOL(RTStrToLatin1Tag);
1115
1116
1117RTDECL(int) RTStrToLatin1ExTag(const char *pszString, size_t cchString,
1118 char **ppsz, size_t cch, size_t *pcch, const char *pszTag)
1119{
1120 /*
1121 * Validate input.
1122 */
1123 Assert(VALID_PTR(pszString));
1124 Assert(VALID_PTR(ppsz));
1125 Assert(!pcch || VALID_PTR(pcch));
1126
1127 /*
1128 * Validate the UTF-8 input and calculate the length of the UTF-16 string.
1129 */
1130 size_t cchResult;
1131 int rc = rtUtf8CalcLatin1Length(pszString, cchString, &cchResult);
1132 if (RT_SUCCESS(rc))
1133 {
1134 if (pcch)
1135 *pcch = cchResult;
1136
1137 /*
1138 * Check buffer size / Allocate buffer.
1139 */
1140 bool fShouldFree;
1141 char *pszResult;
1142 if (cch > 0 && *ppsz)
1143 {
1144 fShouldFree = false;
1145 if (cch <= cchResult)
1146 return VERR_BUFFER_OVERFLOW;
1147 pszResult = *ppsz;
1148 }
1149 else
1150 {
1151 *ppsz = NULL;
1152 fShouldFree = true;
1153 cch = RT_MAX(cchResult + 1, cch);
1154 pszResult = (char *)RTMemAllocTag(cch, pszTag);
1155 }
1156 if (pszResult)
1157 {
1158 /*
1159 * Encode the Latin-1 string.
1160 */
1161 rc = rtUtf8RecodeAsLatin1(pszString, cchString, pszResult, cch - 1);
1162 if (RT_SUCCESS(rc))
1163 {
1164 *ppsz = pszResult;
1165 return rc;
1166 }
1167 if (fShouldFree)
1168 RTMemFree(pszResult);
1169 }
1170 else
1171 rc = VERR_NO_STR_MEMORY;
1172 }
1173 return rc;
1174}
1175RT_EXPORT_SYMBOL(RTStrToLatin1Tag);
1176
1177
1178/**
1179 * Handle invalid encodings passed to RTStrGetCp() and RTStrGetCpEx().
1180 * @returns rc
1181 * @param ppsz The pointer to the string position point.
1182 * @param pCp Where to store RTUNICP_INVALID.
1183 * @param rc The iprt error code.
1184 */
1185static int rtStrGetCpExFailure(const char **ppsz, PRTUNICP pCp, int rc)
1186{
1187 /*
1188 * Try find a valid encoding.
1189 */
1190 (*ppsz)++; /** @todo code this! */
1191 *pCp = RTUNICP_INVALID;
1192 return rc;
1193}
1194
1195
1196RTDECL(RTUNICP) RTStrGetCpInternal(const char *psz)
1197{
1198 RTUNICP Cp;
1199 RTStrGetCpExInternal(&psz, &Cp);
1200 return Cp;
1201}
1202RT_EXPORT_SYMBOL(RTStrGetCpInternal);
1203
1204
1205RTDECL(int) RTStrGetCpExInternal(const char **ppsz, PRTUNICP pCp)
1206{
1207 const unsigned char *puch = (const unsigned char *)*ppsz;
1208 const unsigned char uch = *puch;
1209 RTUNICP uc;
1210
1211 /* ASCII ? */
1212 if (!(uch & RT_BIT(7)))
1213 {
1214 uc = uch;
1215 puch++;
1216 }
1217 else if (uch & RT_BIT(6))
1218 {
1219 /* figure the length and validate the first octet. */
1220 unsigned cb;
1221 if (!(uch & RT_BIT(5)))
1222 cb = 2;
1223 else if (!(uch & RT_BIT(4)))
1224 cb = 3;
1225 else if (!(uch & RT_BIT(3)))
1226 cb = 4;
1227 else if (!(uch & RT_BIT(2)))
1228 cb = 5;
1229 else if (!(uch & RT_BIT(1)))
1230 cb = 6;
1231 else
1232 {
1233 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1234 return rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING);
1235 }
1236
1237 /* validate the rest */
1238 switch (cb)
1239 {
1240 case 6:
1241 RTStrAssertMsgReturn((puch[5] & 0xc0) == 0x80, ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1242 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1243 case 5:
1244 RTStrAssertMsgReturn((puch[4] & 0xc0) == 0x80, ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1245 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1246 case 4:
1247 RTStrAssertMsgReturn((puch[3] & 0xc0) == 0x80, ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1248 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1249 case 3:
1250 RTStrAssertMsgReturn((puch[2] & 0xc0) == 0x80, ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1251 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1252 case 2:
1253 RTStrAssertMsgReturn((puch[1] & 0xc0) == 0x80, ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1254 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1255 break;
1256 }
1257
1258 /* get and validate the code point. */
1259 switch (cb)
1260 {
1261 case 6:
1262 uc = (puch[5] & 0x3f)
1263 | ((RTUNICP)(puch[4] & 0x3f) << 6)
1264 | ((RTUNICP)(puch[3] & 0x3f) << 12)
1265 | ((RTUNICP)(puch[2] & 0x3f) << 18)
1266 | ((RTUNICP)(puch[1] & 0x3f) << 24)
1267 | ((RTUNICP)(uch & 0x01) << 30);
1268 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
1269 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1270 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1271 break;
1272 case 5:
1273 uc = (puch[4] & 0x3f)
1274 | ((RTUNICP)(puch[3] & 0x3f) << 6)
1275 | ((RTUNICP)(puch[2] & 0x3f) << 12)
1276 | ((RTUNICP)(puch[1] & 0x3f) << 18)
1277 | ((RTUNICP)(uch & 0x03) << 24);
1278 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
1279 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1280 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1281 break;
1282 case 4:
1283 uc = (puch[3] & 0x3f)
1284 | ((RTUNICP)(puch[2] & 0x3f) << 6)
1285 | ((RTUNICP)(puch[1] & 0x3f) << 12)
1286 | ((RTUNICP)(uch & 0x07) << 18);
1287 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
1288 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1289 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1290 break;
1291 case 3:
1292 uc = (puch[2] & 0x3f)
1293 | ((RTUNICP)(puch[1] & 0x3f) << 6)
1294 | ((RTUNICP)(uch & 0x0f) << 12);
1295 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
1296 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1297 rtStrGetCpExFailure(ppsz, pCp, uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING));
1298 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
1299 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1300 rtStrGetCpExFailure(ppsz, pCp, VERR_CODE_POINT_SURROGATE));
1301 break;
1302 case 2:
1303 uc = (puch[1] & 0x3f)
1304 | ((RTUNICP)(uch & 0x1f) << 6);
1305 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
1306 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1307 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1308 break;
1309 default: /* impossible, but GCC is bitching. */
1310 uc = RTUNICP_INVALID;
1311 break;
1312 }
1313 puch += cb;
1314 }
1315 else
1316 {
1317 /* 6th bit is always set. */
1318 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1319 return rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING);
1320 }
1321 *pCp = uc;
1322 *ppsz = (const char *)puch;
1323 return VINF_SUCCESS;
1324}
1325RT_EXPORT_SYMBOL(RTStrGetCpExInternal);
1326
1327
1328/**
1329 * Handle invalid encodings passed to RTStrGetCpNEx().
1330 * @returns rc
1331 * @param ppsz The pointer to the string position point.
1332 * @param pcch Pointer to the string length.
1333 * @param pCp Where to store RTUNICP_INVALID.
1334 * @param rc The iprt error code.
1335 */
1336static int rtStrGetCpNExFailure(const char **ppsz, size_t *pcch, PRTUNICP pCp, int rc)
1337{
1338 /*
1339 * Try find a valid encoding.
1340 */
1341 (*ppsz)++; /** @todo code this! */
1342 (*pcch)--;
1343 *pCp = RTUNICP_INVALID;
1344 return rc;
1345}
1346
1347
1348RTDECL(int) RTStrGetCpNExInternal(const char **ppsz, size_t *pcch, PRTUNICP pCp)
1349{
1350 const unsigned char *puch = (const unsigned char *)*ppsz;
1351 const unsigned char uch = *puch;
1352 size_t cch = *pcch;
1353 RTUNICP uc;
1354
1355 if (cch == 0)
1356 {
1357 *pCp = RTUNICP_INVALID;
1358 return VERR_END_OF_STRING;
1359 }
1360
1361 /* ASCII ? */
1362 if (!(uch & RT_BIT(7)))
1363 {
1364 uc = uch;
1365 puch++;
1366 cch--;
1367 }
1368 else if (uch & RT_BIT(6))
1369 {
1370 /* figure the length and validate the first octet. */
1371 unsigned cb;
1372 if (!(uch & RT_BIT(5)))
1373 cb = 2;
1374 else if (!(uch & RT_BIT(4)))
1375 cb = 3;
1376 else if (!(uch & RT_BIT(3)))
1377 cb = 4;
1378 else if (!(uch & RT_BIT(2)))
1379 cb = 5;
1380 else if (!(uch & RT_BIT(1)))
1381 cb = 6;
1382 else
1383 {
1384 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1385 return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1386 }
1387
1388 if (cb > cch)
1389 return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1390
1391 /* validate the rest */
1392 switch (cb)
1393 {
1394 case 6:
1395 RTStrAssertMsgReturn((puch[5] & 0xc0) == 0x80, ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1396 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1397 case 5:
1398 RTStrAssertMsgReturn((puch[4] & 0xc0) == 0x80, ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1399 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1400 case 4:
1401 RTStrAssertMsgReturn((puch[3] & 0xc0) == 0x80, ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1402 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1403 case 3:
1404 RTStrAssertMsgReturn((puch[2] & 0xc0) == 0x80, ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1405 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1406 case 2:
1407 RTStrAssertMsgReturn((puch[1] & 0xc0) == 0x80, ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1408 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1409 break;
1410 }
1411
1412 /* get and validate the code point. */
1413 switch (cb)
1414 {
1415 case 6:
1416 uc = (puch[5] & 0x3f)
1417 | ((RTUNICP)(puch[4] & 0x3f) << 6)
1418 | ((RTUNICP)(puch[3] & 0x3f) << 12)
1419 | ((RTUNICP)(puch[2] & 0x3f) << 18)
1420 | ((RTUNICP)(puch[1] & 0x3f) << 24)
1421 | ((RTUNICP)(uch & 0x01) << 30);
1422 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
1423 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1424 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1425 break;
1426 case 5:
1427 uc = (puch[4] & 0x3f)
1428 | ((RTUNICP)(puch[3] & 0x3f) << 6)
1429 | ((RTUNICP)(puch[2] & 0x3f) << 12)
1430 | ((RTUNICP)(puch[1] & 0x3f) << 18)
1431 | ((RTUNICP)(uch & 0x03) << 24);
1432 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
1433 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1434 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1435 break;
1436 case 4:
1437 uc = (puch[3] & 0x3f)
1438 | ((RTUNICP)(puch[2] & 0x3f) << 6)
1439 | ((RTUNICP)(puch[1] & 0x3f) << 12)
1440 | ((RTUNICP)(uch & 0x07) << 18);
1441 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
1442 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1443 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1444 break;
1445 case 3:
1446 uc = (puch[2] & 0x3f)
1447 | ((RTUNICP)(puch[1] & 0x3f) << 6)
1448 | ((RTUNICP)(uch & 0x0f) << 12);
1449 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
1450 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1451 rtStrGetCpNExFailure(ppsz, pcch, pCp, uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING));
1452 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
1453 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1454 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_CODE_POINT_SURROGATE));
1455 break;
1456 case 2:
1457 uc = (puch[1] & 0x3f)
1458 | ((RTUNICP)(uch & 0x1f) << 6);
1459 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
1460 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1461 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1462 break;
1463 default: /* impossible, but GCC is bitching. */
1464 uc = RTUNICP_INVALID;
1465 break;
1466 }
1467 puch += cb;
1468 cch -= cb;
1469 }
1470 else
1471 {
1472 /* 6th bit is always set. */
1473 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1474 return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1475 }
1476 *pCp = uc;
1477 *ppsz = (const char *)puch;
1478 (*pcch) = cch;
1479 return VINF_SUCCESS;
1480}
1481RT_EXPORT_SYMBOL(RTStrGetCpNExInternal);
1482
1483
1484RTDECL(char *) RTStrPutCpInternal(char *psz, RTUNICP uc)
1485{
1486 unsigned char *puch = (unsigned char *)psz;
1487 if (uc < 0x80)
1488 *puch++ = (unsigned char )uc;
1489 else if (uc < 0x00000800)
1490 {
1491 *puch++ = 0xc0 | (uc >> 6);
1492 *puch++ = 0x80 | (uc & 0x3f);
1493 }
1494 else if (uc < 0x00010000)
1495 {
1496 if ( uc < 0x0000d8000
1497 || ( uc > 0x0000dfff
1498 && uc < 0x0000fffe))
1499 {
1500 *puch++ = 0xe0 | (uc >> 12);
1501 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1502 *puch++ = 0x80 | (uc & 0x3f);
1503 }
1504 else
1505 {
1506 AssertMsgFailed(("Invalid code point U+%05x!\n", uc));
1507 *puch++ = 0x7f;
1508 }
1509 }
1510 else if (uc < 0x00200000)
1511 {
1512 *puch++ = 0xf0 | (uc >> 18);
1513 *puch++ = 0x80 | ((uc >> 12) & 0x3f);
1514 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1515 *puch++ = 0x80 | (uc & 0x3f);
1516 }
1517 else if (uc < 0x04000000)
1518 {
1519 *puch++ = 0xf8 | (uc >> 24);
1520 *puch++ = 0x80 | ((uc >> 18) & 0x3f);
1521 *puch++ = 0x80 | ((uc >> 12) & 0x3f);
1522 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1523 *puch++ = 0x80 | (uc & 0x3f);
1524 }
1525 else if (uc <= 0x7fffffff)
1526 {
1527 *puch++ = 0xfc | (uc >> 30);
1528 *puch++ = 0x80 | ((uc >> 24) & 0x3f);
1529 *puch++ = 0x80 | ((uc >> 18) & 0x3f);
1530 *puch++ = 0x80 | ((uc >> 12) & 0x3f);
1531 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1532 *puch++ = 0x80 | (uc & 0x3f);
1533 }
1534 else
1535 {
1536 AssertMsgFailed(("Invalid code point U+%08x!\n", uc));
1537 *puch++ = 0x7f;
1538 }
1539
1540 return (char *)puch;
1541}
1542RT_EXPORT_SYMBOL(RTStrPutCpInternal);
1543
1544
1545RTDECL(char *) RTStrPrevCp(const char *pszStart, const char *psz)
1546{
1547 if (pszStart < psz)
1548 {
1549 /* simple char? */
1550 const unsigned char *puch = (const unsigned char *)psz;
1551 unsigned uch = *--puch;
1552 if (!(uch & RT_BIT(7)))
1553 return (char *)puch;
1554 RTStrAssertMsgReturn(!(uch & RT_BIT(6)), ("uch=%#x\n", uch), (char *)pszStart);
1555
1556 /* two or more. */
1557 uint32_t uMask = 0xffffffc0;
1558 while ( (const unsigned char *)pszStart < puch
1559 && !(uMask & 1))
1560 {
1561 uch = *--puch;
1562 if ((uch & 0xc0) != 0x80)
1563 {
1564 RTStrAssertMsgReturn((uch & (uMask >> 1)) == (uMask & 0xff),
1565 ("Invalid UTF-8 encoding: %.*Rhxs puch=%p psz=%p\n", psz - (char *)puch, puch, psz),
1566 (char *)pszStart);
1567 return (char *)puch;
1568 }
1569 uMask >>= 1;
1570 }
1571 RTStrAssertMsgFailed(("Invalid UTF-8 encoding: %.*Rhxs puch=%p psz=%p\n", psz - (char *)puch, puch, psz));
1572 }
1573 return (char *)pszStart;
1574}
1575RT_EXPORT_SYMBOL(RTStrPrevCp);
1576
注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette