summaryrefslogtreecommitdiff
path: root/intl/uconv/nsUTF8ToUnicode.cpp
blob: d4e28b04dfcbfd5a4a03b8ce6ef387e08c0d71ba (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "nsUCSupport.h"
#include "nsUTF8ToUnicode.h"
#include "mozilla/CheckedInt.h"
#include "mozilla/SSE.h"
#include "nsCharTraits.h"
#include <algorithm>

#define UNICODE_BYTE_ORDER_MARK    0xFEFF

static char16_t* EmitSurrogatePair(uint32_t ucs4, char16_t* aDest)
{
  NS_ASSERTION(ucs4 > 0xFFFF, "Should be a supplementary character");
  ucs4 -= 0x00010000;
  *aDest++ = 0xD800 | (0x000003FF & (ucs4 >> 10));
  *aDest++ = 0xDC00 | (0x000003FF & ucs4);
  return aDest;
}

//----------------------------------------------------------------------
// Class nsUTF8ToUnicode [implementation]

nsUTF8ToUnicode::nsUTF8ToUnicode()
: nsBasicDecoderSupport()
{
  Reset();
}

//----------------------------------------------------------------------
// Subclassing of nsTableDecoderSupport class [implementation]

/**
 * Normally the maximum length of the output of the UTF8 decoder in UTF16
 *  code units is the same as the length of the input in UTF8 code units,
 *  since 1-byte, 2-byte and 3-byte UTF-8 sequences decode to a single
 *  UTF-16 character, and 4-byte UTF-8 sequences decode to a surrogate pair.
 *
 * However, there is an edge case where the output can be longer than the
 *  input: if the previous buffer ended with an incomplete multi-byte
 *  sequence and this buffer does not begin with a valid continuation
 *  byte, we will return NS_ERROR_ILLEGAL_INPUT and the caller may insert a
 *  replacement character in the output buffer which corresponds to no
 *  character in the input buffer. So in the worst case the destination
 *  will need to be one code unit longer than the source.
 *  See bug 301797.
 */
NS_IMETHODIMP nsUTF8ToUnicode::GetMaxLength(const char * aSrc,
                                            int32_t aSrcLength,
                                            int32_t * aDestLength)
{
  mozilla::CheckedInt32 length = aSrcLength;
  length += 1;

  if (!length.isValid()) {
    return NS_ERROR_OUT_OF_MEMORY;
  }

  *aDestLength = length.value();
  return NS_OK;
}


//----------------------------------------------------------------------
// Subclassing of nsBasicDecoderSupport class [implementation]

NS_IMETHODIMP nsUTF8ToUnicode::Reset()
{

  mUcs4  = 0;     // cached Unicode character
  mState = 0;     // cached expected number of octets after the current octet
                  // until the beginning of the next UTF8 character sequence
  mBytes = 1;     // cached expected number of octets in the current sequence
  mFirst = true;

  return NS_OK;

}

//----------------------------------------------------------------------
// Subclassing of nsBasicDecoderSupport class [implementation]

// Fast ASCII -> UTF16 inner loop implementations
//
// Convert_ascii_run will update src and dst to the new values, and
// len must be the maximum number ascii chars that it would be valid
// to take from src and place into dst.  (That is, the minimum of the
// number of bytes left in src and the number of unichars available in
// dst.)

#if defined(__arm__) || defined(_M_ARM)

// on ARM, do extra work to avoid byte/halfword reads/writes by
// reading/writing a word at a time for as long as we can
static inline void
Convert_ascii_run (const char *&src,
                   char16_t *&dst,
                   int32_t len)
{
  const uint32_t *src32;
  uint32_t *dst32;

  // with some alignments, we'd never actually break out of the slow loop, so
  // check and do the faster slow loop
  if ((((NS_PTR_TO_UINT32(dst) & 3) == 0) && ((NS_PTR_TO_UINT32(src) & 1) == 0)) ||
      (((NS_PTR_TO_UINT32(dst) & 3) == 2) && ((NS_PTR_TO_UINT32(src) & 1) == 1)))
  {
    while (((NS_PTR_TO_UINT32(src) & 3) ||
            (NS_PTR_TO_UINT32(dst) & 3)) &&
           len > 0)
    {
      if (*src & 0x80U)
        return;
      *dst++ = (char16_t) *src++;
      len--;
    }
  } else {
    goto finish;
  }

  // then go 4 bytes at a time
  src32 = (const uint32_t*) src;
  dst32 = (uint32_t*) dst;

  while (len > 4) {
    uint32_t in = *src32++;

    if (in & 0x80808080U) {
      src32--;
      break;
    }

    *dst32++ = ((in & 0x000000ff) >>  0) | ((in & 0x0000ff00) << 8);
    *dst32++ = ((in & 0x00ff0000) >> 16) | ((in & 0xff000000) >> 8);

    len -= 4;
  }

  src = (const char *) src32;
  dst = (char16_t *) dst32;

finish:
  while (len-- > 0 && (*src & 0x80U) == 0) {
    *dst++ = (char16_t) *src++;
  }
}

#else

#ifdef MOZILLA_MAY_SUPPORT_SSE2
namespace mozilla {
namespace SSE2 {

void Convert_ascii_run(const char *&src, char16_t *&dst, int32_t len);

} // namespace SSE2
} // namespace mozilla
#endif

static inline void
Convert_ascii_run (const char *&src,
                   char16_t *&dst,
                   int32_t len)
{
#ifdef MOZILLA_MAY_SUPPORT_SSE2
  if (mozilla::supports_sse2()) {
    mozilla::SSE2::Convert_ascii_run(src, dst, len);
    return;
  }
#endif

  while (len-- > 0 && (*src & 0x80U) == 0) {
    *dst++ = (char16_t) *src++;
  }
}

#endif

NS_IMETHODIMP nsUTF8ToUnicode::Convert(const char * aSrc,
                                       int32_t * aSrcLength,
                                       char16_t * aDest,
                                       int32_t * aDestLength)
{
  uint32_t aSrcLen   = (uint32_t) (*aSrcLength);
  uint32_t aDestLen = (uint32_t) (*aDestLength);

  const char *in, *inend;
  inend = aSrc + aSrcLen;

  char16_t *out, *outend;
  outend = aDest + aDestLen;

  nsresult res = NS_OK; // conversion result

  out = aDest;
  if (mState == 0xFF) {
    // Emit supplementary character left over from previous iteration. It is
    // caller's responsibility to keep a sufficient buffer.
    if (aDestLen < 2) {
      *aSrcLength = *aDestLength = 0;
      return NS_OK_UDEC_MOREOUTPUT;
    }
    out = EmitSurrogatePair(mUcs4, out);
    mUcs4 = 0;
    mState = 0;
    mBytes = 1;
    mFirst = false;
  }

  // alias these locally for speed
  int32_t mUcs4 = this->mUcs4;
  uint8_t mState = this->mState;
  uint8_t mBytes = this->mBytes;
  bool mFirst = this->mFirst;

  // Set mFirst to false now so we don't have to every time through the ASCII
  // branch within the loop.
  if (mFirst && aSrcLen && (0 == (0x80 & (*aSrc))))
    mFirst = false;

  for (in = aSrc; ((in < inend) && (out < outend)); ++in) {
    uint8_t c = *in;
    if (0 == mState) {
      // When mState is zero we expect either a US-ASCII character or a
      // multi-octet sequence.
      if (c < 0x80) {  // 00..7F
        int32_t max_loops = std::min(inend - in, outend - out);
        Convert_ascii_run(in, out, max_loops);
        --in; // match the rest of the cases
        mBytes = 1;
      } else if (c < 0xC2) {  // C0/C1
        // Overlong 2 octet sequence
        if (mErrBehavior == kOnError_Signal) {
          res = NS_ERROR_ILLEGAL_INPUT;
          break;
        }
        *out++ = UCS2_REPLACEMENT_CHAR;
        mFirst = false;
      } else if (c < 0xE0) {  // C2..DF
        // First octet of 2 octet sequence
        mUcs4 = c;
        mUcs4 = (mUcs4 & 0x1F) << 6;
        mState = 1;
        mBytes = 2;
      } else if (c < 0xF0) {  // E0..EF
        // First octet of 3 octet sequence
        mUcs4 = c;
        mUcs4 = (mUcs4 & 0x0F) << 12;
        mState = 2;
        mBytes = 3;
      } else if (c < 0xF5) {  // F0..F4
        // First octet of 4 octet sequence
        mUcs4 = c;
        mUcs4 = (mUcs4 & 0x07) << 18;
        mState = 3;
        mBytes = 4;
      } else {  // F5..FF
        /* Current octet is neither in the US-ASCII range nor a legal first
         * octet of a multi-octet sequence.
         */
        if (mErrBehavior == kOnError_Signal) {
          /* Return an error condition. Caller is responsible for flushing and
           * refilling the buffer and resetting state.
           */
          res = NS_ERROR_ILLEGAL_INPUT;
          break;
        }
        *out++ = UCS2_REPLACEMENT_CHAR;
        mFirst = false;
      }
    } else {
      // When mState is non-zero, we expect a continuation of the multi-octet
      // sequence
      if (0x80 == (0xC0 & c)) {
        if (mState > 1) {
          // If we are here, all possibilities are:
          // mState == 2 && mBytes == 3 ||
          // mState == 2 && mBytes == 4 ||
          // mState == 3 && mBytes == 4
          if ((mBytes == 3 && ((!mUcs4 && c < 0xA0) ||             // E0 80..9F
                               (mUcs4 == 0xD000 && c > 0x9F))) ||  // ED A0..BF
              (mState == 3 && ((!mUcs4 && c < 0x90) ||             // F0 80..8F
                               (mUcs4 == 0x100000 && c > 0x8F)))) {// F4 90..BF
            // illegal sequences or sequences converted into illegal ranges.
            in--;
            if (mErrBehavior == kOnError_Signal) {
              res = NS_ERROR_ILLEGAL_INPUT;
              break;
            }
            *out++ = UCS2_REPLACEMENT_CHAR;
            mState = 0;
            mFirst = false;
            continue;
          }
        }

        // Legal continuation.
        uint32_t shift = (mState - 1) * 6;
        uint32_t tmp = c;
        tmp = (tmp & 0x0000003FL) << shift;
        mUcs4 |= tmp;

        if (0 == --mState) {
          /* End of the multi-octet sequence. mUcs4 now contains the final
           * Unicode codepoint to be output
           */

          if (mUcs4 > 0xFFFF) {
            // mUcs4 is in the range 0x10000 - 0x10FFFF. Output a UTF-16 pair
            if (out + 2 > outend) {
              // insufficient space left in the buffer. Keep mUcs4 for the
              // next iteration.
              mState = 0xFF;
              ++in;
              res = NS_OK_UDEC_MOREOUTPUT;
              break;
            }
            out = EmitSurrogatePair(mUcs4, out);
          } else if (UNICODE_BYTE_ORDER_MARK != mUcs4 || !mFirst) {
            // Don't output the BOM only if it is the first character
            *out++ = mUcs4;
          }
          //initialize UTF8 cache
          mUcs4  = 0;
          mState = 0;
          mBytes = 1;
          mFirst = false;
        }
      } else {
        /* ((0xC0 & c != 0x80) && (mState != 0))
         *
         * Incomplete multi-octet sequence. Unconsume this
         * octet and return an error condition. Caller is responsible
         * for flushing and refilling the buffer and resetting state.
         */
        in--;
        if (mErrBehavior == kOnError_Signal) {
          res = NS_ERROR_ILLEGAL_INPUT;
          break;
        }
        *out++ = UCS2_REPLACEMENT_CHAR;
        mState = 0;
        mFirst = false;
      }
    }
  }

  // output not finished, output buffer too short
  if ((NS_OK == res) && (in < inend) && (out >= outend))
    res = NS_OK_UDEC_MOREOUTPUT;

  // last UCS4 is incomplete, make sure the caller
  // returns with properly aligned continuation of the buffer
  if ((NS_OK == res) && (mState != 0))
    res = NS_OK_UDEC_MOREINPUT;

  *aSrcLength = in - aSrc;
  *aDestLength = out - aDest;

  this->mUcs4 = mUcs4;
  this->mState = mState;
  this->mBytes = mBytes;
  this->mFirst = mFirst;

  return(res);
}