firefox-main/testing/web-platform/tests/encoding/textdecoder-mistakes.any.js (file symbol)

Source code

Revision control

Copy as Markdown

Other Tools

Test Info: Warnings

// These collect actual implementation failures
const u = (...args) => Uint8Array.of(...args)
{
const invalid8 = [
{ bytes: [0, 254, 255], charcodes: [0, 0xff_fd, 0xff_fd] },
{ bytes: [0x80], charcodes: [0xff_fd] },
{ bytes: [0xf0, 0x90, 0x80], charcodes: [0xff_fd] }, // https://npmjs.com/package/buffer is wrong
{ bytes: [0xf0, 0x80, 0x80], charcodes: [0xff_fd, 0xff_fd, 0xff_fd] }, // https://github.com/nodejs/node/issues/16894
]
const invalid16 = [
{ invalid: [0x61, 0x62, 0xd8_00, 0x77, 0x78], replaced: [0x61, 0x62, 0xff_fd, 0x77, 0x78] },
{ invalid: [0xd8_00], replaced: [0xff_fd] },
{ invalid: [0xd8_00, 0xd8_00], replaced: [0xff_fd, 0xff_fd] },
{ invalid: [0x61, 0x62, 0xdf_ff, 0x77, 0x78], replaced: [0x61, 0x62, 0xff_fd, 0x77, 0x78] },
{ invalid: [0xdf_ff, 0xd8_00], replaced: [0xff_fd, 0xff_fd] },
]
// https://npmjs.com/package/buffer is wrong on this
// which means that iconv-lite and whatwg-encoding are wrong on non-Node.js environments
// this test fails on whatwg-encoding in browsers
test(() => {
const d = new TextDecoder()
for (const { bytes, charcodes } of invalid8) {
assert_equals(d.decode(Uint8Array.from(bytes)), String.fromCharCode(...charcodes))
}
}, 'Invalid Unicode input is replaced: utf-8')
// whatwg-encoding, iconv-lite, and any other implementations just using Buffer fail on this
test(() => {
const d = new TextDecoder('utf-16le')
for (const { invalid, replaced } of invalid16) {
const input = new Uint8Array(invalid.length * 2)
for (let i = 0; i < invalid.length; i++) {
input[2 * i] = invalid[i] & 0xff
input[2 * i + 1] = invalid[i] >> 8
}
assert_equals(d.decode(input), String.fromCharCode(...replaced))
}
}, 'Invalid Unicode input is replaced: utf-16le')
// whatwg-encoding, iconv-lite, and any other implementations just using Buffer fail on this
test(() => {
const d = new TextDecoder('utf-16be')
for (const { invalid, replaced } of invalid16) {
const input = new Uint8Array(invalid.length * 2)
for (let i = 0; i < invalid.length; i++) {
input[2 * i] = invalid[i] >> 8
input[2 * i + 1] = invalid[i] & 0xff
}
assert_equals(d.decode(input), String.fromCharCode(...replaced))
}
}, 'Invalid Unicode input is replaced: utf-16be')
}
// Node.js fails on this
for (const encoding of ['windows-1252', 'ibm866']) {
test(() => {
for (const fatal of [false, true]) {
const d = new TextDecoder(encoding, { fatal })
for (const b of [0x00, 0x1a, 0x1c, 0x42, 0x7f]) {
assert_equals(d.decode(u(b)), String.fromCharCode(b), `Byte: ${b}`)
}
}
}, `Single-byte encodings are ASCII supersets: ${encoding}`)
}
// Node.js fails on this
for (const encoding of ['gbk', 'gb18030', 'big5', 'euc-jp', 'shift_jis', 'euc-kr']) {
test(() => {
for (const fatal of [false, true]) {
const d = new TextDecoder(encoding, { fatal })
for (const b of [0x00, 0x1a, 0x1c, 0x42, 0x7f]) {
assert_equals(d.decode(u(b)), String.fromCharCode(b), `Byte: ${b}`)
}
}
}, `Most legacy multi-byte encodings are ASCII supersets: ${encoding}`)
}
// Chrome breaks on this
{
const windows = [874, 1250, 1252, 1253, 1254, 1255, 1257, 1258].map((x) => `windows-${x}`) // these have \x80 mapped to euro sign
for (const encoding of [...windows, 'latin1', 'ascii']) {
test(() => {
for (const fatal of [false, true]) {
const d = new TextDecoder(encoding, { fatal })
for (let i = 1; i <= 33; i++) {
const u8 = new Uint8Array(i)
u8[i - 1] = 0x80
assert_equals(d.decode(u8)[i - 1], '€')
}
}
}, `Fast path misdetection: ${encoding}`)
}
}
test(() => {
const d = new TextDecoder('utf-16le')
assert_equals(d.decode(u(0, 0, 0)), '\0\uFFFD') // two character, 0 was valid
assert_equals(d.decode(u(42, 0, 0)), '*\uFFFD') // two characters, * was valid
assert_equals(d.decode(u(0, 0xd8, 0)), '\uFFFD') // single character
assert_equals(d.decode(u(0, 0xd8, 0xd8)), '\uFFFD') // single character
}, 'utf-16le does not produce more chars than truncated')
test(() => {
const d = new TextDecoder('utf-16be')
assert_equals(d.decode(u(0, 0, 0)), '\0\uFFFD') // two character, 0 was valid
assert_equals(d.decode(u(0, 42, 0)), '*\uFFFD') // two characters, * was valid
assert_equals(d.decode(u(0xd8, 0, 0)), '\uFFFD') // single character
assert_equals(d.decode(u(0xd8, 0, 0xd8)), '\uFFFD') // single character
}, 'utf-16be does not produce more chars than truncated')
// Node.js fails on this
{
const m = '€\x81‚ƒ„…†‡ˆ‰Š‹Œ\x8DŽ\x8F\x90‘’“”•–—˜™š›œ\x9DžŸ'
for (const encoding of ['windows-1252', 'latin1', 'ascii']) {
test(() => {
for (const fatal of [false, true]) {
const d = new TextDecoder(encoding, { fatal })
assert_equals(d.encoding, 'windows-1252')
for (let i = 0; i < m.length; i++) assert_equals(d.decode(u(128 + i)), m[i])
}
}, `windows-1252 maps bytes outside of latin1: ${encoding}`)
}
}
// iconv and whatwg-encoding fails on this
for (const encoding of ['windows-1252', 'latin1', 'ascii']) {
test(() => {
for (const fatal of [false, true]) {
const d = new TextDecoder(encoding, { fatal })
assert_equals(d.encoding, 'windows-1252')
for (const byte of [0x81, 0x8d, 0x8f, 0x90, 0x9d]) {
assert_equals(d.decode(u(byte)), String.fromCharCode(byte))
}
}
}, `windows-1252 does not contain unmapped chars: ${encoding}`)
}
// Node.js misses an implementation
test(() => {
const encoding = 'x-user-defined'
const loose = new TextDecoder(encoding)
const fatal = new TextDecoder(encoding, { fatal: true })
for (let byte = 0; byte < 256; byte++) {
const str = String.fromCodePoint(byte >= 0x80 ? 0xf7_80 + byte - 0x80 : byte)
assert_equals(fatal.decode(Uint8Array.of(byte)), str, byte)
assert_equals(loose.decode(Uint8Array.of(byte)), str, byte)
}
}, 'specific: x-user-defined')
test(() => {
const loose = new TextDecoder('big5')
const fatal = new TextDecoder('big5', { fatal: true })
// Node.js fails on this
assert_equals(loose.decode(u(0x80)), '\uFFFD')
assert_throws_js(TypeError, () => fatal.decode(u(0x80)))
}, 'specific: big5')
test(() => {
const i8 = new TextDecoder('iso-8859-8')
const i8i = new TextDecoder('iso-8859-8-i')
for (let i = 0; i < 256; i++) {
assert_equals(i8.decode(u(i)), i8i.decode(u(i)), `Byte: ${i}`)
}
}, 'iso-8859-8-i decodes bytes the same way as iso-8859-8')
{
const r = 0xff_fd
const fixtures = {
// Node.js fails these (iconv-lite / whatwg-encoding also fails some)
'koi8-u': { 174: 1118, 190: 1038 },
'windows-874': { 129: 129, 219: r, 220: r, 221: r, 222: r, 252: r, 253: r, 254: r, 255: r },
'windows-1252': { 128: 8364, 129: 129, 130: 8218, 131: 402, 141: 141, 158: 382, 159: 376 },
'windows-1253': { 129: 129, 136: 136, 159: 159, 170: r },
'windows-1255': { 129: 129, 138: 138, 159: 159, 202: 1466 },
// iconv-lite / whatwg-encoding fails these
macintosh: { 189: 937, 219: 8364, 240: 63_743 },
'windows-1250': { 129: 129, 131: 131, 136: 136, 144: 144, 152: 152 },
'windows-1251': { 152: 152 },
'windows-1254': { 129: 129, 140: 338, 141: 141, 144: 144, 157: 157, 158: 158, 222: 350 },
'windows-1257': { 129: 129, 131: 131, 138: 138, 145: 8216, 159: 159, 208: 352, 255: 729 },
'windows-1258': { 129: 129, 138: 138, 141: 141, 158: 158, 159: 376, 208: 272, 255: 255 },
// Some impls miss some encodings
'iso-8859-8-i': { 160: 160, 161: r, 162: 162, 222: r, 223: 8215, 254: 8207, 255: r },
'iso-8859-16': { 128: 128, 160: 160, 161: 260, 252: 252, 253: 281, 254: 539, 255: 255 },
'x-mac-cyrillic': { 128: 1040, 214: 247, 254: 1102, 255: 8364 },
}
for (const [encoding, map] of Object.entries(fixtures)) {
test(() => {
const fatal = new TextDecoder(encoding, { fatal: true })
const loose = new TextDecoder(encoding)
for (const [offset, codepoint] of Object.entries(map)) {
const u8 = Uint8Array.of(Number(offset))
const str = String.fromCodePoint(codepoint)
assert_equals(loose.decode(u8), str, `${offset} -> ${codepoint}`)
if (codepoint === r) {
assert_throws_js(TypeError, () => fatal.decode(u8))
} else {
assert_equals(fatal.decode(u8), str, `${offset} -> ${codepoint} (fatal)`)
}
}
}, `selected single-byte: ${encoding}`)
}
}
// Chrome and WebKit fail at this, Firefox passes
// This one might be tricky to get into WPT, as two major impls ignore spec here
test(() => {
const loose = new TextDecoder('iso-2022-jp')
// Roman, example from spec
{
const fatal = new TextDecoder('iso-2022-jp', { fatal: true }) // Fresh instance because of sticky state, which is a separate test
const a = u(0x1b, 0x28, 0x4a, 0x5c, 0x1b, 0x28, 0x42) // switch to Roman, select char, switch to ascii
assert_equals(fatal.decode(a), '\xA5')
assert_equals(loose.decode(a), '\xA5')
assert_throws_js(TypeError, () => fatal.decode(u(...a, ...a)))
assert_equals(loose.decode(u(...a, ...a)), '\xA5\uFFFD\xA5')
}
// jis
{
const fatal = new TextDecoder('iso-2022-jp', { fatal: true }) // Fresh instance because of sticky state, which is a separate test
const a = u(0x1b, 0x24, 0x42, 0x30, 0x30, 0x1b, 0x28, 0x42) // switch to jis, select char, switch to ascii
assert_equals(fatal.decode(a), '\u65ED')
assert_equals(loose.decode(a), '\u65ED')
assert_throws_js(TypeError, () => fatal.decode(u(...a, ...a)))
assert_equals(loose.decode(u(...a, ...a)), '\u65ED\uFFFD\u65ED')
}
}, 'Concatenating two ISO-2022-JP outputs is not always valid')
for (const encoding of ['gb18030', 'gbk']) {
test(() => {
const loose = new TextDecoder(encoding)
const checkAll = (...list) => list.forEach((x) => check(...x))
const check = (bytes, str, invalid = false) => {
// Firefox also breaks if this is reused, due to state - there is a separate test for that
const fatal = new TextDecoder(encoding, { fatal: true })
const u8 = Uint8Array.from(bytes)
assert_equals(loose.decode(u8), str)
if (!invalid) assert_equals(fatal.decode(u8), str)
if (invalid) assert_throws_js(TypeError, () => fatal.decode(u8))
}
// Pointer ranges
check([0x84, 0x31, 0xa4, 0x36], '\uFFFC') // pointer 39416
check([0x84, 0x31, 0xa4, 0x37], '\uFFFD') // pointer 39417, valid representation for the replacement char
check([0x84, 0x31, 0xa4, 0x38], '\uFFFE') // pointer 39418
check([0x84, 0x31, 0xa4, 0x39], '\uFFFF') // pointer 39419
check([0x84, 0x31, 0xa5, 0x30], '\uFFFD', true) // invalid pointer 39420
check([0x8f, 0x39, 0xfe, 0x39], '\uFFFD', true) // invalid pointer 188999
check([0x90, 0x30, 0x81, 0x30], String.fromCodePoint(0x1_00_00)) // pointer 189000
check([0x90, 0x30, 0x81, 0x31], String.fromCodePoint(0x1_00_01)) // pointer 189001
// Max codepoint
check([0xe3, 0x32, 0x9a, 0x35], String.fromCodePoint(0x10_ff_ff)) // pointer 1237575
check([0xe3, 0x32, 0x9a, 0x36], '\uFFFD', true)
check([0xe3, 0x32, 0x9a, 0x37], '\uFFFD', true)
// Max bytes
check([0xfe, 0x39, 0xfe, 0x39], '\uFFFD', true)
check([0xff, 0x39, 0xfe, 0x39], '\uFFFD9\uFFFD', true)
check([0xfe, 0x40, 0xfe, 0x39], '\uFA0C\uFFFD', true)
check([0xfe, 0x39, 0xff, 0x39], '\uFFFD9\uFFFD9', true)
check([0xfe, 0x39, 0xfe, 0x40], '\uFFFD9\uFA0C', true)
checkAll([[0xa8, 0xbb], '\u0251'], [[0xa8, 0xbc], '\u1E3F'], [[0xa8, 0xbd], '\u0144'])
check([0x81, 0x35, 0xf4, 0x36], '\u1E3E') // ajascent
check([0x81, 0x35, 0xf4, 0x37], '\uE7C7')
check([0x81, 0x35, 0xf4, 0x38], '\u1E40') // ajascent
checkAll([[0xa6, 0xd9], '\uFE10'], [[0xa6, 0xed], '\uFE18'], [[0xa6, 0xf3], '\uFE19']) // assymetric
checkAll([[0xfe, 0x59], '\u9FB4'], [[0xfe, 0xa0], '\u9FBB']) // assymetric
}, `${encoding} version and ranges`)
}
// Node.js has this wrong
test(() => {
const gbk = new TextDecoder('gbk')
const gb18030 = new TextDecoder('gb18030')
const check = (...list) => {
for (const bytes of list) {
const u8 = Uint8Array.from(bytes)
assert_equals(gbk.decode(u8), gb18030.decode(u8), bytes)
}
}
check([0, 255], [128, 255], [129, 48], [129, 255], [254, 48], [254, 255], [255, 0], [255, 255])
}, 'gbk decoder is gb18030 decoder')
{
const vectors = {
big5: [
[[0x80], '\uFFFD'], // Node.js fails
[[0x81, 0x40], '\uFFFD@'], // WebKit fails: https://bugs.webkit.org/show_bug.cgi?id=304238. Chrome and Firefox are correct. Node.js fails (see below)
[[0x83, 0x5c], '\uFFFD\x5C'], // Node.js fails: https://github.com/nodejs/node/issues/40091. Chrome and Firefox are correct. WebKit fails (see above)
[[0x87, 0x87, 0x40], '\uFFFD@'], // Chrome fails: https://issues.chromium.org/issues/467727340. Firefox and WebKit are correct. iconv/whatwg-encoding fails
[[0x81, 0x81], '\uFFFD'], // Chrome fails: https://issues.chromium.org/issues/467727340. Firefox and WebKit are correct. iconv/whatwg-encoding fails
],
'iso-2022-jp': [
[[0x1b, 0x24], '\uFFFD$'], // Node.js fails on this. Chrome, Firefox and Safari are correct
[[0x1b, 0x24, 0x40, 0x1b, 0x24], '\uFFFD\uFFFD'], // Last 0x24 is invalid on both attemtps. Chrome, WebKit, text-encoding fail on this. Firefox, Deno, Servo are correct
],
gb18030: [
[[0xa0, 0x30, 0x2b], '\uFFFD0+'],
[[0x81, 0x31], '\uFFFD'], // iconv / whatwg-encoding fails
],
'euc-jp': [
[[0x80], '\uFFFD'], // Node.js fails
[[0x8d, 0x8d], '\uFFFD\uFFFD'], // coherence
[[0x8e, 0x8e], '\uFFFD'], // iconv / whatwg-encoding, text-encoding fail
],
'euc-kr': [
[[0x80], '\uFFFD'], // Node.js fails
[[0xad, 0xad], '\uFFFD'], // iconv / whatwg-encoding fails
[[0x41, 0xc7, 0x41], 'A\uFFFDA'], // text-encoding fails. Chrome, Firefox, Webkit are correct
],
shift_jis: [
[[0x85, 0x85], '\uFFFD'], // iconv / whatwg-encoding fails
],
}
vectors.gbk = vectors.gb18030
for (const [encoding, list] of Object.entries(vectors)) {
test(() => {
for (const fatal of [false, true]) {
for (const [bytes, text] of list) {
const d = new TextDecoder(encoding, { fatal })
if (fatal) {
assert_throws_js(TypeError, () => d.decode(Uint8Array.from(bytes)))
} else {
assert_equals(d.decode(Uint8Array.from(bytes)), text)
}
}
}
}, `Replacement, push back ASCII characters: ${encoding}`)
}
}
// Chrome, Firefox and Safari all fail on this
{
const vectors = {
'iso-2022-jp': [
[[27], '\uFFFD'], // In Safari, attempting to decode this in fatal mode fails also the _next_ decode() call with valid data
[[27, 0x28], '\uFFFD('], // Fails in Chrome
[[0x1b, 0x28, 0x49], ''],
],
gb18030: [
[[0xfe], '\uFFFD'],
[[0xfe, 0x39], '\uFFFD'],
[[0xfe, 0x39, 0xfe], '\uFFFD'],
[[0xfe, 0x39, 0xfe, 0x39], '\uFFFD'],
[[0xff], '\uFFFD'],
[[0xfe, 0xff], '\uFFFD'],
[[0xfe, 0x39, 0xff], '\uFFFD9\uFFFD'],
[[0xfe, 0x39, 0xfe, 0x40], '\uFFFD9\uFA0C'],
[[0x81], '\uFFFD'],
[[0x81, 0x3a], '\uFFFD:'],
[[0x81, 0x3a, 0x81], '\uFFFD:\uFFFD'],
],
big5: [
[[0x87, 0x3a], '\uFFFD:'],
[[0x87, 0x3a, 0x87], '\uFFFD:\uFFFD'],
],
shift_jis: [
[[0x81, 0x3a], '\uFFFD:'],
[[0x81, 0x3a, 0x81], '\uFFFD:\uFFFD'],
],
'euc-kr': [
[[0x81, 0x3a], '\uFFFD:'],
[[0x81, 0x3a, 0x81], '\uFFFD:\uFFFD'],
],
}
vectors.gbk = vectors.gb18030
for (const [encoding, list] of Object.entries(vectors)) {
test(() => {
for (const fatal of [false, true]) {
for (const [bytes, text] of list) {
const d = new TextDecoder(encoding, { fatal })
assert_equals(d.decode(u(0x40)), '@') // ascii
if (fatal && text.includes('\uFFFD')) {
assert_throws_js(TypeError, () => d.decode(Uint8Array.from(bytes)))
} else {
assert_equals(d.decode(Uint8Array.from(bytes)), text)
}
// ascii
assert_equals(d.decode(u(0x40)), '@') // Check that previous decode() call did not affect the next one
assert_equals(d.decode(u(0x2a)), '*') // Or the next one
assert_equals(d.decode(u(0x42)), 'B') // Or the next one (this fails in Safari too)
}
}
}, `Sticky multibyte state: ${encoding}`)
}
test(() => {
for (const fatal of [false, true]) {
// Fails in Safari
const d = new TextDecoder('euc-jp', { fatal })
assert_equals(d.decode(Uint8Array.of(0xa1, 0xa1)), '\u3000')
if (fatal) {
assert_throws_js(TypeError, () => d.decode(Uint8Array.of(0x8f, 0xa1)))
} else {
assert_equals(d.decode(Uint8Array.of(0x8f, 0xa1)), '\uFFFD')
}
assert_equals(d.decode(Uint8Array.of(0xa1, 0xa1)), '\u3000')
}
}, 'Sticky multibyte state: euc-jp')
}
// Firefox fails on this
test(() => {
const d = new TextDecoder('utf-8', { fatal: true })
assert_throws_js(TypeError, () => d.decode(u(0xff)))
assert_equals(d.decode(u(0xef, 0xbb, 0xbf)), '')
assert_throws_js(TypeError, () => d.decode(u(0xff)))
assert_equals(d.decode(u(0xef, 0xbb, 0xbf, 0x40)), '@')
assert_throws_js(TypeError, () => d.decode(u(0xff)))
assert_equals(d.decode(u(0xef, 0xbb, 0xbf, 0xef, 0xbb, 0xbf)), '\uFEFF')
}, 'Sticky fatal BOM: utf-8')
test(() => {
const d = new TextDecoder('utf-16le', { fatal: true })
assert_throws_js(TypeError, () => d.decode(u(0xff)))
assert_equals(d.decode(u(0xff, 0xfe)), '')
assert_throws_js(TypeError, () => d.decode(u(0xff)))
assert_equals(d.decode(u(0xff, 0xfe, 0x40, 0x00)), '@')
assert_throws_js(TypeError, () => d.decode(u(0xff)))
assert_equals(d.decode(u(0xff, 0xfe, 0xff, 0xfe)), '\uFEFF')
}, 'Sticky fatal BOM: utf-16le')
test(() => {
const d = new TextDecoder('utf-16be', { fatal: true })
assert_throws_js(TypeError, () => d.decode(u(0xff)))
assert_equals(d.decode(u(0xfe, 0xff)), '')
assert_throws_js(TypeError, () => d.decode(u(0xff)))
assert_equals(d.decode(u(0xfe, 0xff, 0x00, 0x40)), '@')
assert_throws_js(TypeError, () => d.decode(u(0xff)))
assert_equals(d.decode(u(0xfe, 0xff, 0xfe, 0xff)), '\uFEFF')
}, 'Sticky fatal BOM: utf-16be')
// Bun fails at this
test(() => {
const d = new TextDecoder()
const check = (a, opt, str) => assert_equals(d.decode(Uint8Array.from(a), opt), str)
check([0x01, 0x02], { stream: true }, '\x01\x02')
check([0x03], {}, '\x03') // close
check([0xef, 0xbb], { stream: true }, '')
check([0xbf], { stream: true }, '')
check([0xef, 0xbb], { stream: true }, '')
check([0xbf], { stream: true }, '\uFEFF')
check([0x42], {}, 'B') // close
check([0xef], { stream: true }, '')
check([0xbb], { stream: true }, '')
check([0xbf], { stream: true }, '')
check([0xef], { stream: true }, '')
check([0xbb], { stream: true }, '')
check([0xbf], { stream: true }, '\uFEFF')
check([0xef, 0xbb, 0xbf], { stream: true }, '\uFEFF')
check([0x41], {}, 'A') // close
check([], { stream: true }, '')
check([0xef, 0xbb], { stream: true }, '')
check([0xbf, 0x43], {}, 'C') // close
check([0xef], { stream: true }, '')
check([0xbb, 0xbf, 42, 43], {}, '*+') // close
{
const d = new TextDecoder()
assert_equals(d.decode(u(0, 0), { stream: true }), '\0\0')
assert_equals(d.decode(u(0)), '\0')
assert_equals(d.decode(u(0xef, 0xbb), { stream: true }), '') // empty string
assert_equals(d.decode(u(0xbf)), '') // empty string
}
}, 'BOM splitting / repeats: utf-8')
// Bun fails at this
test(() => {
const d = new TextDecoder('utf-16le')
const check = (a, opt, str) => assert_equals(d.decode(Uint8Array.from(a), opt), str)
check([0xff, 0xfe], { stream: true }, '')
check([0x03, 0x00], {}, '\x03') // close
check([0xff], { stream: true }, '')
check([0xfe, 0x03, 0x00], {}, '\x03') // close
check([0xff, 0xfe], { stream: true }, '')
check([0xff, 0xfe, 0x03, 0x00], {}, '\uFEFF\x03') // close
check([0xff], { stream: true }, '')
check([0xfe, 0xff, 0xfe, 0x03, 0x00], {}, '\uFEFF\x03') // close
}, 'BOM splitting / repeats: utf-16le')
// Bun fails at this
test(() => {
const d = new TextDecoder('utf-16be')
const check = (a, opt, str) => assert_equals(d.decode(Uint8Array.from(a), opt), str)
check([0xfe, 0xff], { stream: true }, '')
check([0x00, 0x03], {}, '\x03') // close
check([0xfe], { stream: true }, '')
check([0xff, 0x00, 0x03], {}, '\x03') // close
check([0xfe, 0xff], { stream: true }, '')
check([0xfe, 0xff, 0x00, 0x03], {}, '\uFEFF\x03') // close
check([0xfe], { stream: true }, '')
check([0xff, 0xfe, 0xff, 0x00, 0x03], {}, '\uFEFF\x03') // close
}, 'BOM splitting / repeats: utf-16be')
{
// Chrome is incorrect. It also decodes fetch() responses wrong for utf-8
test(() => {
const u8 = Uint8Array.of(0xf0, 0xc3, 0x80, 42, 42)
const str = new TextDecoder().decode(u8)
assert_equals(str, '\uFFFD\xC0**')
const d = new TextDecoder()
const chunks = [
d.decode(u8.subarray(0, 1), { stream: true }),
d.decode(u8.subarray(1), { stream: true }),
d.decode(),
]
assert_equals(chunks.join(''), str)
const r = '\uFFFD'
assert_equals(new TextDecoder().decode(u(0xc0), { stream: true }), r)
assert_equals(new TextDecoder().decode(u(0xff), { stream: true }), r)
assert_equals(new TextDecoder().decode(u(0xed, 0xbf), { stream: true }), `${r}${r}`)
}, 'stream: utf-8')
const vectors = {
gbk: [
[[0x81, 0x82], '\u4E97'], // valid
[[0xa0, 0x30, 0x2b], '\uFFFD0+'], // replacement
],
gb18030: [
[[0x81, 0x82], '\u4E97'], // valid
[[0xa0, 0x30, 0x2b], '\uFFFD0+'], // replacement
],
big5: [[[0xfe, 0x40], '\u9442']],
shift_jis: [[[0x81, 0x87], '\u221E']],
'euc-kr': [[[0x81, 0x41], '\uAC02']],
'euc-jp': [[[0xb0, 0xb0], '\u65ED']],
'iso-2022-jp': [[[0x2a, 0x1b], '*\uFFFD']],
}
for (const [encoding, list] of Object.entries(vectors)) {
test(() => {
for (const [bytes, expected] of list) {
const u8 = Uint8Array.from(bytes)
const str = new TextDecoder(encoding).decode(u8)
assert_equals(str, expected)
// Bun is incorrect
{
const d = new TextDecoder(encoding)
const chunks = [d.decode(u8.subarray(0, 1), { stream: true }), d.decode(u8.subarray(1))]
assert_equals(chunks.join(''), str)
}
// Bun is incorrect
{
const d = new TextDecoder(encoding)
const chunks = [
d.decode(u8.subarray(0, 1), { stream: true }),
d.decode(u8.subarray(1), { stream: true }),
d.decode(),
]
assert_equals(chunks.join(''), str)
}
// Deno, Servo and Bun are incorrect on big5, shift_jis, euc-kr
{
const d = new TextDecoder(encoding)
const chunks = [
d.decode(u8.subarray(0, 1), { stream: true }),
d.decode(Uint8Array.of(), { stream: true }),
d.decode(u8.subarray(1), { stream: true }),
d.decode(),
]
assert_equals(chunks.join(''), str)
}
// Node.js throws an "data was not valid for encoding" error in replacement mode
{
const d = new TextDecoder(encoding)
const chunks = [
d.decode(u8.subarray(0, 2), { stream: true }),
d.decode(u8.subarray(2), { stream: true }),
d.decode(),
]
assert_equals(chunks.join(''), str)
}
}
}, `stream: ${encoding}`)
}
}
test(() => {
{
const d = new TextDecoder('utf-8', { fatal: true })
assert_throws_js(TypeError, () => d.decode(u(0xc0), { stream: true }))
assert_throws_js(TypeError, () => d.decode(u(0xff), { stream: true }))
assert_equals(d.decode(), '')
}
{
const loose = new TextDecoder('utf-8')
assert_equals(loose.decode(u(0xfd, 0xef), { stream: true }), '\uFFFD')
assert_equals(loose.decode(), '\uFFFD')
const fatal = new TextDecoder('utf-8', { fatal: true })
assert_throws_js(TypeError, () => fatal.decode(u(0xfd, 0xef), { stream: true }))
assert_equals(fatal.decode(), '')
}
}, 'fatal stream: utf-8')
for (const encoding of ['utf-16le', 'utf-16be']) {
test(() => {
const d = new TextDecoder(encoding, { fatal: true })
assert_equals(d.decode(u(0x00), { stream: true }), '')
assert_throws_js(TypeError, () => d.decode())
assert_equals(d.decode(), '')
}, `fatal stream: ${encoding}`)
}
// Bun is incorrect
test(() => {
// This is the only decoder which does not clear internal state before throwing in stream mode (non-EOF throws)
// So the internal state of this decoder can legitimately persist after an error was thrown
{
const d = new TextDecoder('iso-2022-jp', { fatal: true })
assert_equals(d.decode(Uint8Array.of(0x7e)), '\x7E')
assert_throws_js(TypeError, () => d.decode(u(0x1b, 0x28, 0x4a, 0xff), { stream: true })) // Switch to Roman, error
assert_equals(d.decode(Uint8Array.of(0x7e)), '\u203E')
}
{
const d = new TextDecoder('iso-2022-jp', { fatal: true })
assert_equals(d.decode(Uint8Array.of(0x42)), 'B')
assert_throws_js(TypeError, () => d.decode(u(0x1b, 0x28, 0x49, 0xff), { stream: true })) // Switch to Katakana, error
assert_equals(d.decode(Uint8Array.of(0x42)), '\uFF82')
}
}, 'fatal stream: iso-2022-jp')
// These are mislabeled in WPT html dataset files, their recorded codepoints do not match actual ones
// All browsers (and the script) agree on how these are decoded though, but let's explicitly recheck
{
const vectors = {
'euc-jp': [
[[0x5c], '\x5C'], // Not U+A5
[[0x7e], '\x7E'], // Not U+203E
[[0xa1, 0xdd], '\uFF0D'], // Not U+2212
],
shift_jis: [
[[0x5c], '\x5C'], // Not U+A5
[[0x7e], '\x7E'], // Not U+203E
[[0x81, 0x7c], '\uFF0D'], // Not U+2212
],
'iso-2022-jp': [
[[0x1b, 0x28, 0x4a, 0x5c, 0x1b, 0x28, 0x42], '\xA5'], // Correctly labeled, U+A5
[[0x1b, 0x28, 0x4a, 0x7e, 0x1b, 0x28, 0x42], '\u203E'], // Correctly labeled, U+203E
[[0x1b, 0x24, 0x42, 0x21, 0x5d, 0x1b, 0x28, 0x42], '\uFF0D'], // Not U+2212
],
}
for (const [encoding, list] of Object.entries(vectors)) {
test(() => {
for (const fatal of [false, true]) {
for (const [bytes, string] of list) {
const d = new TextDecoder(encoding, { fatal })
assert_equals(d.decode(Uint8Array.from(bytes)), string)
}
}
}, `WPT mislabels: ${encoding}`)
}
}
// Node.js fails on this
test(() => {
const bad = ['\u212Aoi8-r', '\u212Aoi8-u', 'euc-\u212Ar']
for (const label of bad) assert_throws_js(RangeError, () => new TextDecoder(label))
}, 'labels: invalid non-ascii')
test(() => {
assert_equals(new TextDecoder('UTF-8').encoding, 'utf-8')
assert_equals(new TextDecoder('UTF-8'.toLowerCase()).encoding, 'utf-8') // Do not remove .toLowerCase() from test
}, 'labels: transformed')