Source code

Revision control

Other Tools

1
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2
/* vim: set sw=2 ts=2 et tw=80: */
3
/* This Source Code Form is subject to the terms of the Mozilla Public
4
* License, v. 2.0. If a copy of the MPL was not distributed with this
5
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7
#include "nsHtml5StreamParser.h"
8
9
#include "mozilla/DebugOnly.h"
10
#include "mozilla/Encoding.h"
11
#include "nsContentUtils.h"
12
#include "nsCyrillicDetector.h"
13
#include "nsHtml5Tokenizer.h"
14
#include "nsIHttpChannel.h"
15
#include "nsHtml5Parser.h"
16
#include "nsHtml5TreeBuilder.h"
17
#include "nsHtml5AtomTable.h"
18
#include "nsHtml5Module.h"
19
#include "nsHtml5StreamParserPtr.h"
20
#include "nsIDocShell.h"
21
#include "nsIScriptError.h"
22
#include "mozilla/Preferences.h"
23
#include "mozilla/StaticPrefs.h"
24
#include "mozilla/SystemGroup.h"
25
#include "mozilla/UniquePtrExtensions.h"
26
#include "nsHtml5Highlighter.h"
27
#include "expat_config.h"
28
#include "expat.h"
29
#include "nsINestedURI.h"
30
#include "nsCharsetSource.h"
31
#include "nsIThreadRetargetableRequest.h"
32
#include "nsPrintfCString.h"
33
#include "nsNetUtil.h"
34
#include "nsXULAppAPI.h"
35
#include "mozilla/SchedulerGroup.h"
36
#include "nsJSEnvironment.h"
37
#include "mozilla/dom/Document.h"
38
39
using namespace mozilla;
40
using namespace mozilla::dom;
41
42
/*
43
* Note that nsHtml5StreamParser implements cycle collecting AddRef and
44
* Release. Therefore, nsHtml5StreamParser must never be refcounted from
45
* the parser thread!
46
*
47
* To work around this limitation, runnables posted by the main thread to the
48
* parser thread hold their reference to the stream parser in an
49
* nsHtml5StreamParserPtr. Upon creation, nsHtml5StreamParserPtr addrefs the
50
* object it holds
51
* just like a regular nsRefPtr. This is OK, since the creation of the
52
* runnable and the nsHtml5StreamParserPtr happens on the main thread.
53
*
54
* When the runnable is done on the parser thread, the destructor of
55
* nsHtml5StreamParserPtr runs there. It doesn't call Release on the held object
56
* directly. Instead, it posts another runnable back to the main thread where
57
* that runnable calls Release on the wrapped object.
58
*
59
* When posting runnables in the other direction, the runnables have to be
60
* created on the main thread when nsHtml5StreamParser is instantiated and
61
* held for the lifetime of the nsHtml5StreamParser. This works, because the
62
* same runnabled can be dispatched multiple times and currently runnables
63
* posted from the parser thread to main thread don't need to wrap any
64
* runnable-specific data. (In the other direction, the runnables most notably
65
* wrap the byte data of the stream.)
66
*/
67
NS_IMPL_CYCLE_COLLECTING_ADDREF(nsHtml5StreamParser)
68
NS_IMPL_CYCLE_COLLECTING_RELEASE(nsHtml5StreamParser)
69
70
NS_INTERFACE_TABLE_HEAD(nsHtml5StreamParser)
71
NS_INTERFACE_TABLE(nsHtml5StreamParser, nsICharsetDetectionObserver)
72
NS_INTERFACE_TABLE_TO_MAP_SEGUE_CYCLE_COLLECTION(nsHtml5StreamParser)
73
NS_INTERFACE_MAP_END
74
75
NS_IMPL_CYCLE_COLLECTION_CLASS(nsHtml5StreamParser)
76
77
NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(nsHtml5StreamParser)
78
tmp->DropTimer();
79
NS_IMPL_CYCLE_COLLECTION_UNLINK(mObserver)
80
NS_IMPL_CYCLE_COLLECTION_UNLINK(mRequest)
81
NS_IMPL_CYCLE_COLLECTION_UNLINK(mOwner)
82
tmp->mExecutorFlusher = nullptr;
83
tmp->mLoadFlusher = nullptr;
84
tmp->mExecutor = nullptr;
85
NS_IMPL_CYCLE_COLLECTION_UNLINK(mChardet)
86
NS_IMPL_CYCLE_COLLECTION_UNLINK_END
87
88
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(nsHtml5StreamParser)
89
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mObserver)
90
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mRequest)
91
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOwner)
92
// hack: count the strongly owned edge wrapped in the runnable
93
if (tmp->mExecutorFlusher) {
94
NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(cb, "mExecutorFlusher->mExecutor");
95
cb.NoteXPCOMChild(static_cast<nsIContentSink*>(tmp->mExecutor));
96
}
97
// hack: count the strongly owned edge wrapped in the runnable
98
if (tmp->mLoadFlusher) {
99
NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(cb, "mLoadFlusher->mExecutor");
100
cb.NoteXPCOMChild(static_cast<nsIContentSink*>(tmp->mExecutor));
101
}
102
// hack: count self if held by mChardet
103
if (tmp->mChardet) {
104
NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(cb, "mChardet->mObserver");
105
cb.NoteXPCOMChild(static_cast<nsICharsetDetectionObserver*>(tmp));
106
}
107
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
108
109
class nsHtml5ExecutorFlusher : public Runnable {
110
private:
111
RefPtr<nsHtml5TreeOpExecutor> mExecutor;
112
113
public:
114
explicit nsHtml5ExecutorFlusher(nsHtml5TreeOpExecutor* aExecutor)
115
: Runnable("nsHtml5ExecutorFlusher"), mExecutor(aExecutor) {}
116
NS_IMETHOD Run() override {
117
if (!mExecutor->isInList()) {
118
Document* doc = mExecutor->GetDocument();
119
if (XRE_IsContentProcess() &&
120
nsContentUtils::
121
HighPriorityEventPendingForTopLevelDocumentBeforeContentfulPaint(
122
doc)) {
123
// Possible early paint pending, reuse the runnable and try to
124
// call RunFlushLoop later.
125
nsCOMPtr<nsIRunnable> flusher = this;
126
if (NS_SUCCEEDED(
127
doc->Dispatch(TaskCategory::Network, flusher.forget()))) {
128
PROFILER_ADD_MARKER("HighPrio blocking parser flushing(1)", DOM);
129
return NS_OK;
130
}
131
}
132
mExecutor->RunFlushLoop();
133
}
134
return NS_OK;
135
}
136
};
137
138
class nsHtml5LoadFlusher : public Runnable {
139
private:
140
RefPtr<nsHtml5TreeOpExecutor> mExecutor;
141
142
public:
143
explicit nsHtml5LoadFlusher(nsHtml5TreeOpExecutor* aExecutor)
144
: Runnable("nsHtml5LoadFlusher"), mExecutor(aExecutor) {}
145
NS_IMETHOD Run() override {
146
mExecutor->FlushSpeculativeLoads();
147
return NS_OK;
148
}
149
};
150
151
nsHtml5StreamParser::nsHtml5StreamParser(nsHtml5TreeOpExecutor* aExecutor,
152
nsHtml5Parser* aOwner,
153
eParserMode aMode)
154
: mSniffingLength(0),
155
mBomState(eBomState::BOM_SNIFFING_NOT_STARTED),
156
mCharsetSource(kCharsetUninitialized),
157
mEncoding(WINDOWS_1252_ENCODING),
158
mFeedChardet(true),
159
mReparseForbidden(false),
160
mLastBuffer(nullptr), // Will be filled when starting
161
mExecutor(aExecutor),
162
mTreeBuilder(new nsHtml5TreeBuilder(
163
(aMode == VIEW_SOURCE_HTML || aMode == VIEW_SOURCE_XML)
164
? nullptr
165
: mExecutor->GetStage(),
166
aMode == NORMAL ? mExecutor->GetStage() : nullptr)),
167
mTokenizer(new nsHtml5Tokenizer(mTreeBuilder, aMode == VIEW_SOURCE_XML)),
168
mTokenizerMutex("nsHtml5StreamParser mTokenizerMutex"),
169
mOwner(aOwner),
170
mLastWasCR(false),
171
mStreamState(eHtml5StreamState::STREAM_NOT_STARTED),
172
mSpeculating(false),
173
mAtEOF(false),
174
mSpeculationMutex("nsHtml5StreamParser mSpeculationMutex"),
175
mSpeculationFailureCount(0),
176
mLocalFileBytesBuffered(0),
177
mTerminated(false),
178
mInterrupted(false),
179
mTerminatedMutex("nsHtml5StreamParser mTerminatedMutex"),
180
mEventTarget(nsHtml5Module::GetStreamParserThread()->SerialEventTarget()),
181
mExecutorFlusher(new nsHtml5ExecutorFlusher(aExecutor)),
182
mLoadFlusher(new nsHtml5LoadFlusher(aExecutor)),
183
mJapaneseDetector(mozilla::JapaneseDetector::Create(
184
StaticPrefs::intl_charset_detector_iso2022jp_allowed())),
185
mInitialEncodingWasFromParentFrame(false),
186
mHasHadErrors(false),
187
mDecodingLocalFileAsUTF8(false),
188
mFlushTimer(NS_NewTimer(mEventTarget)),
189
mFlushTimerMutex("nsHtml5StreamParser mFlushTimerMutex"),
190
mFlushTimerArmed(false),
191
mFlushTimerEverFired(false),
192
mMode(aMode) {
193
NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
194
#ifdef DEBUG
195
mAtomTable.SetPermittedLookupEventTarget(mEventTarget);
196
#endif
197
mTokenizer->setInterner(&mAtomTable);
198
mTokenizer->setEncodingDeclarationHandler(this);
199
200
if (aMode == VIEW_SOURCE_HTML || aMode == VIEW_SOURCE_XML) {
201
nsHtml5Highlighter* highlighter =
202
new nsHtml5Highlighter(mExecutor->GetStage());
203
mTokenizer->EnableViewSource(highlighter); // takes ownership
204
mTreeBuilder->EnableViewSource(highlighter); // doesn't own
205
}
206
207
// Chardet instantiation adapted from File.
208
// Chardet is initialized here even if it turns out to be useless
209
// to make the chardet refcount its observer (nsHtml5StreamParser)
210
// on the main thread.
211
nsAutoCString detectorName;
212
Preferences::GetLocalizedCString("intl.charset.detector", detectorName);
213
if (!detectorName.IsEmpty()) {
214
// We recognize one of the two magic strings for Russian and Ukranian.
215
if (detectorName.EqualsLiteral("ruprob")) {
216
mChardet = new nsRUProbDetector();
217
} else if (detectorName.EqualsLiteral("ukprob")) {
218
mChardet = new nsUKProbDetector();
219
}
220
if (mChardet) {
221
(void)mChardet->Init(this);
222
}
223
}
224
225
// There's a zeroing operator new for everything else
226
}
227
228
nsHtml5StreamParser::~nsHtml5StreamParser() {
229
NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
230
mTokenizer->end();
231
if (recordreplay::IsRecordingOrReplaying()) {
232
recordreplay::EndContentParse(this);
233
}
234
#ifdef DEBUG
235
{
236
mozilla::MutexAutoLock flushTimerLock(mFlushTimerMutex);
237
MOZ_ASSERT(!mFlushTimer, "Flush timer was not dropped before dtor!");
238
}
239
mRequest = nullptr;
240
mObserver = nullptr;
241
mUnicodeDecoder = nullptr;
242
mSniffingBuffer = nullptr;
243
mMetaScanner = nullptr;
244
mFirstBuffer = nullptr;
245
mExecutor = nullptr;
246
mTreeBuilder = nullptr;
247
mTokenizer = nullptr;
248
mOwner = nullptr;
249
#endif
250
}
251
252
nsresult nsHtml5StreamParser::GetChannel(nsIChannel** aChannel) {
253
NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
254
return mRequest ? CallQueryInterface(mRequest, aChannel)
255
: NS_ERROR_NOT_AVAILABLE;
256
}
257
258
NS_IMETHODIMP
259
nsHtml5StreamParser::Notify(const char* aCharset, nsDetectionConfident aConf) {
260
NS_ASSERTION(IsParserThread(), "Wrong thread!");
261
if (aConf == eBestAnswer || aConf == eSureAnswer) {
262
mFeedChardet = false; // just in case
263
auto encoding =
264
Encoding::ForLabelNoReplacement(nsDependentCString(aCharset));
265
if (!encoding) {
266
return NS_OK;
267
}
268
if (HasDecoder()) {
269
if (mEncoding == encoding) {
270
MOZ_ASSERT(mCharsetSource < kCharsetFromAutoDetection,
271
"Why are we running chardet at all?");
272
mCharsetSource = kCharsetFromAutoDetection;
273
mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
274
} else {
275
// We've already committed to a decoder. Request a reload from the
276
// docshell.
277
mTreeBuilder->NeedsCharsetSwitchTo(WrapNotNull(encoding),
278
kCharsetFromAutoDetection, 0);
279
FlushTreeOpsAndDisarmTimer();
280
Interrupt();
281
}
282
} else {
283
// Got a confident answer from the sniffing buffer. That code will
284
// take care of setting up the decoder.
285
mEncoding = WrapNotNull(encoding);
286
mCharsetSource = kCharsetFromAutoDetection;
287
mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
288
}
289
}
290
return NS_OK;
291
}
292
293
void nsHtml5StreamParser::FeedJapaneseDetector(Span<const uint8_t> aBuffer,
294
bool aLast) {
295
const Encoding* detected = mJapaneseDetector->Feed(aBuffer, aLast);
296
if (!detected) {
297
return;
298
}
299
mFeedChardet = false;
300
if (mDecodingLocalFileAsUTF8 && detected != ISO_2022_JP_ENCODING) {
301
return;
302
}
303
int32_t source = kCharsetFromAutoDetection;
304
if (mCharsetSource == kCharsetFromParentForced ||
305
mCharsetSource == kCharsetFromUserForced) {
306
source = kCharsetFromUserForcedAutoDetection;
307
}
308
if (detected == mEncoding) {
309
MOZ_ASSERT(mCharsetSource < source, "Why are we running chardet at all?");
310
mCharsetSource = source;
311
mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
312
} else if (HasDecoder()) {
313
// We've already committed to a decoder. Request a reload from the
314
// docshell.
315
mTreeBuilder->NeedsCharsetSwitchTo(WrapNotNull(detected), source, 0);
316
FlushTreeOpsAndDisarmTimer();
317
Interrupt();
318
} else {
319
// Got a confident answer from the sniffing buffer. That code will
320
// take care of setting up the decoder.
321
mEncoding = WrapNotNull(detected);
322
mCharsetSource = source;
323
mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
324
}
325
}
326
327
void nsHtml5StreamParser::FeedDetector(Span<const uint8_t> aBuffer,
328
bool aLast) {
329
if (mEncoding->IsJapaneseLegacy()) {
330
FeedJapaneseDetector(aBuffer, aLast);
331
} else if (mEncoding == WINDOWS_1251_ENCODING && mChardet &&
332
!mDecodingLocalFileAsUTF8) {
333
if (!aBuffer.IsEmpty()) {
334
bool dontFeed = false;
335
mozilla::Unused << mChardet->DoIt((const char*)aBuffer.Elements(),
336
aBuffer.Length(), &dontFeed);
337
if (dontFeed) {
338
mFeedChardet = false;
339
}
340
}
341
if (aLast) {
342
mozilla::Unused << mChardet->Done();
343
}
344
} else {
345
mFeedChardet = false;
346
}
347
}
348
349
void nsHtml5StreamParser::SetViewSourceTitle(nsIURI* aURL) {
350
if (recordreplay::IsRecordingOrReplaying()) {
351
nsAutoCString spec;
352
aURL->GetSpec(spec);
353
recordreplay::BeginContentParse(this, spec.get(), "text/html");
354
}
355
356
if (aURL) {
357
nsCOMPtr<nsIURI> temp;
358
bool isViewSource;
359
aURL->SchemeIs("view-source", &isViewSource);
360
if (isViewSource) {
361
nsCOMPtr<nsINestedURI> nested = do_QueryInterface(aURL);
362
nested->GetInnerURI(getter_AddRefs(temp));
363
} else {
364
temp = aURL;
365
}
366
bool isData;
367
temp->SchemeIs("data", &isData);
368
if (isData) {
369
// Avoid showing potentially huge data: URLs. The three last bytes are
370
// UTF-8 for an ellipsis.
371
mViewSourceTitle.AssignLiteral("data:\xE2\x80\xA6");
372
} else {
373
nsresult rv = temp->GetSpec(mViewSourceTitle);
374
if (NS_FAILED(rv)) {
375
mViewSourceTitle.AssignLiteral("\xE2\x80\xA6");
376
}
377
}
378
}
379
}
380
381
nsresult
382
nsHtml5StreamParser::SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
383
Span<const uint8_t> aFromSegment) {
384
NS_ASSERTION(IsParserThread(), "Wrong thread!");
385
nsresult rv = NS_OK;
386
if (mDecodingLocalFileAsUTF8 && mCharsetSource <= kCharsetFromFileURLGuess) {
387
MOZ_ASSERT(mEncoding != UTF_8_ENCODING);
388
mUnicodeDecoder = UTF_8_ENCODING->NewDecoderWithBOMRemoval();
389
} else {
390
if (mCharsetSource >= kCharsetFromAutoDetection &&
391
!(mCharsetSource == kCharsetFromUserForced ||
392
mCharsetSource == kCharsetFromParentForced)) {
393
mFeedChardet = false;
394
}
395
mDecodingLocalFileAsUTF8 = false;
396
mUnicodeDecoder = mEncoding->NewDecoderWithBOMRemoval();
397
}
398
if (mSniffingBuffer) {
399
rv = WriteStreamBytes(MakeSpan(mSniffingBuffer.get(), mSniffingLength));
400
NS_ENSURE_SUCCESS(rv, rv);
401
mSniffingBuffer = nullptr;
402
}
403
mMetaScanner = nullptr;
404
return WriteStreamBytes(aFromSegment);
405
}
406
407
nsresult nsHtml5StreamParser::SetupDecodingFromBom(
408
NotNull<const Encoding*> aEncoding) {
409
NS_ASSERTION(IsParserThread(), "Wrong thread!");
410
mEncoding = aEncoding;
411
mDecodingLocalFileAsUTF8 = false;
412
mUnicodeDecoder = mEncoding->NewDecoderWithoutBOMHandling();
413
mCharsetSource = kCharsetFromByteOrderMark;
414
mFeedChardet = false;
415
mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
416
mSniffingBuffer = nullptr;
417
mMetaScanner = nullptr;
418
mBomState = BOM_SNIFFING_OVER;
419
return NS_OK;
420
}
421
422
void nsHtml5StreamParser::SniffBOMlessUTF16BasicLatin(
423
Span<const uint8_t> aFromSegment) {
424
// Avoid underspecified heuristic craziness for XHR
425
if (mMode == LOAD_AS_DATA) {
426
return;
427
}
428
// Make sure there's enough data. Require room for "<title></title>"
429
if (mSniffingLength + aFromSegment.Length() < 30) {
430
return;
431
}
432
// even-numbered bytes tracked at 0, odd-numbered bytes tracked at 1
433
bool byteZero[2] = {false, false};
434
bool byteNonZero[2] = {false, false};
435
uint32_t i = 0;
436
if (mSniffingBuffer) {
437
for (; i < mSniffingLength; ++i) {
438
if (mSniffingBuffer[i]) {
439
if (byteNonZero[1 - (i % 2)]) {
440
return;
441
}
442
byteNonZero[i % 2] = true;
443
} else {
444
if (byteZero[1 - (i % 2)]) {
445
return;
446
}
447
byteZero[i % 2] = true;
448
}
449
}
450
}
451
for (size_t j = 0; j < aFromSegment.Length(); ++j) {
452
if (aFromSegment[j]) {
453
if (byteNonZero[1 - ((i + j) % 2)]) {
454
return;
455
}
456
byteNonZero[(i + j) % 2] = true;
457
} else {
458
if (byteZero[1 - ((i + j) % 2)]) {
459
return;
460
}
461
byteZero[(i + j) % 2] = true;
462
}
463
}
464
465
if (byteNonZero[0]) {
466
mEncoding = UTF_16LE_ENCODING;
467
} else {
468
mEncoding = UTF_16BE_ENCODING;
469
}
470
mCharsetSource = kCharsetFromIrreversibleAutoDetection;
471
mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
472
mFeedChardet = false;
473
mTreeBuilder->MaybeComplainAboutCharset("EncBomlessUtf16", true, 0);
474
}
475
476
void nsHtml5StreamParser::SetEncodingFromExpat(const char16_t* aEncoding) {
477
if (aEncoding) {
478
nsDependentString utf16(aEncoding);
479
nsAutoCString utf8;
480
CopyUTF16toUTF8(utf16, utf8);
481
auto encoding = PreferredForInternalEncodingDecl(utf8);
482
if (encoding) {
483
mEncoding = WrapNotNull(encoding);
484
mCharsetSource = kCharsetFromMetaTag; // closest for XML
485
return;
486
}
487
// else the page declared an encoding Gecko doesn't support and we'd
488
// end up defaulting to UTF-8 anyway. Might as well fall through here
489
// right away and let the encoding be set to UTF-8 which we'd default to
490
// anyway.
491
}
492
mEncoding = UTF_8_ENCODING; // XML defaults to UTF-8 without a BOM
493
mCharsetSource = kCharsetFromMetaTag; // means confident
494
}
495
496
// A separate user data struct is used instead of passing the
497
// nsHtml5StreamParser instance as user data in order to avoid including
498
// expat.h in nsHtml5StreamParser.h. Doing that would cause naming conflicts.
499
// Using a separate user data struct also avoids bloating nsHtml5StreamParser
500
// by one pointer.
501
struct UserData {
502
XML_Parser mExpat;
503
nsHtml5StreamParser* mStreamParser;
504
};
505
506
// Using no-namespace handler callbacks to avoid including expat.h in
507
// nsHtml5StreamParser.h, since doing so would cause naming conclicts.
508
static void HandleXMLDeclaration(void* aUserData, const XML_Char* aVersion,
509
const XML_Char* aEncoding, int aStandalone) {
510
UserData* ud = static_cast<UserData*>(aUserData);
511
ud->mStreamParser->SetEncodingFromExpat(
512
reinterpret_cast<const char16_t*>(aEncoding));
513
XML_StopParser(ud->mExpat, false);
514
}
515
516
static void HandleStartElement(void* aUserData, const XML_Char* aName,
517
const XML_Char** aAtts) {
518
UserData* ud = static_cast<UserData*>(aUserData);
519
XML_StopParser(ud->mExpat, false);
520
}
521
522
static void HandleEndElement(void* aUserData, const XML_Char* aName) {
523
UserData* ud = static_cast<UserData*>(aUserData);
524
XML_StopParser(ud->mExpat, false);
525
}
526
527
static void HandleComment(void* aUserData, const XML_Char* aName) {
528
UserData* ud = static_cast<UserData*>(aUserData);
529
XML_StopParser(ud->mExpat, false);
530
}
531
532
static void HandleProcessingInstruction(void* aUserData,
533
const XML_Char* aTarget,
534
const XML_Char* aData) {
535
UserData* ud = static_cast<UserData*>(aUserData);
536
XML_StopParser(ud->mExpat, false);
537
}
538
539
void nsHtml5StreamParser::FinalizeSniffingWithDetector(
540
Span<const uint8_t> aFromSegment, uint32_t aCountToSniffingLimit,
541
bool aEof) {
542
if (mSniffingBuffer) {
543
FeedDetector(MakeSpan(mSniffingBuffer.get(), mSniffingLength), false);
544
}
545
if (mFeedChardet && !aFromSegment.IsEmpty()) {
546
// Avoid buffer boundary-dependent behavior when
547
// reparsing is forbidden. If reparse is forbidden,
548
// act as if we only saw the first 1024 bytes.
549
// When reparsing isn't forbidden, buffer boundaries
550
// can have an effect on whether the page is loaded
551
// once or twice. :-(
552
FeedDetector(mReparseForbidden ? aFromSegment.To(aCountToSniffingLimit)
553
: aFromSegment,
554
false);
555
}
556
if (mFeedChardet && aEof &&
557
(!mReparseForbidden || aCountToSniffingLimit == aFromSegment.Length())) {
558
// Don't signal EOF if reparse is forbidden and we didn't pass all input
559
// to the detector above.
560
mFeedChardet = false;
561
FeedDetector(Span<const uint8_t>(), true);
562
}
563
}
564
565
nsresult nsHtml5StreamParser::FinalizeSniffing(Span<const uint8_t> aFromSegment,
566
uint32_t aCountToSniffingLimit,
567
bool aEof) {
568
MOZ_ASSERT(IsParserThread(), "Wrong thread!");
569
MOZ_ASSERT(mCharsetSource < kCharsetFromUserForcedAutoDetection,
570
"Should not finalize sniffing with strong decision already made.");
571
if (mMode == VIEW_SOURCE_XML) {
572
static const XML_Memory_Handling_Suite memsuite = {
573
(void* (*)(size_t))moz_xmalloc, (void* (*)(void*, size_t))moz_xrealloc,
574
free};
575
576
static const char16_t kExpatSeparator[] = {0xFFFF, '\0'};
577
578
static const char16_t kISO88591[] = {'I', 'S', 'O', '-', '8', '8',
579
'5', '9', '-', '1', '\0'};
580
581
UserData ud;
582
ud.mStreamParser = this;
583
584
// If we got this far, the stream didn't have a BOM. UTF-16-encoded XML
585
// documents MUST begin with a BOM. We don't support EBCDIC and such.
586
// Thus, at this point, what we have is garbage or something encoded using
587
// a rough ASCII superset. ISO-8859-1 allows us to decode ASCII bytes
588
// without throwing errors when bytes have the most significant bit set
589
// and without triggering expat's unknown encoding code paths. This is
590
// enough to be able to use expat to parse the XML declaration in order
591
// to extract the encoding name from it.
592
ud.mExpat = XML_ParserCreate_MM(kISO88591, &memsuite, kExpatSeparator);
593
XML_SetXmlDeclHandler(ud.mExpat, HandleXMLDeclaration);
594
XML_SetElementHandler(ud.mExpat, HandleStartElement, HandleEndElement);
595
XML_SetCommentHandler(ud.mExpat, HandleComment);
596
XML_SetProcessingInstructionHandler(ud.mExpat, HandleProcessingInstruction);
597
XML_SetUserData(ud.mExpat, static_cast<void*>(&ud));
598
599
XML_Status status = XML_STATUS_OK;
600
601
// aFromSegment points to the data obtained from the current network
602
// event. mSniffingBuffer (if it exists) contains the data obtained before
603
// the current event. Thus, mSniffingLenth bytes of mSniffingBuffer
604
// followed by aCountToSniffingLimit bytes from aFromSegment are the
605
// first 1024 bytes of the file (or the file as a whole if the file is
606
// 1024 bytes long or shorter). Thus, we parse both buffers, but if the
607
// first call succeeds already, we skip parsing the second buffer.
608
if (mSniffingBuffer) {
609
status = XML_Parse(ud.mExpat,
610
reinterpret_cast<const char*>(mSniffingBuffer.get()),
611
mSniffingLength, false);
612
}
613
if (status == XML_STATUS_OK && mCharsetSource < kCharsetFromMetaTag) {
614
mozilla::Unused << XML_Parse(
615
ud.mExpat, reinterpret_cast<const char*>(aFromSegment.Elements()),
616
aCountToSniffingLimit, false);
617
}
618
XML_ParserFree(ud.mExpat);
619
620
if (mCharsetSource < kCharsetFromMetaTag) {
621
// Failed to get an encoding from the XML declaration. XML defaults
622
// confidently to UTF-8 in this case.
623
// It is also possible that the document has an XML declaration that is
624
// longer than 1024 bytes, but that case is not worth worrying about.
625
mEncoding = UTF_8_ENCODING;
626
mCharsetSource = kCharsetFromMetaTag; // means confident
627
}
628
629
return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(aFromSegment);
630
}
631
632
// meta scan failed.
633
if (mCharsetSource < kCharsetFromMetaPrescan) {
634
// Check for BOMless UTF-16 with Basic
635
// Latin content for compat with IE. See bug 631751.
636
SniffBOMlessUTF16BasicLatin(aFromSegment.To(aCountToSniffingLimit));
637
}
638
// the charset may have been set now
639
// maybe try chardet now;
640
if (mFeedChardet) {
641
FinalizeSniffingWithDetector(aFromSegment, aCountToSniffingLimit, aEof);
642
// fall thru; callback may have changed charset
643
}
644
if (mCharsetSource == kCharsetUninitialized) {
645
// Hopefully this case is never needed, but dealing with it anyway
646
mEncoding = WINDOWS_1252_ENCODING;
647
mCharsetSource = kCharsetFromFallback;
648
mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
649
} else if (mMode == LOAD_AS_DATA && mCharsetSource == kCharsetFromFallback) {
650
NS_ASSERTION(mReparseForbidden, "Reparse should be forbidden for XHR");
651
NS_ASSERTION(!mFeedChardet, "Should not feed chardet for XHR");
652
NS_ASSERTION(mEncoding == UTF_8_ENCODING, "XHR should default to UTF-8");
653
// Now mark charset source as non-weak to signal that we have a decision
654
mCharsetSource = kCharsetFromDocTypeDefault;
655
mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
656
}
657
return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(aFromSegment);
658
}
659
660
nsresult nsHtml5StreamParser::SniffStreamBytes(
661
Span<const uint8_t> aFromSegment) {
662
NS_ASSERTION(IsParserThread(), "Wrong thread!");
663
nsresult rv = NS_OK;
664
// mEncoding and mCharsetSource potentially have come from channel or higher
665
// by now. If we find a BOM, SetupDecodingFromBom() will overwrite them.
666
// If we don't find a BOM, the previously set values of mEncoding and
667
// mCharsetSource are not modified by the BOM sniffing here.
668
for (uint32_t i = 0;
669
i < aFromSegment.Length() && mBomState != BOM_SNIFFING_OVER; i++) {
670
switch (mBomState) {
671
case BOM_SNIFFING_NOT_STARTED:
672
NS_ASSERTION(i == 0, "Bad BOM sniffing state.");
673
switch (aFromSegment[0]) {
674
case 0xEF:
675
mBomState = SEEN_UTF_8_FIRST_BYTE;
676
break;
677
case 0xFF:
678
mBomState = SEEN_UTF_16_LE_FIRST_BYTE;
679
break;
680
case 0xFE:
681
mBomState = SEEN_UTF_16_BE_FIRST_BYTE;
682
break;
683
default:
684
mBomState = BOM_SNIFFING_OVER;
685
break;
686
}
687
break;
688
case SEEN_UTF_16_LE_FIRST_BYTE:
689
if (aFromSegment[i] == 0xFE) {
690
rv = SetupDecodingFromBom(UTF_16LE_ENCODING);
691
NS_ENSURE_SUCCESS(rv, rv);
692
return WriteStreamBytes(aFromSegment.From(i + 1));
693
}
694
mBomState = BOM_SNIFFING_OVER;
695
break;
696
case SEEN_UTF_16_BE_FIRST_BYTE:
697
if (aFromSegment[i] == 0xFF) {
698
rv = SetupDecodingFromBom(UTF_16BE_ENCODING);
699
NS_ENSURE_SUCCESS(rv, rv);
700
return WriteStreamBytes(aFromSegment.From(i + 1));
701
}
702
mBomState = BOM_SNIFFING_OVER;
703
break;
704
case SEEN_UTF_8_FIRST_BYTE:
705
if (aFromSegment[i] == 0xBB) {
706
mBomState = SEEN_UTF_8_SECOND_BYTE;
707
} else {
708
mBomState = BOM_SNIFFING_OVER;
709
}
710
break;
711
case SEEN_UTF_8_SECOND_BYTE:
712
if (aFromSegment[i] == 0xBF) {
713
rv = SetupDecodingFromBom(UTF_8_ENCODING);
714
NS_ENSURE_SUCCESS(rv, rv);
715
return WriteStreamBytes(aFromSegment.From(i + 1));
716
}
717
mBomState = BOM_SNIFFING_OVER;
718
break;
719
default:
720
mBomState = BOM_SNIFFING_OVER;
721
break;
722
}
723
}
724
// if we get here, there either was no BOM or the BOM sniffing isn't complete
725
// yet
726
727
MOZ_ASSERT(mCharsetSource != kCharsetFromByteOrderMark,
728
"Should not come here if BOM was found.");
729
MOZ_ASSERT(mCharsetSource != kCharsetFromOtherComponent,
730
"kCharsetFromOtherComponent is for XSLT.");
731
732
if (mBomState == BOM_SNIFFING_OVER && mCharsetSource == kCharsetFromChannel) {
733
// There was no BOM and the charset came from channel. mEncoding
734
// still contains the charset from the channel as set by an
735
// earlier call to SetDocumentCharset(), since we didn't find a BOM and
736
// overwrite mEncoding. (Note that if the user has overridden the charset,
737
// we don't come here but check <meta> for XSS-dangerous charsets first.)
738
mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
739
return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(aFromSegment);
740
}
741
742
if (!mMetaScanner &&
743
(mMode == NORMAL || mMode == VIEW_SOURCE_HTML || mMode == LOAD_AS_DATA)) {
744
mMetaScanner = new nsHtml5MetaScanner(mTreeBuilder);
745
}
746
747
if (mSniffingLength + aFromSegment.Length() >= SNIFFING_BUFFER_SIZE) {
748
// this is the last buffer
749
uint32_t countToSniffingLimit = SNIFFING_BUFFER_SIZE - mSniffingLength;
750
if (mMode == NORMAL || mMode == VIEW_SOURCE_HTML || mMode == LOAD_AS_DATA) {
751
nsHtml5ByteReadable readable(
752
aFromSegment.Elements(),
753
aFromSegment.Elements() + countToSniffingLimit);
754
nsAutoCString charset;
755
auto encoding = mMetaScanner->sniff(&readable);
756
// Due to the way nsHtml5Portability reports OOM, ask the tree buider
757
nsresult rv;
758
if (NS_FAILED((rv = mTreeBuilder->IsBroken()))) {
759
MarkAsBroken(rv);
760
return rv;
761
}
762
if (encoding) {
763
// meta scan successful; honor overrides unless meta is XSS-dangerous
764
if ((mCharsetSource == kCharsetFromParentForced ||
765
mCharsetSource == kCharsetFromUserForced) &&
766
(encoding->IsAsciiCompatible() ||
767
encoding == ISO_2022_JP_ENCODING)) {
768
// Honor override
769
if (mEncoding->IsJapaneseLegacy()) {
770
mFeedChardet = true;
771
FinalizeSniffingWithDetector(aFromSegment, countToSniffingLimit,
772
false);
773
}
774
return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
775
aFromSegment);
776
}
777
mEncoding = WrapNotNull(encoding);
778
mCharsetSource = kCharsetFromMetaPrescan;
779
mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
780
return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
781
aFromSegment);
782
}
783
}
784
if (mCharsetSource == kCharsetFromParentForced ||
785
mCharsetSource == kCharsetFromUserForced) {
786
// meta not found, honor override
787
if (mEncoding->IsJapaneseLegacy()) {
788
mFeedChardet = true;
789
FinalizeSniffingWithDetector(aFromSegment, countToSniffingLimit, false);
790
}
791
return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(aFromSegment);
792
}
793
return FinalizeSniffing(aFromSegment, countToSniffingLimit, false);
794
}
795
796
// not the last buffer
797
if (mMode == NORMAL || mMode == VIEW_SOURCE_HTML || mMode == LOAD_AS_DATA) {
798
nsHtml5ByteReadable readable(
799
aFromSegment.Elements(),
800
aFromSegment.Elements() + aFromSegment.Length());
801
auto encoding = mMetaScanner->sniff(&readable);
802
// Due to the way nsHtml5Portability reports OOM, ask the tree buider
803
nsresult rv;
804
if (NS_FAILED((rv = mTreeBuilder->IsBroken()))) {
805
MarkAsBroken(rv);
806
return rv;
807
}
808
if (encoding) {
809
// meta scan successful; honor overrides unless meta is XSS-dangerous
810
if ((mCharsetSource == kCharsetFromParentForced ||
811
mCharsetSource == kCharsetFromUserForced) &&
812
(encoding->IsAsciiCompatible() || encoding == ISO_2022_JP_ENCODING)) {
813
// Honor override
814
return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
815
aFromSegment);
816
}
817
mEncoding = WrapNotNull(encoding);
818
mCharsetSource = kCharsetFromMetaPrescan;
819
mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
820
return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(aFromSegment);
821
}
822
}
823
824
if (!mSniffingBuffer) {
825
mSniffingBuffer = MakeUniqueFallible<uint8_t[]>(SNIFFING_BUFFER_SIZE);
826
if (!mSniffingBuffer) {
827
return NS_ERROR_OUT_OF_MEMORY;
828
}
829
}
830
memcpy(&mSniffingBuffer[mSniffingLength], aFromSegment.Elements(),
831
aFromSegment.Length());
832
mSniffingLength += aFromSegment.Length();
833
return NS_OK;
834
}
835
836
nsresult nsHtml5StreamParser::WriteStreamBytes(
837
Span<const uint8_t> aFromSegment) {
838
NS_ASSERTION(IsParserThread(), "Wrong thread!");
839
// mLastBuffer should always point to a buffer of the size
840
// READ_BUFFER_SIZE.
841
if (!mLastBuffer) {
842
NS_WARNING("mLastBuffer should not be null!");
843
MarkAsBroken(NS_ERROR_NULL_POINTER);
844
return NS_ERROR_NULL_POINTER;
845
}
846
size_t totalRead = 0;
847
auto src = aFromSegment;
848
for (;;) {
849
auto dst = mLastBuffer->TailAsSpan(READ_BUFFER_SIZE);
850
uint32_t result;
851
size_t read;
852
size_t written;
853
bool hadErrors;
854
Tie(result, read, written, hadErrors) =
855
mUnicodeDecoder->DecodeToUTF16(src, dst, false);
856
if (!mDecodingLocalFileAsUTF8 && recordreplay::IsRecordingOrReplaying()) {
857
recordreplay::AddContentParseData16(this, dst.data(), written);
858
}
859
if (hadErrors && !mHasHadErrors) {
860
if (mDecodingLocalFileAsUTF8) {
861
ReDecodeLocalFile();
862
return NS_OK;
863
}
864
mHasHadErrors = true;
865
if (mEncoding == UTF_8_ENCODING) {
866
mTreeBuilder->TryToEnableEncodingMenu();
867
}
868
}
869
src = src.From(read);
870
totalRead += read;
871
mLastBuffer->AdvanceEnd(written);
872
if (result == kOutputFull) {
873
RefPtr<nsHtml5OwningUTF16Buffer> newBuf =
874
nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE);
875
if (!newBuf) {
876
MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
877
return NS_ERROR_OUT_OF_MEMORY;
878
}
879
mLastBuffer = (mLastBuffer->next = newBuf.forget());
880
} else {
881
MOZ_ASSERT(totalRead == aFromSegment.Length(),
882
"The Unicode decoder consumed the wrong number of bytes.");
883
if (mDecodingLocalFileAsUTF8 &&
884
mLocalFileBytesBuffered == LOCAL_FILE_UTF_8_BUFFER_SIZE) {
885
CommitLocalFileToUTF8();
886
}
887
return NS_OK;
888
}
889
}
890
}
891
892
void nsHtml5StreamParser::ReDecodeLocalFile() {
893
MOZ_ASSERT(mDecodingLocalFileAsUTF8);
894
mDecodingLocalFileAsUTF8 = false;
895
mUnicodeDecoder = mEncoding->NewDecoderWithBOMRemoval();
896
mHasHadErrors = false;
897
898
// We need the detector to start with fresh state.
899
// Turn off ISO-2022-JP detection, because if this doc was
900
// ISO-2022-JP, it would have already been detected.
901
mJapaneseDetector = mozilla::JapaneseDetector::Create(false);
902
mFeedChardet = true;
903
904
// Throw away previous decoded data
905
mLastBuffer = mFirstBuffer;
906
mLastBuffer->next = nullptr;
907
mLastBuffer->setStart(0);
908
mLastBuffer->setEnd(0);
909
910
// Decode again
911
for (auto&& buffer : mBufferedLocalFileData) {
912
DoDataAvailable(buffer);
913
}
914
}
915
916
void nsHtml5StreamParser::CommitLocalFileToUTF8() {
917
MOZ_ASSERT(mDecodingLocalFileAsUTF8);
918
mDecodingLocalFileAsUTF8 = false;
919
mFeedChardet = false;
920
mEncoding = UTF_8_ENCODING;
921
mCharsetSource = kCharsetFromFileURLGuess;
922
mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
923
924
if (recordreplay::IsRecordingOrReplaying()) {
925
nsHtml5OwningUTF16Buffer* buffer = mLastBuffer;
926
while (buffer) {
927
recordreplay::AddContentParseData16(
928
this, buffer->getBuffer() + buffer->getStart(), buffer->getLength());
929
buffer = buffer->next;
930
}
931
}
932
}
933
934
class MaybeRunCollector : public Runnable {
935
public:
936
explicit MaybeRunCollector(nsIDocShell* aDocShell)
937
: Runnable("MaybeRunCollector"), mDocShell(aDocShell) {}
938
939
NS_IMETHOD Run() override {
940
nsJSContext::MaybeRunNextCollectorSlice(mDocShell,
941
JS::GCReason::HTML_PARSER);
942
return NS_OK;
943
}
944
945
nsCOMPtr<nsIDocShell> mDocShell;
946
};
947
948
nsresult nsHtml5StreamParser::OnStartRequest(nsIRequest* aRequest) {
949
MOZ_RELEASE_ASSERT(STREAM_NOT_STARTED == mStreamState,
950
"Got OnStartRequest when the stream had already started.");
951
MOZ_ASSERT(
952
!mExecutor->HasStarted(),
953
"Got OnStartRequest at the wrong stage in the executor life cycle.");
954
NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
955
if (mObserver) {
956
mObserver->OnStartRequest(aRequest);
957
}
958
mRequest = aRequest;
959
960
mStreamState = STREAM_BEING_READ;
961
962
if (mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML) {
963
mTokenizer->StartViewSource(NS_ConvertUTF8toUTF16(mViewSourceTitle));
964
}
965
966
// For View Source, the parser should run with scripts "enabled" if a normal
967
// load would have scripts enabled.
968
bool scriptingEnabled =
969
mMode == LOAD_AS_DATA ? false : mExecutor->IsScriptEnabled();
970
mOwner->StartTokenizer(scriptingEnabled);
971
972
MOZ_ASSERT(!mDecodingLocalFileAsUTF8);
973
bool isSrcdoc = false;
974
nsCOMPtr<nsIChannel> channel;
975
nsresult rv = GetChannel(getter_AddRefs(channel));
976
if (NS_SUCCEEDED(rv)) {
977
isSrcdoc = NS_IsSrcdocChannel(channel);
978
if (!isSrcdoc && mCharsetSource <= kCharsetFromFileURLGuess) {
979
nsCOMPtr<nsIURI> originalURI;
980
rv = channel->GetOriginalURI(getter_AddRefs(originalURI));
981
if (NS_SUCCEEDED(rv)) {
982
bool originalIsResource;
983
originalURI->SchemeIs("resource", &originalIsResource);
984
if (originalIsResource) {
985
mCharsetSource = kCharsetFromBuiltIn;
986
mEncoding = UTF_8_ENCODING;
987
} else {
988
nsCOMPtr<nsIURI> currentURI;
989
rv = channel->GetURI(getter_AddRefs(currentURI));
990
if (NS_SUCCEEDED(rv)) {
991
nsCOMPtr<nsIURI> innermost = NS_GetInnermostURI(currentURI);
992
bool innermostIsFile;
993
innermost->SchemeIs("file", &innermostIsFile);
994
mDecodingLocalFileAsUTF8 = innermostIsFile;
995
}
996
}
997
}
998
}
999
}
1000
mTreeBuilder->setIsSrcdocDocument(isSrcdoc);
1001
mTreeBuilder->setScriptingEnabled(scriptingEnabled);
1002
mTreeBuilder->SetPreventScriptExecution(
1003
!((mMode == NORMAL) && scriptingEnabled));
1004
mTokenizer->start();
1005
mExecutor->Start();
1006
mExecutor->StartReadingFromStage();
1007
1008
if (mMode == PLAIN_TEXT) {
1009
mTreeBuilder->StartPlainText();
1010
mTokenizer->StartPlainText();
1011
} else if (mMode == VIEW_SOURCE_PLAIN) {
1012
nsAutoString viewSourceTitle;
1013
CopyUTF8toUTF16(mViewSourceTitle, viewSourceTitle);
1014
mTreeBuilder->EnsureBufferSpace(viewSourceTitle.Length());
1015
mTreeBuilder->StartPlainTextViewSource(viewSourceTitle);
1016
mTokenizer->StartPlainText();
1017
}
1018
1019
/*
1020
* If you move the following line, be very careful not to cause
1021
* WillBuildModel to be called before the document has had its
1022
* script global object set.
1023
*/
1024
rv = mExecutor->WillBuildModel(eDTDMode_unknown);
1025
NS_ENSURE_SUCCESS(rv, rv);
1026
1027
RefPtr<nsHtml5OwningUTF16Buffer> newBuf =
1028
nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE);
1029
if (!newBuf) {
1030
// marks this stream parser as terminated,
1031
// which prevents entry to code paths that
1032
// would use mFirstBuffer or mLastBuffer.
1033
return mExecutor->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1034
}
1035
MOZ_ASSERT(!mFirstBuffer, "How come we have the first buffer set?");
1036
MOZ_ASSERT(!mLastBuffer, "How come we have the last buffer set?");
1037
mFirstBuffer = mLastBuffer = newBuf;
1038
1039
rv = NS_OK;
1040
1041
// The line below means that the encoding can end up being wrong if
1042
// a view-source URL is loaded without having the encoding hint from a
1043
// previous normal load in the history.
1044
mReparseForbidden = !(mMode == NORMAL || mMode == PLAIN_TEXT);
1045
1046
mDocGroup = mExecutor->GetDocument()->GetDocGroup();
1047
1048
nsCOMPtr<nsIHttpChannel> httpChannel(do_QueryInterface(mRequest, &rv));
1049
if (NS_SUCCEEDED(rv)) {
1050
// Non-HTTP channels are bogus enough that we let them work with unlabeled
1051
// runnables for now. Asserting for HTTP channels only.
1052
MOZ_ASSERT(mDocGroup || mMode == LOAD_AS_DATA,
1053
"How come the doc group is still null?");
1054
1055
nsAutoCString method;
1056
Unused << httpChannel->GetRequestMethod(method);
1057
// XXX does Necko have a way to renavigate POST, etc. without hitting
1058
// the network?
1059
if (!method.EqualsLiteral("GET")) {
1060
// This is the old Gecko behavior but the HTML5 spec disagrees.
1061
// Don't reparse on POST.
1062
mReparseForbidden = true;
1063
mFeedChardet = false; // can't restart anyway
1064
}
1065
}
1066
1067
// Attempt to retarget delivery of data (via OnDataAvailable) to the parser
1068
// thread, rather than through the main thread.
1069
nsCOMPtr<nsIThreadRetargetableRequest> threadRetargetableRequest =
1070
do_QueryInterface(mRequest, &rv);
1071
if (threadRetargetableRequest) {
1072
rv = threadRetargetableRequest->RetargetDeliveryTo(mEventTarget);
1073
if (NS_SUCCEEDED(rv)) {
1074
// Parser thread should be now ready to get data from necko and parse it
1075
// and main thread might have a chance to process a collector slice.
1076
// We need to do this asynchronously so that necko may continue processing
1077
// the request.
1078
nsCOMPtr<nsIRunnable> runnable =
1079
new MaybeRunCollector(mExecutor->GetDocument()->GetDocShell());
1080
mozilla::SystemGroup::Dispatch(mozilla::TaskCategory::GarbageCollection,
1081
runnable.forget());
1082
}
1083
}
1084
1085
if (NS_FAILED(rv)) {
1086
NS_WARNING("Failed to retarget HTML data delivery to the parser thread.");
1087
}
1088
1089
if (mCharsetSource == kCharsetFromParentFrame) {
1090
// Remember this in case chardet overwrites mCharsetSource
1091
mInitialEncodingWasFromParentFrame = true;
1092
}
1093
1094
if (mCharsetSource >= kCharsetFromAutoDetection &&
1095
!(mCharsetSource == kCharsetFromParentForced ||
1096
mCharsetSource == kCharsetFromUserForced)) {
1097
mFeedChardet = false;
1098
}
1099
1100
if (mCharsetSource < kCharsetFromUtf8OnlyMime) {
1101
// we aren't ready to commit to an encoding yet
1102
// leave converter uninstantiated for now
1103
return NS_OK;
1104
}
1105
1106
// We are loading JSON/WebVTT/etc. into a browsing context.
1107
// There's no need to remove the BOM manually here, because
1108
// the UTF-8 decoder removes it.
1109
mReparseForbidden = true;
1110
mFeedChardet = false;
1111
1112
// Instantiate the converter here to avoid BOM sniffing.
1113
mDecodingLocalFileAsUTF8 = false;
1114
mUnicodeDecoder = mEncoding->NewDecoderWithBOMRemoval();
1115
return NS_OK;
1116
}
1117
1118
nsresult nsHtml5StreamParser::CheckListenerChain() {
1119
NS_ASSERTION(NS_IsMainThread(), "Should be on the main thread!");
1120
if (!mObserver) {
1121
return NS_OK;
1122
}
1123
nsresult rv;
1124
nsCOMPtr<nsIThreadRetargetableStreamListener> retargetable =
1125
do_QueryInterface(mObserver, &rv);
1126
if (NS_SUCCEEDED(rv) && retargetable) {
1127
rv = retargetable->CheckListenerChain();
1128
}
1129
return rv;
1130
}
1131
1132
void nsHtml5StreamParser::DoStopRequest() {
1133
NS_ASSERTION(IsParserThread(), "Wrong thread!");
1134
MOZ_RELEASE_ASSERT(STREAM_BEING_READ == mStreamState,
1135
"Stream ended without being open.");
1136
mTokenizerMutex.AssertCurrentThreadOwns();
1137
1138
if (IsTerminated()) {
1139
return;
1140
}
1141
1142
if (!mUnicodeDecoder) {
1143
nsresult rv;
1144
Span<const uint8_t> empty;
1145
if (NS_FAILED(rv = FinalizeSniffing(empty, 0, true))) {
1146
MarkAsBroken(rv);
1147
return;
1148
}
1149
}
1150
if (mFeedChardet) {
1151
mFeedChardet = false;
1152
FeedDetector(Span<uint8_t>(), true);
1153
}
1154
1155
MOZ_ASSERT(mUnicodeDecoder,
1156
"Should have a decoder after finalizing sniffing.");
1157
1158
// mLastBuffer should always point to a buffer of the size
1159
// READ_BUFFER_SIZE.
1160
if (!mLastBuffer) {
1161
NS_WARNING("mLastBuffer should not be null!");
1162
MarkAsBroken(NS_ERROR_NULL_POINTER);
1163
return;
1164
}
1165
1166
Span<uint8_t> src; // empty span
1167
for (;;) {
1168
auto dst = mLastBuffer->TailAsSpan(READ_BUFFER_SIZE);
1169
uint32_t result;
1170
size_t read;
1171
size_t written;
1172
bool hadErrors;
1173
Tie(result, read, written, hadErrors) =
1174
mUnicodeDecoder->DecodeToUTF16(src, dst, true);
1175
if (!mDecodingLocalFileAsUTF8 && recordreplay::IsRecordingOrReplaying()) {
1176
recordreplay::AddContentParseData16(this, dst.data(), written);
1177
}
1178
if (hadErrors && !mHasHadErrors) {
1179
if (mDecodingLocalFileAsUTF8) {
1180
ReDecodeLocalFile();
1181
DoStopRequest();
1182
return;
1183
}
1184
mHasHadErrors = true;
1185
if (mEncoding == UTF_8_ENCODING) {
1186
mTreeBuilder->TryToEnableEncodingMenu();
1187
}
1188
}
1189
MOZ_ASSERT(read == 0, "How come an empty span was read form?");
1190
mLastBuffer->AdvanceEnd(written);
1191
if (result == kOutputFull) {
1192
RefPtr<nsHtml5OwningUTF16Buffer> newBuf =
1193
nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE);
1194
if (!newBuf) {
1195
MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1196
return;
1197
}
1198
mLastBuffer = (mLastBuffer->next = newBuf.forget());
1199
} else {
1200
if (mDecodingLocalFileAsUTF8) {
1201
MOZ_ASSERT(mLocalFileBytesBuffered < LOCAL_FILE_UTF_8_BUFFER_SIZE);
1202
CommitLocalFileToUTF8();
1203
}
1204
break;
1205
}
1206
}
1207
1208
mStreamState = STREAM_ENDED;
1209
1210
if (IsTerminatedOrInterrupted()) {
1211
return;
1212
}
1213
1214
ParseAvailableData();
1215
}
1216
1217
class nsHtml5RequestStopper : public Runnable {
1218
private:
1219
nsHtml5StreamParserPtr mStreamParser;
1220
1221
public:
1222
explicit nsHtml5RequestStopper(nsHtml5StreamParser* aStreamParser)
1223
: Runnable("nsHtml5RequestStopper"), mStreamParser(aStreamParser) {}
1224
NS_IMETHOD Run() override {
1225
mozilla::MutexAutoLock autoLock(mStreamParser->mTokenizerMutex);
1226
mStreamParser->DoStopRequest();
1227
return NS_OK;
1228
}
1229
};
1230
1231
nsresult nsHtml5StreamParser::OnStopRequest(nsIRequest* aRequest,
1232
nsresult status) {
1233
NS_ASSERTION(mRequest == aRequest, "Got Stop on wrong stream.");
1234
NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
1235
if (mObserver) {
1236
mObserver->OnStopRequest(aRequest, status);
1237
}
1238
nsCOMPtr<nsIRunnable> stopper = new nsHtml5RequestStopper(this);
1239
if (NS_FAILED(mEventTarget->Dispatch(stopper, nsIThread::DISPATCH_NORMAL))) {
1240
NS_WARNING("Dispatching StopRequest event failed.");
1241
}
1242
return NS_OK;
1243
}
1244
1245
void nsHtml5StreamParser::DoDataAvailableBuffer(
1246
mozilla::Buffer<uint8_t>&& aBuffer) {
1247
if (MOZ_LIKELY(!mDecodingLocalFileAsUTF8)) {
1248
DoDataAvailable(aBuffer);
1249
return;
1250
}
1251
CheckedInt<size_t> bufferedPlusLength(aBuffer.Length());
1252
bufferedPlusLength += mLocalFileBytesBuffered;
1253
if (!bufferedPlusLength.isValid()) {
1254
MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1255
return;
1256
}
1257
// Ensure that WriteStreamBytes() sees a buffer ending
1258
// exactly at LOCAL_FILE_UTF_8_BUFFER_SIZE
1259
// if we are about to cross the threshold. This way,
1260
// Necko buffer boundaries don't affect user-visible
1261
// behavior.
1262
if (bufferedPlusLength.value() <= LOCAL_FILE_UTF_8_BUFFER_SIZE) {
1263
// Truncation OK, because we just checked the range.
1264
mLocalFileBytesBuffered = bufferedPlusLength.value();
1265
mBufferedLocalFileData.AppendElement(std::move(aBuffer));
1266
DoDataAvailable(mBufferedLocalFileData.LastElement());
1267
} else {
1268
// Truncation OK, because the constant is small enough.
1269
size_t overBoundary =
1270
bufferedPlusLength.value() - LOCAL_FILE_UTF_8_BUFFER_SIZE;
1271
MOZ_RELEASE_ASSERT(overBoundary < aBuffer.Length());
1272
size_t untilBoundary = aBuffer.Length() - overBoundary;
1273
auto span = aBuffer.AsSpan();
1274
auto head = span.To(untilBoundary);
1275
auto tail = span.From(untilBoundary);
1276
MOZ_RELEASE_ASSERT(mLocalFileBytesBuffered + untilBoundary ==
1277
LOCAL_FILE_UTF_8_BUFFER_SIZE);
1278
// We make a theoretically useless copy here, because avoiding
1279
// the copy adds too much complexity.
1280
Maybe<Buffer<uint8_t>> maybe = Buffer<uint8_t>::CopyFrom(head);
1281
if (maybe.isNothing()) {
1282
MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1283
return;
1284
}
1285
mLocalFileBytesBuffered = LOCAL_FILE_UTF_8_BUFFER_SIZE;
1286
mBufferedLocalFileData.AppendElement(std::move(*maybe));
1287
1288
DoDataAvailable(head);
1289
// Re-decode may have happened here.
1290
DoDataAvailable(tail);
1291
}
1292
// Do this clean-up here to avoid use-after-free when
1293
// DoDataAvailable is passed a span pointing into an
1294
// element of mBufferedLocalFileData.
1295
if (!mDecodingLocalFileAsUTF8) {
1296
mBufferedLocalFileData.Clear();
1297
}
1298
}
1299
1300
void nsHtml5StreamParser::DoDataAvailable(Span<const uint8_t> aBuffer) {
1301
NS_ASSERTION(IsParserThread(), "Wrong thread!");
1302
MOZ_RELEASE_ASSERT(STREAM_BEING_READ == mStreamState,
1303
"DoDataAvailable called when stream not open.");
1304
mTokenizerMutex.AssertCurrentThreadOwns();
1305
1306
if (IsTerminated()) {
1307
return;
1308
}
1309
1310
nsresult rv;
1311
if (HasDecoder()) {
1312
if (mFeedChardet) {
1313
FeedDetector(aBuffer, false);
1314
}
1315
rv = WriteStreamBytes(aBuffer);
1316
} else {
1317
rv = SniffStreamBytes(aBuffer);
1318
}
1319
if (NS_FAILED(rv)) {
1320
MarkAsBroken(rv);
1321
return;
1322
}
1323
1324
if (IsTerminatedOrInterrupted()) {
1325
return;
1326
}
1327
1328
if (mDecodingLocalFileAsUTF8) {
1329
return;
1330
}
1331
1332
ParseAvailableData();
1333
1334
if (mFlushTimerArmed || mSpeculating) {
1335
return;
1336
}
1337
1338
{
1339
mozilla::MutexAutoLock flushTimerLock(mFlushTimerMutex);
1340
mFlushTimer->InitWithNamedFuncCallback(
1341
nsHtml5StreamParser::TimerCallback, static_cast<void*>(this),
1342
mFlushTimerEverFired ? StaticPrefs::html5_flushtimer_initialdelay()
1343
: StaticPrefs::html5_flushtimer_subsequentdelay(),
1344
nsITimer::TYPE_ONE_SHOT, "nsHtml5StreamParser::DoDataAvailable");
1345
}
1346
mFlushTimerArmed = true;
1347
}
1348
1349
class nsHtml5DataAvailable : public Runnable {
1350
private:
1351
nsHtml5StreamParserPtr mStreamParser;
1352
Buffer<uint8_t> mData;
1353
1354
public:
1355
nsHtml5DataAvailable(nsHtml5StreamParser* aStreamParser,
1356
Buffer<uint8_t>&& aData)
1357
: Runnable("nsHtml5DataAvailable"),
1358
mStreamParser(aStreamParser),
1359
mData(std::move(aData)) {}
1360
NS_IMETHOD Run() override {
1361
mozilla::MutexAutoLock autoLock(mStreamParser->mTokenizerMutex);
1362
mStreamParser->DoDataAvailableBuffer(std::move(mData));
1363
return NS_OK;
1364
}
1365
};
1366
1367
nsresult nsHtml5StreamParser::OnDataAvailable(nsIRequest* aRequest,
1368
nsIInputStream* aInStream,
1369
uint64_t aSourceOffset,
1370
uint32_t aLength) {
1371
nsresult rv;
1372
if (NS_FAILED(rv = mExecutor->IsBroken())) {
1373
return rv;
1374
}
1375
1376
MOZ_ASSERT(mRequest == aRequest, "Got data on wrong stream.");
1377
uint32_t totalRead;
1378
// Main thread to parser thread dispatch requires copying to buffer first.
1379
if (MOZ_UNLIKELY(NS_IsMainThread())) {
1380
Maybe<Buffer<uint8_t>> maybe = Buffer<uint8_t>::Alloc(aLength);
1381
if (maybe.isNothing()) {
1382
return mExecutor->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1383
}
1384
Buffer<uint8_t> data(std::move(*maybe));
1385
rv = aInStream->Read(reinterpret_cast<char*>(data.Elements()),
1386
data.Length(), &totalRead);
1387
NS_ENSURE_SUCCESS(rv, rv);
1388
MOZ_ASSERT(totalRead == aLength);
1389
1390
nsCOMPtr<nsIRunnable> dataAvailable =
1391
new nsHtml5DataAvailable(this, std::move(data));
1392
if (NS_FAILED(mEventTarget->Dispatch(dataAvailable,
1393
nsIThread::DISPATCH_NORMAL))) {
1394
NS_WARNING("Dispatching DataAvailable event failed.");
1395
}
1396
return rv;
1397
}
1398
MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1399
mozilla::MutexAutoLock autoLock(mTokenizerMutex);
1400
1401
if (MOZ_UNLIKELY(mDecodingLocalFileAsUTF8)) {
1402
// It's a bit sad to potentially buffer the first 1024
1403
// bytes in two places, but it's a lot simpler than trying
1404
// to optitize out that copy. It only happens for local files
1405
// and not for the http(s) content anyway.
1406
Maybe<Buffer<uint8_t>> maybe = Buffer<uint8_t>::Alloc(aLength);
1407
if (maybe.isNothing()) {
1408
MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1409
return NS_ERROR_OUT_OF_MEMORY;
1410
}
1411
Buffer<uint8_t> data(std::move(*maybe));
1412
rv = aInStream->Read(reinterpret_cast<char*>(data.Elements()),
1413
data.Length(), &totalRead);
1414
NS_ENSURE_SUCCESS(rv, rv);
1415
MOZ_ASSERT(totalRead == aLength);
1416
DoDataAvailableBuffer(std::move(data));
1417
return rv;
1418
}
1419
// Read directly from response buffer.
1420
rv = aInStream->ReadSegments(CopySegmentsToParser, this, aLength, &totalRead);
1421
NS_ENSURE_SUCCESS(rv, rv);
1422
MOZ_ASSERT(totalRead == aLength);
1423
return rv;
1424
}
1425
1426
/* static */
1427
nsresult nsHtml5StreamParser::CopySegmentsToParser(
1428
nsIInputStream* aInStream, void* aClosure, const char* aFromSegment,
1429
uint32_t aToOffset, uint32_t aCount, uint32_t* aWriteCount) {
1430
nsHtml5StreamParser* parser = static_cast<nsHtml5StreamParser*>(aClosure);
1431
1432
parser->DoDataAvailable(AsBytes(MakeSpan(aFromSegment, aCount)));
1433
// Assume DoDataAvailable consumed all available bytes.
1434
*aWriteCount = aCount;
1435
return NS_OK;
1436
}
1437
1438
const Encoding* nsHtml5StreamParser::PreferredForInternalEncodingDecl(
1439
const nsACString& aEncoding) {
1440
const Encoding* newEncoding = Encoding::ForLabel(aEncoding);
1441
if (!newEncoding) {
1442
// the encoding name is bogus
1443
mTreeBuilder->MaybeComplainAboutCharset("EncMetaUnsupported", true,
1444
mTokenizer->getLineNumber());
1445
return nullptr;
1446
}
1447
1448
if (newEncoding == UTF_16BE_ENCODING || newEncoding == UTF_16LE_ENCODING) {
1449
mTreeBuilder->MaybeComplainAboutCharset("EncMetaUtf16", true,
1450
mTokenizer->getLineNumber());
1451
newEncoding = UTF_8_ENCODING;
1452
}
1453
1454
if (newEncoding == X_USER_DEFINED_ENCODING) {
1455
// WebKit/Blink hack for Indian and Armenian legacy sites
1456
mTreeBuilder->MaybeComplainAboutCharset("EncMetaUserDefined", true,
1457
mTokenizer->getLineNumber());
1458
newEncoding = WINDOWS_1252_ENCODING;
1459
}
1460
1461
if (newEncoding == mEncoding) {
1462
if (mCharsetSource < kCharsetFromMetaPrescan) {
1463
if (mInitialEncodingWasFromParentFrame) {
1464
mTreeBuilder->MaybeComplainAboutCharset("EncLateMetaFrame", false,
1465
mTokenizer->getLineNumber());
1466
} else {
1467
mTreeBuilder->MaybeComplainAboutCharset("EncLateMeta", false,
1468
mTokenizer->getLineNumber());
1469
}
1470
}
1471
mCharsetSource = kCharsetFromMetaTag; // become confident
1472
mFeedChardet = false; // don't feed chardet when confident
1473
return nullptr;
1474
}
1475
1476
return newEncoding;
1477
}
1478
1479
bool nsHtml5StreamParser::internalEncodingDeclaration(nsHtml5String aEncoding) {
1480
// This code needs to stay in sync with
1481
// nsHtml5MetaScanner::tryCharset. Unfortunately, the
1482
// trickery with member fields there leads to some copy-paste reuse. :-(
1483
NS_ASSERTION(IsParserThread(), "Wrong thread!");
1484
if (mCharsetSource >= kCharsetFromMetaTag) { // this threshold corresponds to
1485
// "confident" in the HTML5 spec
1486
return false;
1487
}
1488
1489
nsString newEncoding16; // Not Auto, because using it to hold nsStringBuffer*
1490
aEncoding.ToString(newEncoding16);
1491
nsAutoCString newEncoding;
1492
CopyUTF16toUTF8(newEncoding16, newEncoding);
1493
1494
auto encoding = PreferredForInternalEncodingDecl(newEncoding);
1495
if (!encoding) {
1496
return false;
1497
}
1498
1499
if (mReparseForbidden) {
1500
// This mReparseForbidden check happens after the call to
1501
// PreferredForInternalEncodingDecl so that if that method calls
1502
// MaybeComplainAboutCharset, its charset complaint wins over the one
1503
// below.
1504
mTreeBuilder->MaybeComplainAboutCharset("EncLateMetaTooLate", true,
1505
mTokenizer->getLineNumber());
1506
return false; // not reparsing even if we wanted to
1507
}
1508
1509
// Avoid having the chardet ask for another restart after this restart
1510
// request.
1511
mFeedChardet = false;
1512
mTreeBuilder->NeedsCharsetSwitchTo(WrapNotNull(encoding), kCharsetFromMetaTag,
1513
mTokenizer->getLineNumber());
1514
FlushTreeOpsAndDisarmTimer();
1515
Interrupt();
1516
// the tree op executor will cause the stream parser to terminate
1517
// if the charset switch request is accepted or it'll uninterrupt
1518
// if the request failed. Note that if the restart request fails,
1519
// we don't bother trying to make chardet resume. Might as well
1520
// assume that chardet-requested restarts would fail, too.
1521
return true;
1522
}
1523
1524
void nsHtml5StreamParser::FlushTreeOpsAndDisarmTimer() {
1525
NS_ASSERTION(IsParserThread(), "Wrong thread!");
1526
if (mFlushTimerArmed) {
1527
// avoid calling Cancel if the flush timer isn't armed to avoid acquiring
1528
// a mutex
1529
{
1530
mozilla::MutexAutoLock flushTimerLock(mFlushTimerMutex);
1531
mFlushTimer->Cancel();
1532
}
1533
mFlushTimerArmed = false;
1534
}
1535
if (mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML) {
1536
mTokenizer->FlushViewSource();
1537
}
1538
mTreeBuilder->Flush();
1539
nsCOMPtr<nsIRunnable> runnable(mExecutorFlusher);
1540
if (NS_FAILED(DispatchToMain(runnable.forget()))) {
1541
NS_WARNING("failed to dispatch executor flush event");
1542
}
1543
}
1544
1545
void nsHtml5StreamParser::ParseAvailableData() {
1546
MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1547
mTokenizerMutex.AssertCurrentThreadOwns();
1548
MOZ_ASSERT(!mDecodingLocalFileAsUTF8);
1549
1550
if (IsTerminatedOrInterrupted()) {
1551
return;
1552
}
1553
1554
if (mSpeculating && !IsSpeculationEnabled()) {
1555
return;
1556
}
1557
1558
for (;;) {
1559
if (!mFirstBuffer->hasMore()) {
1560
if (mFirstBuffer == mLastBuffer) {
1561
switch (mStreamState) {
1562
case STREAM_BEING_READ:
1563
// never release the last buffer.
1564
if (!mSpeculating) {
1565
// reuse buffer space if not speculating
1566
mFirstBuffer->setStart(0);
1567
mFirstBuffer->setEnd(0);
1568
}
1569
mTreeBuilder->FlushLoads();
1570
{
1571
// Dispatch this runnable unconditionally, because the loads
1572
// that need flushing may have been flushed earlier even if the
1573
// flush right above here did nothing.
1574
nsCOMPtr<nsIRunnable> runnable(mLoadFlusher);
1575
if (NS_FAILED(DispatchToMain(runnable.forget()))) {
1576
NS_WARNING("failed to dispatch load flush event");
1577
}
1578
}
1579
return; // no more data for now but expecting more
1580
case STREAM_ENDED:
1581
if (mAtEOF) {
1582
return;
1583
}
1584
mAtEOF = true;
1585
if (mCharsetSource < kCharsetFromMetaTag) {
1586
if (mInitialEncodingWasFromParentFrame) {
1587
// Unfortunately, this check doesn't take effect for
1588
// cross-origin frames, so cross-origin ad frames that have
1589
// no text and only an image or a Flash embed get the more
1590
// severe message from the next if block. The message is
1591
// technically accurate, though.
1592
mTreeBuilder->MaybeComplainAboutCharset("EncNoDeclarationFrame",
1593
false, 0);
1594
} else if (mMode == NORMAL) {
1595
mTreeBuilder->MaybeComplainAboutCharset("EncNoDeclaration",
1596
true, 0);
1597
} else if (mMode == PLAIN_TEXT) {
1598
mTreeBuilder->MaybeComplainAboutCharset("EncNoDeclarationPlain",
1599
true, 0);
1600
}
1601
}
1602
if (NS_SUCCEEDED(mTreeBuilder->IsBroken())) {
1603
mTokenizer->eof();
1604
nsresult rv;
1605
if (NS_FAILED((rv = mTreeBuilder->IsBroken()))) {
1606
MarkAsBroken(rv);
1607
} else {
1608
mTreeBuilder->StreamEnded();
1609
if (mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML) {
1610
mTokenizer->EndViewSource();
1611
}
1612
}
1613
}
1614
FlushTreeOpsAndDisarmTimer();
1615
return; // no more data and not expecting more
1616
default:
1617
MOZ_ASSERT_UNREACHABLE("It should be impossible to reach this.");
1618
return;
1619
}
1620
}
1621
mFirstBuffer = mFirstBuffer->next;
1622
continue;
1623
}
1624
1625
// now we have a non-empty buffer
1626
mFirstBuffer->adjust(mLastWasCR);
1627
mLastWasCR = false;
1628
if (mFirstBuffer->hasMore()) {
1629
if (!mTokenizer->EnsureBufferSpace(mFirstBuffer->getLength())) {
1630
MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1631
return;
1632
}
1633
mLastWasCR = mTokenizer->tokenizeBuffer(mFirstBuffer);
1634
nsresult rv;
1635
if (NS_FAILED((rv = mTreeBuilder->IsBroken()))) {
1636
MarkAsBroken(rv);
1637
return;
1638
}
1639
// At this point, internalEncodingDeclaration() may have called
1640
// Terminate, but that never happens together with script.
1641
// Can't assert that here, though, because it's possible that the main
1642
// thread has called Terminate() while this thread was parsing.
1643
if (mTreeBuilder->HasScript()) {
1644
// HasScript() cannot return true if the tree builder is preventing
1645
// script execution.
1646
MOZ_ASSERT(mMode == NORMAL);
1647
mozilla::MutexAutoLock speculationAutoLock(mSpeculationMutex);
1648
nsHtml5Speculation* speculation = new nsHtml5Speculation(
1649
mFirstBuffer, mFirstBuffer->getStart(), mTokenizer->getLineNumber(),
1650
mTreeBuilder->newSnapshot());
1651
mTreeBuilder->AddSnapshotToScript(speculation->GetSnapshot(),
1652
speculation->GetStartLineNumber());
1653
FlushTreeOpsAndDisarmTimer();
1654
mTreeBuilder->SetOpSink(speculation);
1655
mSpeculations.AppendElement(speculation); // adopts the pointer
1656
mSpeculating = true;
1657
}
1658
if (IsTerminatedOrInterrupted()) {
1659
return;
1660
}
1661
}
1662
}
1663
}
1664
1665
class nsHtml5StreamParserContinuation : public Runnable {
1666
private:
1667
nsHtml5StreamParserPtr mStreamParser;
1668
1669
public:
1670
explicit nsHtml5StreamParserContinuation(nsHtml5StreamParser* aStreamParser)
1671
: Runnable("nsHtml5StreamParserContinuation"),
1672
mStreamParser(aStreamParser) {}
1673
NS_IMETHOD Run() override {
1674
mozilla::MutexAutoLock autoLock(mStreamParser->mTokenizerMutex);
1675
mStreamParser->Uninterrupt();
1676
mStreamParser->ParseAvailableData();
1677
return NS_OK;
1678
}
1679
};
1680
1681
void nsHtml5StreamParser::ContinueAfterScripts(nsHtml5Tokenizer* aTokenizer,
1682
nsHtml5TreeBuilder* aTreeBuilder,
1683
bool aLastWasCR) {
1684
NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
1685
NS_ASSERTION(!(mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML),
1686
"ContinueAfterScripts called in view source mode!");
1687
if (NS_FAILED(mExecutor->IsBroken())) {
1688
return;
1689
}
1690
#ifdef DEBUG
1691
mExecutor->AssertStageEmpty();
1692
#endif
1693
bool speculationFailed = false;
1694
{
1695
mozilla::MutexAutoLock speculationAutoLock(mSpeculationMutex);
1696
if (mSpeculations.IsEmpty()) {
1697
MOZ_ASSERT_UNREACHABLE(
1698
"ContinueAfterScripts called without "
1699
"speculations.");
1700
return;
1701
}
1702
1703
nsHtml5Speculation* speculation = mSpeculations.ElementAt(0);
1704
if (aLastWasCR || !aTokenizer->isInDataState() ||
1705
!aTreeBuilder->snapshotMatches(speculation->GetSnapshot())) {
1706
speculationFailed = true;
1707
// We've got a failed speculation :-(
1708
MaybeDisableFutureSpeculation();
1709
Interrupt(); // Make the parser thread release the tokenizer mutex sooner
1710
// now fall out of the speculationAutoLock into the tokenizerAutoLock
1711
// block
1712
} else {
1713
// We've got a successful speculation!
1714
if (mSpeculations.Length() > 1) {
1715
// the first speculation isn't the current speculation, so there's
1716
// no need to bother the parser thread.
1717
speculation->FlushToSink(mExecutor);
1718
NS_ASSERTION(!mExecutor->IsScriptExecuting(),
1719
"ParseUntilBlocked() was supposed to ensure we don't come "
1720
"here when scripts are executing.");
1721
NS_ASSERTION(
1722
mExecutor->IsInFlushLoop(),
1723
"How are we here if "
1724
"RunFlushLoop() didn't call ParseUntilBlocked() which is the "
1725
"only caller of this method?");
1726
mSpeculations.RemoveElementAt(0);
1727
return;
1728
}
1729
// else
1730
Interrupt(); // Make the parser thread release the tokenizer mutex sooner
1731
1732
// now fall through
1733
// the first speculation is the current speculation. Need to
1734
// release the the speculation mutex and acquire the tokenizer
1735
// mutex. (Just acquiring the other mutex here would deadlock)
1736
}
1737
}
1738
{
1739
mozilla::MutexAutoLock tokenizerAutoLock(mTokenizerMutex);
1740
#ifdef DEBUG
1741
{
1742
mAtomTable.SetPermittedLookupEventTarget(
1743
GetMainThreadSerialEventTarget());
1744
}
1745
#endif
1746
// In principle, the speculation mutex should be acquired here,
1747
// but there's no point, because the parser thread only acquires it
1748
// when it has also acquired the tokenizer mutex and we are already
1749
// holding the tokenizer mutex.
1750
if (speculationFailed) {
1751
// Rewind the stream
1752
mAtEOF = false;
1753
nsHtml5Speculation* speculation = mSpeculations.ElementAt(0);
1754
mFirstBuffer = speculation->GetBuffer();
1755
mFirstBuffer->setStart(speculation->GetStart());
1756
mTokenizer->setLineNumber(speculation->GetStartLineNumber());
1757
1758
nsContentUtils::ReportToConsole(
1759
nsIScriptError::warningFlag, NS_LITERAL_CSTRING("DOM Events"),
1760
mExecutor->GetDocument(), nsContentUtils::eDOM_PROPERTIES,
1761
"SpeculationFailed", nsTArray<nsString>(), nullptr, EmptyString(),
1762
speculation->GetStartLineNumber());
1763
1764
nsHtml5OwningUTF16Buffer* buffer = mFirstBuffer->next;
1765
while (buffer) {
1766
buffer->setStart(0);
1767
buffer = buffer->next;
1768
}
1769
1770
mSpeculations.Clear(); // potentially a huge number of destructors
1771
// run here synchronously on the main thread...
1772
1773
mTreeBuilder->flushCharacters(); // empty the pending buffer
1774
mTreeBuilder->ClearOps(); // now get rid of the failed ops
1775
1776
mTreeBuilder->SetOpSink(mExecutor->GetStage());
1777
mExecutor->StartReadingFromStage();
1778
mSpeculating = false;
1779
1780
// Copy state over
1781
mLastWasCR = aLastWasCR;
1782
mTokenizer->loadState(aTokenizer);
1783
mTreeBuilder->loadState(aTreeBuilder);
1784
} else {
1785
// We've got a successful speculation and at least a moment ago it was
1786
// the current speculation
1787
mSpeculations.ElementAt(0)->FlushToSink(mExecutor);
1788
NS_ASSERTION(!mExecutor->IsScriptExecuting(),
1789
"ParseUntilBlocked() was supposed to ensure we don't come "
1790
"here when scripts are executing.");
1791
NS_ASSERTION(
1792
mExecutor->IsInFlushLoop(),
1793
"How are we here if "
1794
"RunFlushLoop() didn't call ParseUntilBlocked() which is the "
1795
"only caller of this method?");
1796
mSpeculations.RemoveElementAt(0);
1797
if (mSpeculations.IsEmpty()) {
1798
// yes, it was still the only speculation. Now stop speculating
1799
// However, before telling the executor to read from stage, flush
1800
// any pending ops straight to the executor, because otherwise
1801
// they remain unflushed until we get more data from the network.
1802
mTreeBuilder->SetOpSink(mExecutor);
1803
mTreeBuilder->Flush(true);
1804
mTreeBuilder->SetOpSink(mExecutor->GetStage());
1805
mExecutor->StartReadingFromStage();
1806
mSpeculating = false;
1807
}
1808
}
1809
nsCOMPtr<nsIRunnable> event = new nsHtml5StreamParserContinuation(this);
1810
if (NS_FAILED(mEventTarget->Dispatch(event, nsIThread::DISPATCH_NORMAL))) {
1811
NS_WARNING("Failed to dispatch nsHtml5StreamParserContinuation");
1812
}
1813
// A stream event might run before this event runs, but that's harmless.
1814
#ifdef DEBUG
1815
mAtomTable.SetPermittedLookupEventTarget(mEventTarget);
1816
#endif
1817
}
1818
}
1819
1820
void nsHtml5StreamParser::ContinueAfterFailedCharsetSwitch() {
1821
NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
1822
nsCOMPtr<nsIRunnable> event = new nsHtml5StreamParserContinuation(this);
1823
if (NS_FAILED(mEventTarget->Dispatch(event, nsIThread::DISPATCH_NORMAL))) {
1824
NS_WARNING("Failed to dispatch nsHtml5StreamParserContinuation");
1825
}
1826
}
1827
1828
class nsHtml5TimerKungFu : public Runnable {
1829
private:
1830
nsHtml5StreamParserPtr mStreamParser;
1831
1832
public:
1833
explicit nsHtml5TimerKungFu(nsHtml5StreamParser* aStreamParser)
1834
: Runnable("nsHtml5TimerKungFu"), mStreamParser(aStreamParser) {}
1835
NS_IMETHOD Run() override {
1836
mozilla::MutexAutoLock flushTimerLock(mStreamParser->mFlushTimerMutex);
1837
if (mStreamParser->mFlushTimer) {
1838
mStreamParser->mFlushTimer->Cancel();
1839
mStreamParser->mFlushTimer = nullptr;
1840
}
1841
return NS_OK;
1842
}
1843
};
1844
1845
void nsHtml5StreamParser::DropTimer() {