Source code

Revision control

Other Tools

1
//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2
/* This Source Code Form is subject to the terms of the Mozilla Public
3
* License, v. 2.0. If a copy of the MPL was not distributed with this
4
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6
#include "ProtocolParser.h"
7
#include "LookupCache.h"
8
#include "nsNetCID.h"
9
#include "mozilla/Components.h"
10
#include "mozilla/Logging.h"
11
#include "prnetdb.h"
12
#include "prprf.h"
13
14
#include "nsUrlClassifierDBService.h"
15
#include "nsUrlClassifierUtils.h"
16
#include "nsPrintfCString.h"
17
#include "mozilla/Base64.h"
18
#include "RiceDeltaDecoder.h"
19
#include "mozilla/EndianUtils.h"
20
#include "mozilla/ErrorNames.h"
21
#include "mozilla/IntegerPrintfMacros.h"
22
23
// MOZ_LOG=UrlClassifierProtocolParser:5
24
mozilla::LazyLogModule gUrlClassifierProtocolParserLog(
25
"UrlClassifierProtocolParser");
26
#define PARSER_LOG(args) \
27
MOZ_LOG(gUrlClassifierProtocolParserLog, mozilla::LogLevel::Debug, args)
28
29
namespace mozilla {
30
namespace safebrowsing {
31
32
// Updates will fail if fed chunks larger than this
33
const uint32_t MAX_CHUNK_SIZE = (4 * 1024 * 1024);
34
// Updates will fail if the total number of tocuhed chunks is larger than this
35
const uint32_t MAX_CHUNK_RANGE = 1000000;
36
37
const uint32_t DOMAIN_SIZE = 4;
38
39
// Parse one stringified range of chunks of the form "n" or "n-m" from a
40
// comma-separated list of chunks. Upon return, 'begin' will point to the
41
// next range of chunks in the list of chunks.
42
static bool ParseChunkRange(nsACString::const_iterator& aBegin,
43
const nsACString::const_iterator& aEnd,
44
uint32_t* aFirst, uint32_t* aLast) {
45
nsACString::const_iterator iter = aBegin;
46
FindCharInReadable(',', iter, aEnd);
47
48
nsAutoCString element(Substring(aBegin, iter));
49
aBegin = iter;
50
if (aBegin != aEnd) aBegin++;
51
52
uint32_t numRead = PR_sscanf(element.get(), "%u-%u", aFirst, aLast);
53
if (numRead == 2) {
54
if (*aFirst > *aLast) {
55
uint32_t tmp = *aFirst;
56
*aFirst = *aLast;
57
*aLast = tmp;
58
}
59
return true;
60
}
61
62
if (numRead == 1) {
63
*aLast = *aFirst;
64
return true;
65
}
66
67
return false;
68
}
69
70
///////////////////////////////////////////////////////////////
71
// ProtocolParser implementation
72
73
ProtocolParser::ProtocolParser() : mUpdateStatus(NS_OK), mUpdateWaitSec(0) {}
74
75
ProtocolParser::~ProtocolParser() {}
76
77
nsresult ProtocolParser::Begin(const nsACString& aTable,
78
const nsTArray<nsCString>& aUpdateTables) {
79
// ProtocolParser objects should never be reused.
80
MOZ_ASSERT(mPending.IsEmpty());
81
MOZ_ASSERT(mTableUpdates.IsEmpty());
82
MOZ_ASSERT(mForwards.IsEmpty());
83
MOZ_ASSERT(mRequestedTables.IsEmpty());
84
MOZ_ASSERT(mTablesToReset.IsEmpty());
85
86
if (!aTable.IsEmpty()) {
87
SetCurrentTable(aTable);
88
}
89
SetRequestedTables(aUpdateTables);
90
91
return NS_OK;
92
}
93
94
RefPtr<TableUpdate> ProtocolParser::GetTableUpdate(const nsACString& aTable) {
95
for (uint32_t i = 0; i < mTableUpdates.Length(); i++) {
96
if (aTable.Equals(mTableUpdates[i]->TableName())) {
97
return mTableUpdates[i];
98
}
99
}
100
101
// We free automatically on destruction, ownership of these
102
// updates can be transferred to DBServiceWorker, which passes
103
// them back to Classifier when doing the updates, and that
104
// will free them.
105
RefPtr<TableUpdate> update = CreateTableUpdate(aTable);
106
mTableUpdates.AppendElement(update);
107
return update;
108
}
109
110
///////////////////////////////////////////////////////////////////////
111
// ProtocolParserV2
112
113
ProtocolParserV2::ProtocolParserV2()
114
: mState(PROTOCOL_STATE_CONTROL), mTableUpdate(nullptr) {}
115
116
ProtocolParserV2::~ProtocolParserV2() {}
117
118
void ProtocolParserV2::SetCurrentTable(const nsACString& aTable) {
119
RefPtr<TableUpdate> update = GetTableUpdate(aTable);
120
mTableUpdate = TableUpdate::Cast<TableUpdateV2>(update);
121
}
122
123
nsresult ProtocolParserV2::AppendStream(const nsACString& aData) {
124
if (NS_FAILED(mUpdateStatus)) return mUpdateStatus;
125
126
nsresult rv;
127
if (!mPending.Append(aData, mozilla::fallible)) {
128
return NS_ERROR_OUT_OF_MEMORY;
129
}
130
#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
131
mRawUpdate.Append(aData);
132
#endif
133
134
bool done = false;
135
while (!done) {
136
if (nsUrlClassifierDBService::ShutdownHasStarted()) {
137
return NS_ERROR_ABORT;
138
}
139
140
if (mState == PROTOCOL_STATE_CONTROL) {
141
rv = ProcessControl(&done);
142
} else if (mState == PROTOCOL_STATE_CHUNK) {
143
rv = ProcessChunk(&done);
144
} else {
145
NS_ERROR("Unexpected protocol state");
146
rv = NS_ERROR_FAILURE;
147
}
148
if (NS_FAILED(rv)) {
149
mUpdateStatus = rv;
150
return rv;
151
}
152
}
153
return NS_OK;
154
}
155
156
void ProtocolParserV2::End() {
157
// Inbound data has already been processed in every AppendStream() call.
158
mTableUpdate = nullptr;
159
}
160
161
nsresult ProtocolParserV2::ProcessControl(bool* aDone) {
162
nsresult rv;
163
164
nsAutoCString line;
165
*aDone = true;
166
while (NextLine(line)) {
167
PARSER_LOG(("Processing %s\n", line.get()));
168
169
if (StringBeginsWith(line, NS_LITERAL_CSTRING("i:"))) {
170
// Set the table name from the table header line.
171
SetCurrentTable(Substring(line, 2));
172
} else if (StringBeginsWith(line, NS_LITERAL_CSTRING("n:"))) {
173
if (PR_sscanf(line.get(), "n:%d", &mUpdateWaitSec) != 1) {
174
PARSER_LOG(("Error parsing n: '%s' (%d)", line.get(), mUpdateWaitSec));
175
return NS_ERROR_FAILURE;
176
}
177
} else if (line.EqualsLiteral("r:pleasereset")) {
178
PARSER_LOG(("All tables will be reset."));
179
mTablesToReset = mRequestedTables;
180
} else if (StringBeginsWith(line, NS_LITERAL_CSTRING("u:"))) {
181
rv = ProcessForward(line);
182
NS_ENSURE_SUCCESS(rv, rv);
183
} else if (StringBeginsWith(line, NS_LITERAL_CSTRING("a:")) ||
184
StringBeginsWith(line, NS_LITERAL_CSTRING("s:"))) {
185
rv = ProcessChunkControl(line);
186
NS_ENSURE_SUCCESS(rv, rv);
187
*aDone = false;
188
return NS_OK;
189
} else if (StringBeginsWith(line, NS_LITERAL_CSTRING("ad:")) ||
190
StringBeginsWith(line, NS_LITERAL_CSTRING("sd:"))) {
191
rv = ProcessExpirations(line);
192
NS_ENSURE_SUCCESS(rv, rv);
193
}
194
}
195
196
*aDone = true;
197
return NS_OK;
198
}
199
200
nsresult ProtocolParserV2::ProcessExpirations(const nsCString& aLine) {
201
if (!mTableUpdate) {
202
NS_WARNING("Got an expiration without a table.");
203
return NS_ERROR_FAILURE;
204
}
205
const nsACString& list = Substring(aLine, 3);
206
nsACString::const_iterator begin, end;
207
list.BeginReading(begin);
208
list.EndReading(end);
209
while (begin != end) {
210
uint32_t first, last;
211
if (ParseChunkRange(begin, end, &first, &last)) {
212
if (last < first) return NS_ERROR_FAILURE;
213
if (last - first > MAX_CHUNK_RANGE) return NS_ERROR_FAILURE;
214
for (uint32_t num = first; num <= last; num++) {
215
if (aLine[0] == 'a') {
216
nsresult rv = mTableUpdate->NewAddExpiration(num);
217
if (NS_FAILED(rv)) {
218
return rv;
219
}
220
} else {
221
nsresult rv = mTableUpdate->NewSubExpiration(num);
222
if (NS_FAILED(rv)) {
223
return rv;
224
}
225
}
226
}
227
} else {
228
return NS_ERROR_FAILURE;
229
}
230
}
231
return NS_OK;
232
}
233
234
nsresult ProtocolParserV2::ProcessChunkControl(const nsCString& aLine) {
235
if (!mTableUpdate) {
236
NS_WARNING("Got a chunk before getting a table.");
237
return NS_ERROR_FAILURE;
238
}
239
240
mState = PROTOCOL_STATE_CHUNK;
241
char command;
242
243
mChunkState.Clear();
244
245
if (PR_sscanf(aLine.get(), "%c:%d:%d:%d", &command, &mChunkState.num,
246
&mChunkState.hashSize, &mChunkState.length) != 4) {
247
NS_WARNING(("PR_sscanf failed"));
248
return NS_ERROR_FAILURE;
249
}
250
251
if (mChunkState.length > MAX_CHUNK_SIZE) {
252
NS_WARNING("Invalid length specified in update.");
253
return NS_ERROR_FAILURE;
254
}
255
256
if (!(mChunkState.hashSize == PREFIX_SIZE ||
257
mChunkState.hashSize == COMPLETE_SIZE)) {
258
NS_WARNING("Invalid hash size specified in update.");
259
return NS_ERROR_FAILURE;
260
}
261
262
if (StringEndsWith(mTableUpdate->TableName(),
263
NS_LITERAL_CSTRING("-shavar")) ||
264
StringEndsWith(mTableUpdate->TableName(),
265
NS_LITERAL_CSTRING("-simple"))) {
266
// Accommodate test tables ending in -simple for now.
267
mChunkState.type = (command == 'a') ? CHUNK_ADD : CHUNK_SUB;
268
} else if (StringEndsWith(mTableUpdate->TableName(),
269
NS_LITERAL_CSTRING("-digest256"))) {
270
mChunkState.type = (command == 'a') ? CHUNK_ADD_DIGEST : CHUNK_SUB_DIGEST;
271
}
272
nsresult rv;
273
switch (mChunkState.type) {
274
case CHUNK_ADD:
275
rv = mTableUpdate->NewAddChunk(mChunkState.num);
276
if (NS_FAILED(rv)) {
277
return rv;
278
}
279
break;
280
case CHUNK_SUB:
281
rv = mTableUpdate->NewSubChunk(mChunkState.num);
282
if (NS_FAILED(rv)) {
283
return rv;
284
}
285
break;
286
case CHUNK_ADD_DIGEST:
287
rv = mTableUpdate->NewAddChunk(mChunkState.num);
288
if (NS_FAILED(rv)) {
289
return rv;
290
}
291
break;
292
case CHUNK_SUB_DIGEST:
293
rv = mTableUpdate->NewSubChunk(mChunkState.num);
294
if (NS_FAILED(rv)) {
295
return rv;
296
}
297
break;
298
}
299
300
return NS_OK;
301
}
302
303
nsresult ProtocolParserV2::ProcessForward(const nsCString& aLine) {
304
const nsACString& forward = Substring(aLine, 2);
305
return AddForward(forward);
306
}
307
308
nsresult ProtocolParserV2::AddForward(const nsACString& aUrl) {
309
if (!mTableUpdate) {
310
NS_WARNING("Forward without a table name.");
311
return NS_ERROR_FAILURE;
312
}
313
314
ForwardedUpdate* forward = mForwards.AppendElement();
315
forward->table = mTableUpdate->TableName();
316
forward->url.Assign(aUrl);
317
318
return NS_OK;
319
}
320
321
nsresult ProtocolParserV2::ProcessChunk(bool* aDone) {
322
if (!mTableUpdate) {
323
NS_WARNING("Processing chunk without an active table.");
324
return NS_ERROR_FAILURE;
325
}
326
327
NS_ASSERTION(mChunkState.num != 0, "Must have a chunk number.");
328
329
if (mPending.Length() < mChunkState.length) {
330
*aDone = true;
331
return NS_OK;
332
}
333
334
// Pull the chunk out of the pending stream data.
335
nsAutoCString chunk;
336
chunk.Assign(Substring(mPending, 0, mChunkState.length));
337
mPending.Cut(0, mChunkState.length);
338
339
*aDone = false;
340
mState = PROTOCOL_STATE_CONTROL;
341
342
if (StringEndsWith(mTableUpdate->TableName(),
343
NS_LITERAL_CSTRING("-shavar"))) {
344
return ProcessShaChunk(chunk);
345
}
346
if (StringEndsWith(mTableUpdate->TableName(),
347
NS_LITERAL_CSTRING("-digest256"))) {
348
return ProcessDigestChunk(chunk);
349
}
350
return ProcessPlaintextChunk(chunk);
351
}
352
353
/**
354
* Process a plaintext chunk (currently only used in unit tests).
355
*/
356
nsresult ProtocolParserV2::ProcessPlaintextChunk(const nsACString& aChunk) {
357
if (!mTableUpdate) {
358
NS_WARNING("Chunk received with no table.");
359
return NS_ERROR_FAILURE;
360
}
361
362
PARSER_LOG(("Handling a %d-byte simple chunk", aChunk.Length()));
363
364
nsTArray<nsCString> lines;
365
ParseString(PromiseFlatCString(aChunk), '\n', lines);
366
367
// non-hashed tables need to be hashed
368
for (uint32_t i = 0; i < lines.Length(); i++) {
369
nsCString& line = lines[i];
370
371
if (mChunkState.type == CHUNK_ADD) {
372
if (mChunkState.hashSize == COMPLETE_SIZE) {
373
Completion hash;
374
hash.FromPlaintext(line);
375
nsresult rv = mTableUpdate->NewAddComplete(mChunkState.num, hash);
376
if (NS_FAILED(rv)) {
377
return rv;
378
}
379
} else {
380
NS_ASSERTION(mChunkState.hashSize == 4,
381
"Only 32- or 4-byte hashes can be used for add chunks.");
382
Prefix hash;
383
hash.FromPlaintext(line);
384
nsresult rv = mTableUpdate->NewAddPrefix(mChunkState.num, hash);
385
if (NS_FAILED(rv)) {
386
return rv;
387
}
388
}
389
} else {
390
nsCString::const_iterator begin, iter, end;
391
line.BeginReading(begin);
392
line.EndReading(end);
393
iter = begin;
394
uint32_t addChunk;
395
if (!FindCharInReadable(':', iter, end) ||
396
PR_sscanf(lines[i].get(), "%d:", &addChunk) != 1) {
397
NS_WARNING("Received sub chunk without associated add chunk.");
398
return NS_ERROR_FAILURE;
399
}
400
iter++;
401
402
if (mChunkState.hashSize == COMPLETE_SIZE) {
403
Completion hash;
404
hash.FromPlaintext(Substring(iter, end));
405
nsresult rv =
406
mTableUpdate->NewSubComplete(addChunk, hash, mChunkState.num);
407
if (NS_FAILED(rv)) {
408
return rv;
409
}
410
} else {
411
NS_ASSERTION(mChunkState.hashSize == 4,
412
"Only 32- or 4-byte hashes can be used for add chunks.");
413
Prefix hash;
414
hash.FromPlaintext(Substring(iter, end));
415
nsresult rv =
416
mTableUpdate->NewSubPrefix(addChunk, hash, mChunkState.num);
417
if (NS_FAILED(rv)) {
418
return rv;
419
}
420
}
421
}
422
}
423
424
return NS_OK;
425
}
426
427
nsresult ProtocolParserV2::ProcessShaChunk(const nsACString& aChunk) {
428
uint32_t start = 0;
429
while (start < aChunk.Length()) {
430
// First four bytes are the domain key.
431
Prefix domain;
432
domain.Assign(Substring(aChunk, start, DOMAIN_SIZE));
433
start += DOMAIN_SIZE;
434
435
// Then a count of entries.
436
uint8_t numEntries = static_cast<uint8_t>(aChunk[start]);
437
start++;
438
439
PARSER_LOG(
440
("Handling a %d-byte shavar chunk containing %u entries"
441
" for domain %X",
442
aChunk.Length(), numEntries, domain.ToUint32()));
443
444
nsresult rv;
445
if (mChunkState.type == CHUNK_ADD && mChunkState.hashSize == PREFIX_SIZE) {
446
rv = ProcessHostAdd(domain, numEntries, aChunk, &start);
447
} else if (mChunkState.type == CHUNK_ADD &&
448
mChunkState.hashSize == COMPLETE_SIZE) {
449
rv = ProcessHostAddComplete(numEntries, aChunk, &start);
450
} else if (mChunkState.type == CHUNK_SUB &&
451
mChunkState.hashSize == PREFIX_SIZE) {
452
rv = ProcessHostSub(domain, numEntries, aChunk, &start);
453
} else if (mChunkState.type == CHUNK_SUB &&
454
mChunkState.hashSize == COMPLETE_SIZE) {
455
rv = ProcessHostSubComplete(numEntries, aChunk, &start);
456
} else {
457
NS_WARNING("Unexpected chunk type/hash size!");
458
PARSER_LOG(("Got an unexpected chunk type/hash size: %s:%d",
459
mChunkState.type == CHUNK_ADD ? "add" : "sub",
460
mChunkState.hashSize));
461
return NS_ERROR_FAILURE;
462
}
463
NS_ENSURE_SUCCESS(rv, rv);
464
}
465
466
return NS_OK;
467
}
468
469
nsresult ProtocolParserV2::ProcessDigestChunk(const nsACString& aChunk) {
470
PARSER_LOG(("Handling a %d-byte digest256 chunk", aChunk.Length()));
471
472
if (mChunkState.type == CHUNK_ADD_DIGEST) {
473
return ProcessDigestAdd(aChunk);
474
}
475
if (mChunkState.type == CHUNK_SUB_DIGEST) {
476
return ProcessDigestSub(aChunk);
477
}
478
return NS_ERROR_UNEXPECTED;
479
}
480
481
nsresult ProtocolParserV2::ProcessDigestAdd(const nsACString& aChunk) {
482
MOZ_ASSERT(mTableUpdate);
483
// The ABNF format for add chunks is (HASH)+, where HASH is 32 bytes.
484
MOZ_ASSERT(aChunk.Length() % 32 == 0,
485
"Chunk length in bytes must be divisible by 4");
486
uint32_t start = 0;
487
while (start < aChunk.Length()) {
488
Completion hash;
489
hash.Assign(Substring(aChunk, start, COMPLETE_SIZE));
490
start += COMPLETE_SIZE;
491
nsresult rv = mTableUpdate->NewAddComplete(mChunkState.num, hash);
492
if (NS_FAILED(rv)) {
493
return rv;
494
}
495
}
496
return NS_OK;
497
}
498
499
nsresult ProtocolParserV2::ProcessDigestSub(const nsACString& aChunk) {
500
MOZ_ASSERT(mTableUpdate);
501
// The ABNF format for sub chunks is (ADDCHUNKNUM HASH)+, where ADDCHUNKNUM
502
// is a 4 byte chunk number, and HASH is 32 bytes.
503
MOZ_ASSERT(aChunk.Length() % 36 == 0,
504
"Chunk length in bytes must be divisible by 36");
505
uint32_t start = 0;
506
while (start < aChunk.Length()) {
507
// Read ADDCHUNKNUM
508
const nsACString& addChunkStr = Substring(aChunk, start, 4);
509
start += 4;
510
511
uint32_t addChunk;
512
memcpy(&addChunk, addChunkStr.BeginReading(), 4);
513
addChunk = PR_ntohl(addChunk);
514
515
// Read the hash
516
Completion hash;
517
hash.Assign(Substring(aChunk, start, COMPLETE_SIZE));
518
start += COMPLETE_SIZE;
519
520
nsresult rv = mTableUpdate->NewSubComplete(addChunk, hash, mChunkState.num);
521
if (NS_FAILED(rv)) {
522
return rv;
523
}
524
}
525
return NS_OK;
526
}
527
528
nsresult ProtocolParserV2::ProcessHostAdd(const Prefix& aDomain,
529
uint8_t aNumEntries,
530
const nsACString& aChunk,
531
uint32_t* aStart) {
532
MOZ_ASSERT(mTableUpdate);
533
NS_ASSERTION(mChunkState.hashSize == PREFIX_SIZE,
534
"ProcessHostAdd should only be called for prefix hashes.");
535
536
if (aNumEntries == 0) {
537
nsresult rv = mTableUpdate->NewAddPrefix(mChunkState.num, aDomain);
538
if (NS_FAILED(rv)) {
539
return rv;
540
}
541
return NS_OK;
542
}
543
544
if (*aStart + (PREFIX_SIZE * aNumEntries) > aChunk.Length()) {
545
NS_WARNING("Chunk is not long enough to contain the expected entries.");
546
return NS_ERROR_FAILURE;
547
}
548
549
for (uint8_t i = 0; i < aNumEntries; i++) {
550
Prefix hash;
551
hash.Assign(Substring(aChunk, *aStart, PREFIX_SIZE));
552
PARSER_LOG(("Add prefix %X", hash.ToUint32()));
553
nsresult rv = mTableUpdate->NewAddPrefix(mChunkState.num, hash);
554
if (NS_FAILED(rv)) {
555
return rv;
556
}
557
*aStart += PREFIX_SIZE;
558
}
559
560
return NS_OK;
561
}
562
563
nsresult ProtocolParserV2::ProcessHostSub(const Prefix& aDomain,
564
uint8_t aNumEntries,
565
const nsACString& aChunk,
566
uint32_t* aStart) {
567
MOZ_ASSERT(mTableUpdate);
568
NS_ASSERTION(mChunkState.hashSize == PREFIX_SIZE,
569
"ProcessHostSub should only be called for prefix hashes.");
570
571
if (aNumEntries == 0) {
572
if ((*aStart) + 4 > aChunk.Length()) {
573
NS_WARNING("Received a zero-entry sub chunk without an associated add.");
574
return NS_ERROR_FAILURE;
575
}
576
577
const nsACString& addChunkStr = Substring(aChunk, *aStart, 4);
578
*aStart += 4;
579
580
uint32_t addChunk;
581
memcpy(&addChunk, addChunkStr.BeginReading(), 4);
582
addChunk = PR_ntohl(addChunk);
583
584
PARSER_LOG(("Sub prefix (addchunk=%u)", addChunk));
585
nsresult rv =
586
mTableUpdate->NewSubPrefix(addChunk, aDomain, mChunkState.num);
587
if (NS_FAILED(rv)) {
588
return rv;
589
}
590
return NS_OK;
591
}
592
593
if (*aStart + ((PREFIX_SIZE + 4) * aNumEntries) > aChunk.Length()) {
594
NS_WARNING("Chunk is not long enough to contain the expected entries.");
595
return NS_ERROR_FAILURE;
596
}
597
598
for (uint8_t i = 0; i < aNumEntries; i++) {
599
const nsACString& addChunkStr = Substring(aChunk, *aStart, 4);
600
*aStart += 4;
601
602
uint32_t addChunk;
603
memcpy(&addChunk, addChunkStr.BeginReading(), 4);
604
addChunk = PR_ntohl(addChunk);
605
606
Prefix prefix;
607
prefix.Assign(Substring(aChunk, *aStart, PREFIX_SIZE));
608
*aStart += PREFIX_SIZE;
609
610
PARSER_LOG(("Sub prefix %X (addchunk=%u)", prefix.ToUint32(), addChunk));
611
nsresult rv = mTableUpdate->NewSubPrefix(addChunk, prefix, mChunkState.num);
612
if (NS_FAILED(rv)) {
613
return rv;
614
}
615
}
616
617
return NS_OK;
618
}
619
620
nsresult ProtocolParserV2::ProcessHostAddComplete(uint8_t aNumEntries,
621
const nsACString& aChunk,
622
uint32_t* aStart) {
623
MOZ_ASSERT(mTableUpdate);
624
NS_ASSERTION(
625
mChunkState.hashSize == COMPLETE_SIZE,
626
"ProcessHostAddComplete should only be called for complete hashes.");
627
628
if (aNumEntries == 0) {
629
// this is totally comprehensible.
630
// My sarcasm detector is going off!
631
NS_WARNING("Expected > 0 entries for a 32-byte hash add.");
632
return NS_OK;
633
}
634
635
if (*aStart + (COMPLETE_SIZE * aNumEntries) > aChunk.Length()) {
636
NS_WARNING("Chunk is not long enough to contain the expected entries.");
637
return NS_ERROR_FAILURE;
638
}
639
640
for (uint8_t i = 0; i < aNumEntries; i++) {
641
Completion hash;
642
hash.Assign(Substring(aChunk, *aStart, COMPLETE_SIZE));
643
nsresult rv = mTableUpdate->NewAddComplete(mChunkState.num, hash);
644
if (NS_FAILED(rv)) {
645
return rv;
646
}
647
*aStart += COMPLETE_SIZE;
648
}
649
650
return NS_OK;
651
}
652
653
nsresult ProtocolParserV2::ProcessHostSubComplete(uint8_t aNumEntries,
654
const nsACString& aChunk,
655
uint32_t* aStart) {
656
MOZ_ASSERT(mTableUpdate);
657
NS_ASSERTION(
658
mChunkState.hashSize == COMPLETE_SIZE,
659
"ProcessHostSubComplete should only be called for complete hashes.");
660
661
if (aNumEntries == 0) {
662
// this is totally comprehensible.
663
NS_WARNING("Expected > 0 entries for a 32-byte hash sub.");
664
return NS_OK;
665
}
666
667
if (*aStart + ((COMPLETE_SIZE + 4) * aNumEntries) > aChunk.Length()) {
668
NS_WARNING("Chunk is not long enough to contain the expected entries.");
669
return NS_ERROR_FAILURE;
670
}
671
672
for (uint8_t i = 0; i < aNumEntries; i++) {
673
Completion hash;
674
hash.Assign(Substring(aChunk, *aStart, COMPLETE_SIZE));
675
*aStart += COMPLETE_SIZE;
676
677
const nsACString& addChunkStr = Substring(aChunk, *aStart, 4);
678
*aStart += 4;
679
680
uint32_t addChunk;
681
memcpy(&addChunk, addChunkStr.BeginReading(), 4);
682
addChunk = PR_ntohl(addChunk);
683
684
nsresult rv = mTableUpdate->NewSubComplete(addChunk, hash, mChunkState.num);
685
if (NS_FAILED(rv)) {
686
return rv;
687
}
688
}
689
690
return NS_OK;
691
}
692
693
bool ProtocolParserV2::NextLine(nsACString& aLine) {
694
int32_t newline = mPending.FindChar('\n');
695
if (newline == kNotFound) {
696
return false;
697
}
698
aLine.Assign(Substring(mPending, 0, newline));
699
mPending.Cut(0, newline + 1);
700
return true;
701
}
702
703
RefPtr<TableUpdate> ProtocolParserV2::CreateTableUpdate(
704
const nsACString& aTableName) const {
705
return new TableUpdateV2(aTableName);
706
}
707
708
///////////////////////////////////////////////////////////////////////
709
// ProtocolParserProtobuf
710
711
ProtocolParserProtobuf::ProtocolParserProtobuf() {}
712
713
ProtocolParserProtobuf::~ProtocolParserProtobuf() {}
714
715
void ProtocolParserProtobuf::SetCurrentTable(const nsACString& aTable) {
716
// Should never occur.
717
MOZ_ASSERT_UNREACHABLE("SetCurrentTable shouldn't be called");
718
}
719
720
RefPtr<TableUpdate> ProtocolParserProtobuf::CreateTableUpdate(
721
const nsACString& aTableName) const {
722
return new TableUpdateV4(aTableName);
723
}
724
725
nsresult ProtocolParserProtobuf::AppendStream(const nsACString& aData) {
726
// Protobuf data cannot be parsed progressively. Just save the incoming data.
727
if (!mPending.Append(aData, mozilla::fallible)) {
728
return NS_ERROR_OUT_OF_MEMORY;
729
}
730
return NS_OK;
731
}
732
733
void ProtocolParserProtobuf::End() {
734
// mUpdateStatus will be updated to success as long as not all
735
// the responses are invalid.
736
mUpdateStatus = NS_ERROR_FAILURE;
737
738
FetchThreatListUpdatesResponse response;
739
if (!response.ParseFromArray(mPending.get(), mPending.Length())) {
740
NS_WARNING("ProtocolParserProtobuf failed parsing data.");
741
return;
742
}
743
744
auto minWaitDuration = response.minimum_wait_duration();
745
mUpdateWaitSec =
746
minWaitDuration.seconds() + minWaitDuration.nanos() / 1000000000;
747
748
for (int i = 0; i < response.list_update_responses_size(); i++) {
749
auto r = response.list_update_responses(i);
750
nsAutoCString listName;
751
nsresult rv = ProcessOneResponse(r, listName);
752
if (NS_SUCCEEDED(rv)) {
753
mUpdateStatus = rv;
754
} else {
755
nsAutoCString errorName;
756
mozilla::GetErrorName(rv, errorName);
757
NS_WARNING(nsPrintfCString("Failed to process one response for '%s': %s",
758
listName.get(), errorName.get())
759
.get());
760
if (!listName.IsEmpty()) {
761
PARSER_LOG(("Table %s will be reset.", listName.get()));
762
mTablesToReset.AppendElement(listName);
763
}
764
}
765
}
766
}
767
768
nsresult ProtocolParserProtobuf::ProcessOneResponse(
769
const ListUpdateResponse& aResponse, nsACString& aListName) {
770
MOZ_ASSERT(aListName.IsEmpty());
771
772
// A response must have a threat type.
773
if (!aResponse.has_threat_type()) {
774
NS_WARNING(
775
"Threat type not initialized. This seems to be an invalid response.");
776
return NS_ERROR_UC_PARSER_MISSING_PARAM;
777
}
778
779
nsUrlClassifierUtils* urlUtil = nsUrlClassifierUtils::GetInstance();
780
if (NS_WARN_IF(!urlUtil)) {
781
return NS_ERROR_FAILURE;
782
}
783
784
// Convert threat type to list name.
785
nsCString possibleListNames;
786
nsresult rv = urlUtil->ConvertThreatTypeToListNames(aResponse.threat_type(),
787
possibleListNames);
788
if (NS_FAILED(rv)) {
789
PARSER_LOG(("Threat type to list name conversion error: %d",
790
aResponse.threat_type()));
791
return NS_ERROR_UC_PARSER_UNKNOWN_THREAT;
792
}
793
794
// Match the table name we received with one of the ones we requested.
795
// We ignore the case where a threat type matches more than one list
796
// per provider and return the first one. See bug 1287059."
797
nsTArray<nsCString> possibleListNameArray;
798
Classifier::SplitTables(possibleListNames, possibleListNameArray);
799
for (auto possibleName : possibleListNameArray) {
800
if (mRequestedTables.Contains(possibleName)) {
801
aListName = possibleName;
802
break;
803
}
804
}
805
806
if (aListName.IsEmpty()) {
807
PARSER_LOG(
808
("We received an update for a list we didn't ask for. Ignoring it."));
809
return NS_ERROR_FAILURE;
810
}
811
812
// Test if this is a full update.
813
bool isFullUpdate = false;
814
if (aResponse.has_response_type()) {
815
isFullUpdate = aResponse.response_type() == ListUpdateResponse::FULL_UPDATE;
816
} else {
817
NS_WARNING("Response type not initialized.");
818
return NS_ERROR_UC_PARSER_MISSING_PARAM;
819
}
820
821
// Warn if there's no new state.
822
if (!aResponse.has_new_client_state()) {
823
NS_WARNING("New state not initialized.");
824
return NS_ERROR_UC_PARSER_MISSING_PARAM;
825
}
826
827
auto tu = GetTableUpdate(aListName);
828
auto tuV4 = TableUpdate::Cast<TableUpdateV4>(tu);
829
NS_ENSURE_TRUE(tuV4, NS_ERROR_FAILURE);
830
831
nsCString state(aResponse.new_client_state().c_str(),
832
aResponse.new_client_state().size());
833
tuV4->SetNewClientState(state);
834
835
if (aResponse.has_checksum()) {
836
tuV4->SetSHA256(aResponse.checksum().sha256());
837
}
838
839
PARSER_LOG(
840
("==== Update for threat type '%d' ====", aResponse.threat_type()));
841
PARSER_LOG(("* aListName: %s\n", PromiseFlatCString(aListName).get()));
842
PARSER_LOG(("* newState: %s\n", aResponse.new_client_state().c_str()));
843
PARSER_LOG(("* isFullUpdate: %s\n", (isFullUpdate ? "yes" : "no")));
844
PARSER_LOG(
845
("* hasChecksum: %s\n", (aResponse.has_checksum() ? "yes" : "no")));
846
PARSER_LOG(("* additions: %d\n", aResponse.additions().size()));
847
PARSER_LOG(("* removals: %d\n", aResponse.removals().size()));
848
849
tuV4->SetFullUpdate(isFullUpdate);
850
851
rv = ProcessAdditionOrRemoval(*tuV4, aResponse.additions(),
852
true /*aIsAddition*/);
853
NS_ENSURE_SUCCESS(rv, rv);
854
rv = ProcessAdditionOrRemoval(*tuV4, aResponse.removals(), false);
855
NS_ENSURE_SUCCESS(rv, rv);
856
857
PARSER_LOG(("\n\n"));
858
859
return NS_OK;
860
}
861
862
nsresult ProtocolParserProtobuf::ProcessAdditionOrRemoval(
863
TableUpdateV4& aTableUpdate, const ThreatEntrySetList& aUpdate,
864
bool aIsAddition) {
865
nsresult ret = NS_OK;
866
867
for (int i = 0; i < aUpdate.size(); i++) {
868
auto update = aUpdate.Get(i);
869
if (!update.has_compression_type()) {
870
NS_WARNING(nsPrintfCString("%s with no compression type.",
871
aIsAddition ? "Addition" : "Removal")
872
.get());
873
continue;
874
}
875
876
switch (update.compression_type()) {
877
case COMPRESSION_TYPE_UNSPECIFIED:
878
NS_WARNING("Unspecified compression type.");
879
break;
880
881
case RAW:
882
ret = (aIsAddition ? ProcessRawAddition(aTableUpdate, update)
883
: ProcessRawRemoval(aTableUpdate, update));
884
break;
885
886
case RICE:
887
ret = (aIsAddition ? ProcessEncodedAddition(aTableUpdate, update)
888
: ProcessEncodedRemoval(aTableUpdate, update));
889
break;
890
}
891
}
892
893
return ret;
894
}
895
896
nsresult ProtocolParserProtobuf::ProcessRawAddition(
897
TableUpdateV4& aTableUpdate, const ThreatEntrySet& aAddition) {
898
if (!aAddition.has_raw_hashes()) {
899
PARSER_LOG(("* No raw addition."));
900
return NS_OK;
901
}
902
903
auto rawHashes = aAddition.raw_hashes();
904
if (!rawHashes.has_prefix_size()) {
905
NS_WARNING("Raw hash has no prefix size");
906
return NS_OK;
907
}
908
909
uint32_t prefixSize = rawHashes.prefix_size();
910
MOZ_ASSERT(prefixSize >= PREFIX_SIZE && prefixSize <= COMPLETE_SIZE);
911
912
nsCString prefixes;
913
if (!prefixes.Assign(rawHashes.raw_hashes().c_str(),
914
rawHashes.raw_hashes().size(), mozilla::fallible)) {
915
return NS_ERROR_OUT_OF_MEMORY;
916
}
917
MOZ_ASSERT(prefixes.Length() % prefixSize == 0,
918
"PrefixString length must be a multiple of the prefix size.");
919
920
if (LOG_ENABLED()) {
921
PARSER_LOG((" Raw addition (%d-byte prefixes)", prefixSize));
922
PARSER_LOG((" - # of prefixes: %u", prefixes.Length() / prefixSize));
923
if (4 == prefixSize) {
924
uint32_t* fixedLengthPrefixes = (uint32_t*)prefixes.get();
925
PARSER_LOG((" - Memory address: 0x%p", fixedLengthPrefixes));
926
}
927
}
928
929
aTableUpdate.NewPrefixes(prefixSize, prefixes);
930
return NS_OK;
931
}
932
933
nsresult ProtocolParserProtobuf::ProcessRawRemoval(
934
TableUpdateV4& aTableUpdate, const ThreatEntrySet& aRemoval) {
935
if (!aRemoval.has_raw_indices()) {
936
NS_WARNING("A removal has no indices.");
937
return NS_OK;
938
}
939
940
// indices is an array of int32.
941
auto indices = aRemoval.raw_indices().indices();
942
PARSER_LOG(("* Raw removal"));
943
PARSER_LOG((" - # of removal: %d", indices.size()));
944
945
nsresult rv = aTableUpdate.NewRemovalIndices((const uint32_t*)indices.data(),
946
indices.size());
947
if (NS_FAILED(rv)) {
948
PARSER_LOG(("Failed to create new removal indices."));
949
return rv;
950
}
951
952
return NS_OK;
953
}
954
955
static nsresult DoRiceDeltaDecode(const RiceDeltaEncoding& aEncoding,
956
nsTArray<uint32_t>& aDecoded) {
957
if (aEncoding.num_entries() > 0 &&
958
(!aEncoding.has_rice_parameter() || !aEncoding.has_encoded_data())) {
959
PARSER_LOG(("Rice parameter or encoded data is missing."));
960
return NS_ERROR_UC_PARSER_MISSING_PARAM;
961
} else if (aEncoding.num_entries() == 0 && !aEncoding.has_first_value()) {
962
PARSER_LOG(("Missing first_value for an single-integer Rice encoding."));
963
return NS_ERROR_UC_PARSER_MISSING_VALUE;
964
}
965
966
auto first_value = aEncoding.has_first_value() ? aEncoding.first_value() : 0;
967
968
PARSER_LOG(("* Encoding info:"));
969
PARSER_LOG((" - First value: %" PRId64, first_value));
970
PARSER_LOG((" - Num of entries: %d", aEncoding.num_entries()));
971
PARSER_LOG((" - Rice parameter: %d", aEncoding.rice_parameter()));
972
973
// Set up the input buffer. Note that the bits should be read
974
// from LSB to MSB so that we in-place reverse the bits before
975
// feeding to the decoder.
976
auto encoded =
977
const_cast<RiceDeltaEncoding&>(aEncoding).mutable_encoded_data();
978
RiceDeltaDecoder decoder((uint8_t*)encoded->c_str(), encoded->size());
979
980
// Setup the output buffer. The "first value" is included in
981
// the output buffer.
982
if (!aDecoded.SetLength(aEncoding.num_entries() + 1, mozilla::fallible)) {
983
NS_WARNING("Not enough memory to decode the RiceDelta input.");
984
return NS_ERROR_OUT_OF_MEMORY;
985
}
986
987
// Decode!
988
bool rv = decoder.Decode(
989
aEncoding.rice_parameter(), first_value,
990
aEncoding.num_entries(), // # of entries (first value not included).
991
&aDecoded[0]);
992
993
NS_ENSURE_TRUE(rv, NS_ERROR_UC_PARSER_DECODE_FAILURE);
994
995
return NS_OK;
996
}
997
998
nsresult ProtocolParserProtobuf::ProcessEncodedAddition(
999
TableUpdateV4& aTableUpdate, const ThreatEntrySet& aAddition) {
1000
if (!aAddition.has_rice_hashes()) {
1001
PARSER_LOG(("* No rice encoded addition."));
1002
return NS_OK;
1003
}
1004
1005
nsTArray<uint32_t> decoded;
1006
nsresult rv = DoRiceDeltaDecode(aAddition.rice_hashes(), decoded);
1007
if (NS_FAILED(rv)) {
1008
PARSER_LOG(("Failed to parse encoded prefixes."));
1009
return rv;
1010
}
1011
1012
// Say we have the following raw prefixes
1013
// BE LE
1014
// 00 00 00 01 1 16777216
1015
// 00 00 02 00 512 131072
1016
// 00 03 00 00 196608 768
1017
// 04 00 00 00 67108864 4
1018
//
1019
// which can be treated as uint32 (big-endian) sorted in increasing order:
1020
//
1021
// [1, 512, 196608, 67108864]
1022
//
1024
// the following should be done prior to compression:
1025
//
1026
// 1) re-interpret in little-endian ==> [16777216, 131072, 768, 4]
1027
// 2) sort in increasing order ==> [4, 768, 131072, 16777216]
1028
//
1029
// In order to get the original byte stream from |decoded|
1030
// ([4, 768, 131072, 16777216] in this case), we have to:
1031
//
1032
// 1) sort in big-endian order ==> [16777216, 131072, 768, 4]
1033
// 2) copy each uint32 in little-endian to the result string
1034
//
1035
1036
// The 4-byte prefixes have to be re-sorted in Big-endian increasing order.
1037
struct CompareBigEndian {
1038
bool Equals(const uint32_t& aA, const uint32_t& aB) const {
1039
return aA == aB;
1040
}
1041
1042
bool LessThan(const uint32_t& aA, const uint32_t& aB) const {
1043
return NativeEndian::swapToBigEndian(aA) <
1044
NativeEndian::swapToBigEndian(aB);
1045
}
1046
};
1047
decoded.Sort(CompareBigEndian());
1048
1049
// The encoded prefixes are always 4 bytes.
1050
nsCString prefixes;
1051
if (!prefixes.SetCapacity(decoded.Length() * 4, mozilla::fallible)) {
1052
return NS_ERROR_OUT_OF_MEMORY;
1053
}
1054
for (size_t i = 0; i < decoded.Length(); i++) {
1055
// Note that the third argument is the number of elements we want
1056
// to copy (and swap) but not the number of bytes we want to copy.
1057
char p[4];
1058
NativeEndian::copyAndSwapToLittleEndian(p, &decoded[i], 1);
1059
prefixes.Append(p, 4);
1060
}
1061
1062
aTableUpdate.NewPrefixes(4, prefixes);
1063
return NS_OK;
1064
}
1065
1066
nsresult ProtocolParserProtobuf::ProcessEncodedRemoval(
1067
TableUpdateV4& aTableUpdate, const ThreatEntrySet& aRemoval) {
1068
if (!aRemoval.has_rice_indices()) {
1069
PARSER_LOG(("* No rice encoded removal."));
1070
return NS_OK;
1071
}
1072
1073
nsTArray<uint32_t> decoded;
1074
nsresult rv = DoRiceDeltaDecode(aRemoval.rice_indices(), decoded);
1075
if (NS_FAILED(rv)) {
1076
PARSER_LOG(("Failed to decode encoded removal indices."));
1077
return rv;
1078
}
1079
1080
// The encoded prefixes are always 4 bytes.
1081
rv = aTableUpdate.NewRemovalIndices(&decoded[0], decoded.Length());
1082
if (NS_FAILED(rv)) {
1083
PARSER_LOG(("Failed to create new removal indices."));
1084
return rv;
1085
}
1086
1087
return NS_OK;
1088
}
1089
1090
} // namespace safebrowsing
1091
} // namespace mozilla