Source code

Revision control

Other Tools

1
//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2
/* This Source Code Form is subject to the terms of the Mozilla Public
3
* License, v. 2.0. If a copy of the MPL was not distributed with this
4
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6
#include "ProtocolParser.h"
7
#include "LookupCache.h"
8
#include "nsNetCID.h"
9
#include "mozilla/Components.h"
10
#include "mozilla/Logging.h"
11
#include "prnetdb.h"
12
#include "prprf.h"
13
14
#include "nsUrlClassifierDBService.h"
15
#include "nsUrlClassifierUtils.h"
16
#include "nsPrintfCString.h"
17
#include "mozilla/Base64.h"
18
#include "RiceDeltaDecoder.h"
19
#include "mozilla/EndianUtils.h"
20
#include "mozilla/ErrorNames.h"
21
#include "mozilla/IntegerPrintfMacros.h"
22
23
// MOZ_LOG=UrlClassifierProtocolParser:5
24
mozilla::LazyLogModule gUrlClassifierProtocolParserLog(
25
"UrlClassifierProtocolParser");
26
#define PARSER_LOG(args) \
27
MOZ_LOG(gUrlClassifierProtocolParserLog, mozilla::LogLevel::Debug, args)
28
29
namespace mozilla {
30
namespace safebrowsing {
31
32
// Updates will fail if fed chunks larger than this
33
const uint32_t MAX_CHUNK_SIZE = (1024 * 1024);
34
// Updates will fail if the total number of tocuhed chunks is larger than this
35
const uint32_t MAX_CHUNK_RANGE = 1000000;
36
37
const uint32_t DOMAIN_SIZE = 4;
38
39
// Parse one stringified range of chunks of the form "n" or "n-m" from a
40
// comma-separated list of chunks. Upon return, 'begin' will point to the
41
// next range of chunks in the list of chunks.
42
static bool ParseChunkRange(nsACString::const_iterator& aBegin,
43
const nsACString::const_iterator& aEnd,
44
uint32_t* aFirst, uint32_t* aLast) {
45
nsACString::const_iterator iter = aBegin;
46
FindCharInReadable(',', iter, aEnd);
47
48
nsAutoCString element(Substring(aBegin, iter));
49
aBegin = iter;
50
if (aBegin != aEnd) aBegin++;
51
52
uint32_t numRead = PR_sscanf(element.get(), "%u-%u", aFirst, aLast);
53
if (numRead == 2) {
54
if (*aFirst > *aLast) {
55
uint32_t tmp = *aFirst;
56
*aFirst = *aLast;
57
*aLast = tmp;
58
}
59
return true;
60
}
61
62
if (numRead == 1) {
63
*aLast = *aFirst;
64
return true;
65
}
66
67
return false;
68
}
69
70
///////////////////////////////////////////////////////////////
71
// ProtocolParser implementation
72
73
ProtocolParser::ProtocolParser() : mUpdateStatus(NS_OK), mUpdateWaitSec(0) {}
74
75
ProtocolParser::~ProtocolParser() {}
76
77
nsresult ProtocolParser::Begin(const nsACString& aTable,
78
const nsTArray<nsCString>& aUpdateTables) {
79
// ProtocolParser objects should never be reused.
80
MOZ_ASSERT(mPending.IsEmpty());
81
MOZ_ASSERT(mTableUpdates.IsEmpty());
82
MOZ_ASSERT(mForwards.IsEmpty());
83
MOZ_ASSERT(mRequestedTables.IsEmpty());
84
MOZ_ASSERT(mTablesToReset.IsEmpty());
85
86
if (!aTable.IsEmpty()) {
87
SetCurrentTable(aTable);
88
}
89
SetRequestedTables(aUpdateTables);
90
91
return NS_OK;
92
}
93
94
RefPtr<TableUpdate> ProtocolParser::GetTableUpdate(const nsACString& aTable) {
95
for (uint32_t i = 0; i < mTableUpdates.Length(); i++) {
96
if (aTable.Equals(mTableUpdates[i]->TableName())) {
97
return mTableUpdates[i];
98
}
99
}
100
101
// We free automatically on destruction, ownership of these
102
// updates can be transferred to DBServiceWorker, which passes
103
// them back to Classifier when doing the updates, and that
104
// will free them.
105
RefPtr<TableUpdate> update = CreateTableUpdate(aTable);
106
mTableUpdates.AppendElement(update);
107
return update;
108
}
109
110
///////////////////////////////////////////////////////////////////////
111
// ProtocolParserV2
112
113
ProtocolParserV2::ProtocolParserV2()
114
: mState(PROTOCOL_STATE_CONTROL), mTableUpdate(nullptr) {}
115
116
ProtocolParserV2::~ProtocolParserV2() {}
117
118
void ProtocolParserV2::SetCurrentTable(const nsACString& aTable) {
119
RefPtr<TableUpdate> update = GetTableUpdate(aTable);
120
mTableUpdate = TableUpdate::Cast<TableUpdateV2>(update);
121
}
122
123
nsresult ProtocolParserV2::AppendStream(const nsACString& aData) {
124
if (NS_FAILED(mUpdateStatus)) return mUpdateStatus;
125
126
nsresult rv;
127
mPending.Append(aData);
128
#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
129
mRawUpdate.Append(aData);
130
#endif
131
132
bool done = false;
133
while (!done) {
134
if (nsUrlClassifierDBService::ShutdownHasStarted()) {
135
return NS_ERROR_ABORT;
136
}
137
138
if (mState == PROTOCOL_STATE_CONTROL) {
139
rv = ProcessControl(&done);
140
} else if (mState == PROTOCOL_STATE_CHUNK) {
141
rv = ProcessChunk(&done);
142
} else {
143
NS_ERROR("Unexpected protocol state");
144
rv = NS_ERROR_FAILURE;
145
}
146
if (NS_FAILED(rv)) {
147
mUpdateStatus = rv;
148
return rv;
149
}
150
}
151
return NS_OK;
152
}
153
154
void ProtocolParserV2::End() {
155
// Inbound data has already been processed in every AppendStream() call.
156
mTableUpdate = nullptr;
157
}
158
159
nsresult ProtocolParserV2::ProcessControl(bool* aDone) {
160
nsresult rv;
161
162
nsAutoCString line;
163
*aDone = true;
164
while (NextLine(line)) {
165
PARSER_LOG(("Processing %s\n", line.get()));
166
167
if (StringBeginsWith(line, NS_LITERAL_CSTRING("i:"))) {
168
// Set the table name from the table header line.
169
SetCurrentTable(Substring(line, 2));
170
} else if (StringBeginsWith(line, NS_LITERAL_CSTRING("n:"))) {
171
if (PR_sscanf(line.get(), "n:%d", &mUpdateWaitSec) != 1) {
172
PARSER_LOG(("Error parsing n: '%s' (%d)", line.get(), mUpdateWaitSec));
173
return NS_ERROR_FAILURE;
174
}
175
} else if (line.EqualsLiteral("r:pleasereset")) {
176
PARSER_LOG(("All tables will be reset."));
177
mTablesToReset = mRequestedTables;
178
} else if (StringBeginsWith(line, NS_LITERAL_CSTRING("u:"))) {
179
rv = ProcessForward(line);
180
NS_ENSURE_SUCCESS(rv, rv);
181
} else if (StringBeginsWith(line, NS_LITERAL_CSTRING("a:")) ||
182
StringBeginsWith(line, NS_LITERAL_CSTRING("s:"))) {
183
rv = ProcessChunkControl(line);
184
NS_ENSURE_SUCCESS(rv, rv);
185
*aDone = false;
186
return NS_OK;
187
} else if (StringBeginsWith(line, NS_LITERAL_CSTRING("ad:")) ||
188
StringBeginsWith(line, NS_LITERAL_CSTRING("sd:"))) {
189
rv = ProcessExpirations(line);
190
NS_ENSURE_SUCCESS(rv, rv);
191
}
192
}
193
194
*aDone = true;
195
return NS_OK;
196
}
197
198
nsresult ProtocolParserV2::ProcessExpirations(const nsCString& aLine) {
199
if (!mTableUpdate) {
200
NS_WARNING("Got an expiration without a table.");
201
return NS_ERROR_FAILURE;
202
}
203
const nsACString& list = Substring(aLine, 3);
204
nsACString::const_iterator begin, end;
205
list.BeginReading(begin);
206
list.EndReading(end);
207
while (begin != end) {
208
uint32_t first, last;
209
if (ParseChunkRange(begin, end, &first, &last)) {
210
if (last < first) return NS_ERROR_FAILURE;
211
if (last - first > MAX_CHUNK_RANGE) return NS_ERROR_FAILURE;
212
for (uint32_t num = first; num <= last; num++) {
213
if (aLine[0] == 'a') {
214
nsresult rv = mTableUpdate->NewAddExpiration(num);
215
if (NS_FAILED(rv)) {
216
return rv;
217
}
218
} else {
219
nsresult rv = mTableUpdate->NewSubExpiration(num);
220
if (NS_FAILED(rv)) {
221
return rv;
222
}
223
}
224
}
225
} else {
226
return NS_ERROR_FAILURE;
227
}
228
}
229
return NS_OK;
230
}
231
232
nsresult ProtocolParserV2::ProcessChunkControl(const nsCString& aLine) {
233
if (!mTableUpdate) {
234
NS_WARNING("Got a chunk before getting a table.");
235
return NS_ERROR_FAILURE;
236
}
237
238
mState = PROTOCOL_STATE_CHUNK;
239
char command;
240
241
mChunkState.Clear();
242
243
if (PR_sscanf(aLine.get(), "%c:%d:%d:%d", &command, &mChunkState.num,
244
&mChunkState.hashSize, &mChunkState.length) != 4) {
245
NS_WARNING(("PR_sscanf failed"));
246
return NS_ERROR_FAILURE;
247
}
248
249
if (mChunkState.length > MAX_CHUNK_SIZE) {
250
NS_WARNING("Invalid length specified in update.");
251
return NS_ERROR_FAILURE;
252
}
253
254
if (!(mChunkState.hashSize == PREFIX_SIZE ||
255
mChunkState.hashSize == COMPLETE_SIZE)) {
256
NS_WARNING("Invalid hash size specified in update.");
257
return NS_ERROR_FAILURE;
258
}
259
260
if (StringEndsWith(mTableUpdate->TableName(),
261
NS_LITERAL_CSTRING("-shavar")) ||
262
StringEndsWith(mTableUpdate->TableName(),
263
NS_LITERAL_CSTRING("-simple"))) {
264
// Accommodate test tables ending in -simple for now.
265
mChunkState.type = (command == 'a') ? CHUNK_ADD : CHUNK_SUB;
266
} else if (StringEndsWith(mTableUpdate->TableName(),
267
NS_LITERAL_CSTRING("-digest256"))) {
268
mChunkState.type = (command == 'a') ? CHUNK_ADD_DIGEST : CHUNK_SUB_DIGEST;
269
}
270
nsresult rv;
271
switch (mChunkState.type) {
272
case CHUNK_ADD:
273
rv = mTableUpdate->NewAddChunk(mChunkState.num);
274
if (NS_FAILED(rv)) {
275
return rv;
276
}
277
break;
278
case CHUNK_SUB:
279
rv = mTableUpdate->NewSubChunk(mChunkState.num);
280
if (NS_FAILED(rv)) {
281
return rv;
282
}
283
break;
284
case CHUNK_ADD_DIGEST:
285
rv = mTableUpdate->NewAddChunk(mChunkState.num);
286
if (NS_FAILED(rv)) {
287
return rv;
288
}
289
break;
290
case CHUNK_SUB_DIGEST:
291
rv = mTableUpdate->NewSubChunk(mChunkState.num);
292
if (NS_FAILED(rv)) {
293
return rv;
294
}
295
break;
296
}
297
298
return NS_OK;
299
}
300
301
nsresult ProtocolParserV2::ProcessForward(const nsCString& aLine) {
302
const nsACString& forward = Substring(aLine, 2);
303
return AddForward(forward);
304
}
305
306
nsresult ProtocolParserV2::AddForward(const nsACString& aUrl) {
307
if (!mTableUpdate) {
308
NS_WARNING("Forward without a table name.");
309
return NS_ERROR_FAILURE;
310
}
311
312
ForwardedUpdate* forward = mForwards.AppendElement();
313
forward->table = mTableUpdate->TableName();
314
forward->url.Assign(aUrl);
315
316
return NS_OK;
317
}
318
319
nsresult ProtocolParserV2::ProcessChunk(bool* aDone) {
320
if (!mTableUpdate) {
321
NS_WARNING("Processing chunk without an active table.");
322
return NS_ERROR_FAILURE;
323
}
324
325
NS_ASSERTION(mChunkState.num != 0, "Must have a chunk number.");
326
327
if (mPending.Length() < mChunkState.length) {
328
*aDone = true;
329
return NS_OK;
330
}
331
332
// Pull the chunk out of the pending stream data.
333
nsAutoCString chunk;
334
chunk.Assign(Substring(mPending, 0, mChunkState.length));
335
mPending.Cut(0, mChunkState.length);
336
337
*aDone = false;
338
mState = PROTOCOL_STATE_CONTROL;
339
340
if (StringEndsWith(mTableUpdate->TableName(),
341
NS_LITERAL_CSTRING("-shavar"))) {
342
return ProcessShaChunk(chunk);
343
}
344
if (StringEndsWith(mTableUpdate->TableName(),
345
NS_LITERAL_CSTRING("-digest256"))) {
346
return ProcessDigestChunk(chunk);
347
}
348
return ProcessPlaintextChunk(chunk);
349
}
350
351
/**
352
* Process a plaintext chunk (currently only used in unit tests).
353
*/
354
nsresult ProtocolParserV2::ProcessPlaintextChunk(const nsACString& aChunk) {
355
if (!mTableUpdate) {
356
NS_WARNING("Chunk received with no table.");
357
return NS_ERROR_FAILURE;
358
}
359
360
PARSER_LOG(("Handling a %d-byte simple chunk", aChunk.Length()));
361
362
nsTArray<nsCString> lines;
363
ParseString(PromiseFlatCString(aChunk), '\n', lines);
364
365
// non-hashed tables need to be hashed
366
for (uint32_t i = 0; i < lines.Length(); i++) {
367
nsCString& line = lines[i];
368
369
if (mChunkState.type == CHUNK_ADD) {
370
if (mChunkState.hashSize == COMPLETE_SIZE) {
371
Completion hash;
372
hash.FromPlaintext(line);
373
nsresult rv = mTableUpdate->NewAddComplete(mChunkState.num, hash);
374
if (NS_FAILED(rv)) {
375
return rv;
376
}
377
} else {
378
NS_ASSERTION(mChunkState.hashSize == 4,
379
"Only 32- or 4-byte hashes can be used for add chunks.");
380
Prefix hash;
381
hash.FromPlaintext(line);
382
nsresult rv = mTableUpdate->NewAddPrefix(mChunkState.num, hash);
383
if (NS_FAILED(rv)) {
384
return rv;
385
}
386
}
387
} else {
388
nsCString::const_iterator begin, iter, end;
389
line.BeginReading(begin);
390
line.EndReading(end);
391
iter = begin;
392
uint32_t addChunk;
393
if (!FindCharInReadable(':', iter, end) ||
394
PR_sscanf(lines[i].get(), "%d:", &addChunk) != 1) {
395
NS_WARNING("Received sub chunk without associated add chunk.");
396
return NS_ERROR_FAILURE;
397
}
398
iter++;
399
400
if (mChunkState.hashSize == COMPLETE_SIZE) {
401
Completion hash;
402
hash.FromPlaintext(Substring(iter, end));
403
nsresult rv =
404
mTableUpdate->NewSubComplete(addChunk, hash, mChunkState.num);
405
if (NS_FAILED(rv)) {
406
return rv;
407
}
408
} else {
409
NS_ASSERTION(mChunkState.hashSize == 4,
410
"Only 32- or 4-byte hashes can be used for add chunks.");
411
Prefix hash;
412
hash.FromPlaintext(Substring(iter, end));
413
nsresult rv =
414
mTableUpdate->NewSubPrefix(addChunk, hash, mChunkState.num);
415
if (NS_FAILED(rv)) {
416
return rv;
417
}
418
}
419
}
420
}
421
422
return NS_OK;
423
}
424
425
nsresult ProtocolParserV2::ProcessShaChunk(const nsACString& aChunk) {
426
uint32_t start = 0;
427
while (start < aChunk.Length()) {
428
// First four bytes are the domain key.
429
Prefix domain;
430
domain.Assign(Substring(aChunk, start, DOMAIN_SIZE));
431
start += DOMAIN_SIZE;
432
433
// Then a count of entries.
434
uint8_t numEntries = static_cast<uint8_t>(aChunk[start]);
435
start++;
436
437
PARSER_LOG(
438
("Handling a %d-byte shavar chunk containing %u entries"
439
" for domain %X",
440
aChunk.Length(), numEntries, domain.ToUint32()));
441
442
nsresult rv;
443
if (mChunkState.type == CHUNK_ADD && mChunkState.hashSize == PREFIX_SIZE) {
444
rv = ProcessHostAdd(domain, numEntries, aChunk, &start);
445
} else if (mChunkState.type == CHUNK_ADD &&
446
mChunkState.hashSize == COMPLETE_SIZE) {
447
rv = ProcessHostAddComplete(numEntries, aChunk, &start);
448
} else if (mChunkState.type == CHUNK_SUB &&
449
mChunkState.hashSize == PREFIX_SIZE) {
450
rv = ProcessHostSub(domain, numEntries, aChunk, &start);
451
} else if (mChunkState.type == CHUNK_SUB &&
452
mChunkState.hashSize == COMPLETE_SIZE) {
453
rv = ProcessHostSubComplete(numEntries, aChunk, &start);
454
} else {
455
NS_WARNING("Unexpected chunk type/hash size!");
456
PARSER_LOG(("Got an unexpected chunk type/hash size: %s:%d",
457
mChunkState.type == CHUNK_ADD ? "add" : "sub",
458
mChunkState.hashSize));
459
return NS_ERROR_FAILURE;
460
}
461
NS_ENSURE_SUCCESS(rv, rv);
462
}
463
464
return NS_OK;
465
}
466
467
nsresult ProtocolParserV2::ProcessDigestChunk(const nsACString& aChunk) {
468
PARSER_LOG(("Handling a %d-byte digest256 chunk", aChunk.Length()));
469
470
if (mChunkState.type == CHUNK_ADD_DIGEST) {
471
return ProcessDigestAdd(aChunk);
472
}
473
if (mChunkState.type == CHUNK_SUB_DIGEST) {
474
return ProcessDigestSub(aChunk);
475
}
476
return NS_ERROR_UNEXPECTED;
477
}
478
479
nsresult ProtocolParserV2::ProcessDigestAdd(const nsACString& aChunk) {
480
MOZ_ASSERT(mTableUpdate);
481
// The ABNF format for add chunks is (HASH)+, where HASH is 32 bytes.
482
MOZ_ASSERT(aChunk.Length() % 32 == 0,
483
"Chunk length in bytes must be divisible by 4");
484
uint32_t start = 0;
485
while (start < aChunk.Length()) {
486
Completion hash;
487
hash.Assign(Substring(aChunk, start, COMPLETE_SIZE));
488
start += COMPLETE_SIZE;
489
nsresult rv = mTableUpdate->NewAddComplete(mChunkState.num, hash);
490
if (NS_FAILED(rv)) {
491
return rv;
492
}
493
}
494
return NS_OK;
495
}
496
497
nsresult ProtocolParserV2::ProcessDigestSub(const nsACString& aChunk) {
498
MOZ_ASSERT(mTableUpdate);
499
// The ABNF format for sub chunks is (ADDCHUNKNUM HASH)+, where ADDCHUNKNUM
500
// is a 4 byte chunk number, and HASH is 32 bytes.
501
MOZ_ASSERT(aChunk.Length() % 36 == 0,
502
"Chunk length in bytes must be divisible by 36");
503
uint32_t start = 0;
504
while (start < aChunk.Length()) {
505
// Read ADDCHUNKNUM
506
const nsACString& addChunkStr = Substring(aChunk, start, 4);
507
start += 4;
508
509
uint32_t addChunk;
510
memcpy(&addChunk, addChunkStr.BeginReading(), 4);
511
addChunk = PR_ntohl(addChunk);
512
513
// Read the hash
514
Completion hash;
515
hash.Assign(Substring(aChunk, start, COMPLETE_SIZE));
516
start += COMPLETE_SIZE;
517
518
nsresult rv = mTableUpdate->NewSubComplete(addChunk, hash, mChunkState.num);
519
if (NS_FAILED(rv)) {
520
return rv;
521
}
522
}
523
return NS_OK;
524
}
525
526
nsresult ProtocolParserV2::ProcessHostAdd(const Prefix& aDomain,
527
uint8_t aNumEntries,
528
const nsACString& aChunk,
529
uint32_t* aStart) {
530
MOZ_ASSERT(mTableUpdate);
531
NS_ASSERTION(mChunkState.hashSize == PREFIX_SIZE,
532
"ProcessHostAdd should only be called for prefix hashes.");
533
534
if (aNumEntries == 0) {
535
nsresult rv = mTableUpdate->NewAddPrefix(mChunkState.num, aDomain);
536
if (NS_FAILED(rv)) {
537
return rv;
538
}
539
return NS_OK;
540
}
541
542
if (*aStart + (PREFIX_SIZE * aNumEntries) > aChunk.Length()) {
543
NS_WARNING("Chunk is not long enough to contain the expected entries.");
544
return NS_ERROR_FAILURE;
545
}
546
547
for (uint8_t i = 0; i < aNumEntries; i++) {
548
Prefix hash;
549
hash.Assign(Substring(aChunk, *aStart, PREFIX_SIZE));
550
PARSER_LOG(("Add prefix %X", hash.ToUint32()));
551
nsresult rv = mTableUpdate->NewAddPrefix(mChunkState.num, hash);
552
if (NS_FAILED(rv)) {
553
return rv;
554
}
555
*aStart += PREFIX_SIZE;
556
}
557
558
return NS_OK;
559
}
560
561
nsresult ProtocolParserV2::ProcessHostSub(const Prefix& aDomain,
562
uint8_t aNumEntries,
563
const nsACString& aChunk,
564
uint32_t* aStart) {
565
MOZ_ASSERT(mTableUpdate);
566
NS_ASSERTION(mChunkState.hashSize == PREFIX_SIZE,
567
"ProcessHostSub should only be called for prefix hashes.");
568
569
if (aNumEntries == 0) {
570
if ((*aStart) + 4 > aChunk.Length()) {
571
NS_WARNING("Received a zero-entry sub chunk without an associated add.");
572
return NS_ERROR_FAILURE;
573
}
574
575
const nsACString& addChunkStr = Substring(aChunk, *aStart, 4);
576
*aStart += 4;
577
578
uint32_t addChunk;
579
memcpy(&addChunk, addChunkStr.BeginReading(), 4);
580
addChunk = PR_ntohl(addChunk);
581
582
PARSER_LOG(("Sub prefix (addchunk=%u)", addChunk));
583
nsresult rv =
584
mTableUpdate->NewSubPrefix(addChunk, aDomain, mChunkState.num);
585
if (NS_FAILED(rv)) {
586
return rv;
587
}
588
return NS_OK;
589
}
590
591
if (*aStart + ((PREFIX_SIZE + 4) * aNumEntries) > aChunk.Length()) {
592
NS_WARNING("Chunk is not long enough to contain the expected entries.");
593
return NS_ERROR_FAILURE;
594
}
595
596
for (uint8_t i = 0; i < aNumEntries; i++) {
597
const nsACString& addChunkStr = Substring(aChunk, *aStart, 4);
598
*aStart += 4;
599
600
uint32_t addChunk;
601
memcpy(&addChunk, addChunkStr.BeginReading(), 4);
602
addChunk = PR_ntohl(addChunk);
603
604
Prefix prefix;
605
prefix.Assign(Substring(aChunk, *aStart, PREFIX_SIZE));
606
*aStart += PREFIX_SIZE;
607
608
PARSER_LOG(("Sub prefix %X (addchunk=%u)", prefix.ToUint32(), addChunk));
609
nsresult rv = mTableUpdate->NewSubPrefix(addChunk, prefix, mChunkState.num);
610
if (NS_FAILED(rv)) {
611
return rv;
612
}
613
}
614
615
return NS_OK;
616
}
617
618
nsresult ProtocolParserV2::ProcessHostAddComplete(uint8_t aNumEntries,
619
const nsACString& aChunk,
620
uint32_t* aStart) {
621
MOZ_ASSERT(mTableUpdate);
622
NS_ASSERTION(
623
mChunkState.hashSize == COMPLETE_SIZE,
624
"ProcessHostAddComplete should only be called for complete hashes.");
625
626
if (aNumEntries == 0) {
627
// this is totally comprehensible.
628
// My sarcasm detector is going off!
629
NS_WARNING("Expected > 0 entries for a 32-byte hash add.");
630
return NS_OK;
631
}
632
633
if (*aStart + (COMPLETE_SIZE * aNumEntries) > aChunk.Length()) {
634
NS_WARNING("Chunk is not long enough to contain the expected entries.");
635
return NS_ERROR_FAILURE;
636
}
637
638
for (uint8_t i = 0; i < aNumEntries; i++) {
639
Completion hash;
640
hash.Assign(Substring(aChunk, *aStart, COMPLETE_SIZE));
641
nsresult rv = mTableUpdate->NewAddComplete(mChunkState.num, hash);
642
if (NS_FAILED(rv)) {
643
return rv;
644
}
645
*aStart += COMPLETE_SIZE;
646
}
647
648
return NS_OK;
649
}
650
651
nsresult ProtocolParserV2::ProcessHostSubComplete(uint8_t aNumEntries,
652
const nsACString& aChunk,
653
uint32_t* aStart) {
654
MOZ_ASSERT(mTableUpdate);
655
NS_ASSERTION(
656
mChunkState.hashSize == COMPLETE_SIZE,
657
"ProcessHostSubComplete should only be called for complete hashes.");
658
659
if (aNumEntries == 0) {
660
// this is totally comprehensible.
661
NS_WARNING("Expected > 0 entries for a 32-byte hash sub.");
662
return NS_OK;
663
}
664
665
if (*aStart + ((COMPLETE_SIZE + 4) * aNumEntries) > aChunk.Length()) {
666
NS_WARNING("Chunk is not long enough to contain the expected entries.");
667
return NS_ERROR_FAILURE;
668
}
669
670
for (uint8_t i = 0; i < aNumEntries; i++) {
671
Completion hash;
672
hash.Assign(Substring(aChunk, *aStart, COMPLETE_SIZE));
673
*aStart += COMPLETE_SIZE;
674
675
const nsACString& addChunkStr = Substring(aChunk, *aStart, 4);
676
*aStart += 4;
677
678
uint32_t addChunk;
679
memcpy(&addChunk, addChunkStr.BeginReading(), 4);
680
addChunk = PR_ntohl(addChunk);
681
682
nsresult rv = mTableUpdate->NewSubComplete(addChunk, hash, mChunkState.num);
683
if (NS_FAILED(rv)) {
684
return rv;
685
}
686
}
687
688
return NS_OK;
689
}
690
691
bool ProtocolParserV2::NextLine(nsACString& aLine) {
692
int32_t newline = mPending.FindChar('\n');
693
if (newline == kNotFound) {
694
return false;
695
}
696
aLine.Assign(Substring(mPending, 0, newline));
697
mPending.Cut(0, newline + 1);
698
return true;
699
}
700
701
RefPtr<TableUpdate> ProtocolParserV2::CreateTableUpdate(
702
const nsACString& aTableName) const {
703
return new TableUpdateV2(aTableName);
704
}
705
706
///////////////////////////////////////////////////////////////////////
707
// ProtocolParserProtobuf
708
709
ProtocolParserProtobuf::ProtocolParserProtobuf() {}
710
711
ProtocolParserProtobuf::~ProtocolParserProtobuf() {}
712
713
void ProtocolParserProtobuf::SetCurrentTable(const nsACString& aTable) {
714
// Should never occur.
715
MOZ_ASSERT_UNREACHABLE("SetCurrentTable shouldn't be called");
716
}
717
718
RefPtr<TableUpdate> ProtocolParserProtobuf::CreateTableUpdate(
719
const nsACString& aTableName) const {
720
return new TableUpdateV4(aTableName);
721
}
722
723
nsresult ProtocolParserProtobuf::AppendStream(const nsACString& aData) {
724
// Protobuf data cannot be parsed progressively. Just save the incoming data.
725
mPending.Append(aData);
726
return NS_OK;
727
}
728
729
void ProtocolParserProtobuf::End() {
730
// mUpdateStatus will be updated to success as long as not all
731
// the responses are invalid.
732
mUpdateStatus = NS_ERROR_FAILURE;
733
734
FetchThreatListUpdatesResponse response;
735
if (!response.ParseFromArray(mPending.get(), mPending.Length())) {
736
NS_WARNING("ProtocolParserProtobuf failed parsing data.");
737
return;
738
}
739
740
auto minWaitDuration = response.minimum_wait_duration();
741
mUpdateWaitSec =
742
minWaitDuration.seconds() + minWaitDuration.nanos() / 1000000000;
743
744
for (int i = 0; i < response.list_update_responses_size(); i++) {
745
auto r = response.list_update_responses(i);
746
nsAutoCString listName;
747
nsresult rv = ProcessOneResponse(r, listName);
748
if (NS_SUCCEEDED(rv)) {
749
mUpdateStatus = rv;
750
} else {
751
nsAutoCString errorName;
752
mozilla::GetErrorName(rv, errorName);
753
NS_WARNING(nsPrintfCString("Failed to process one response for '%s': %s",
754
listName.get(), errorName.get())
755
.get());
756
if (!listName.IsEmpty()) {
757
PARSER_LOG(("Table %s will be reset.", listName.get()));
758
mTablesToReset.AppendElement(listName);
759
}
760
}
761
}
762
}
763
764
nsresult ProtocolParserProtobuf::ProcessOneResponse(
765
const ListUpdateResponse& aResponse, nsACString& aListName) {
766
MOZ_ASSERT(aListName.IsEmpty());
767
768
// A response must have a threat type.
769
if (!aResponse.has_threat_type()) {
770
NS_WARNING(
771
"Threat type not initialized. This seems to be an invalid response.");
772
return NS_ERROR_UC_PARSER_MISSING_PARAM;
773
}
774
775
nsUrlClassifierUtils* urlUtil = nsUrlClassifierUtils::GetInstance();
776
if (NS_WARN_IF(!urlUtil)) {
777
return NS_ERROR_FAILURE;
778
}
779
780
// Convert threat type to list name.
781
nsCString possibleListNames;
782
nsresult rv = urlUtil->ConvertThreatTypeToListNames(aResponse.threat_type(),
783
possibleListNames);
784
if (NS_FAILED(rv)) {
785
PARSER_LOG(("Threat type to list name conversion error: %d",
786
aResponse.threat_type()));
787
return NS_ERROR_UC_PARSER_UNKNOWN_THREAT;
788
}
789
790
// Match the table name we received with one of the ones we requested.
791
// We ignore the case where a threat type matches more than one list
792
// per provider and return the first one. See bug 1287059."
793
nsTArray<nsCString> possibleListNameArray;
794
Classifier::SplitTables(possibleListNames, possibleListNameArray);
795
for (auto possibleName : possibleListNameArray) {
796
if (mRequestedTables.Contains(possibleName)) {
797
aListName = possibleName;
798
break;
799
}
800
}
801
802
if (aListName.IsEmpty()) {
803
PARSER_LOG(
804
("We received an update for a list we didn't ask for. Ignoring it."));
805
return NS_ERROR_FAILURE;
806
}
807
808
// Test if this is a full update.
809
bool isFullUpdate = false;
810
if (aResponse.has_response_type()) {
811
isFullUpdate = aResponse.response_type() == ListUpdateResponse::FULL_UPDATE;
812
} else {
813
NS_WARNING("Response type not initialized.");
814
return NS_ERROR_UC_PARSER_MISSING_PARAM;
815
}
816
817
// Warn if there's no new state.
818
if (!aResponse.has_new_client_state()) {
819
NS_WARNING("New state not initialized.");
820
return NS_ERROR_UC_PARSER_MISSING_PARAM;
821
}
822
823
auto tu = GetTableUpdate(aListName);
824
auto tuV4 = TableUpdate::Cast<TableUpdateV4>(tu);
825
NS_ENSURE_TRUE(tuV4, NS_ERROR_FAILURE);
826
827
nsCString state(aResponse.new_client_state().c_str(),
828
aResponse.new_client_state().size());
829
tuV4->SetNewClientState(state);
830
831
if (aResponse.has_checksum()) {
832
tuV4->SetSHA256(aResponse.checksum().sha256());
833
}
834
835
PARSER_LOG(
836
("==== Update for threat type '%d' ====", aResponse.threat_type()));
837
PARSER_LOG(("* aListName: %s\n", PromiseFlatCString(aListName).get()));
838
PARSER_LOG(("* newState: %s\n", aResponse.new_client_state().c_str()));
839
PARSER_LOG(("* isFullUpdate: %s\n", (isFullUpdate ? "yes" : "no")));
840
PARSER_LOG(
841
("* hasChecksum: %s\n", (aResponse.has_checksum() ? "yes" : "no")));
842
PARSER_LOG(("* additions: %d\n", aResponse.additions().size()));
843
PARSER_LOG(("* removals: %d\n", aResponse.removals().size()));
844
845
tuV4->SetFullUpdate(isFullUpdate);
846
847
rv = ProcessAdditionOrRemoval(*tuV4, aResponse.additions(),
848
true /*aIsAddition*/);
849
NS_ENSURE_SUCCESS(rv, rv);
850
rv = ProcessAdditionOrRemoval(*tuV4, aResponse.removals(), false);
851
NS_ENSURE_SUCCESS(rv, rv);
852
853
PARSER_LOG(("\n\n"));
854
855
return NS_OK;
856
}
857
858
nsresult ProtocolParserProtobuf::ProcessAdditionOrRemoval(
859
TableUpdateV4& aTableUpdate, const ThreatEntrySetList& aUpdate,
860
bool aIsAddition) {
861
nsresult ret = NS_OK;
862
863
for (int i = 0; i < aUpdate.size(); i++) {
864
auto update = aUpdate.Get(i);
865
if (!update.has_compression_type()) {
866
NS_WARNING(nsPrintfCString("%s with no compression type.",
867
aIsAddition ? "Addition" : "Removal")
868
.get());
869
continue;
870
}
871
872
switch (update.compression_type()) {
873
case COMPRESSION_TYPE_UNSPECIFIED:
874
NS_WARNING("Unspecified compression type.");
875
break;
876
877
case RAW:
878
ret = (aIsAddition ? ProcessRawAddition(aTableUpdate, update)
879
: ProcessRawRemoval(aTableUpdate, update));
880
break;
881
882
case RICE:
883
ret = (aIsAddition ? ProcessEncodedAddition(aTableUpdate, update)
884
: ProcessEncodedRemoval(aTableUpdate, update));
885
break;
886
}
887
}
888
889
return ret;
890
}
891
892
nsresult ProtocolParserProtobuf::ProcessRawAddition(
893
TableUpdateV4& aTableUpdate, const ThreatEntrySet& aAddition) {
894
if (!aAddition.has_raw_hashes()) {
895
PARSER_LOG(("* No raw addition."));
896
return NS_OK;
897
}
898
899
auto rawHashes = aAddition.raw_hashes();
900
if (!rawHashes.has_prefix_size()) {
901
NS_WARNING("Raw hash has no prefix size");
902
return NS_OK;
903
}
904
905
uint32_t prefixSize = rawHashes.prefix_size();
906
MOZ_ASSERT(prefixSize >= PREFIX_SIZE && prefixSize <= COMPLETE_SIZE);
907
908
nsCString prefixes;
909
if (!prefixes.Assign(rawHashes.raw_hashes().c_str(),
910
rawHashes.raw_hashes().size(), mozilla::fallible)) {
911
return NS_ERROR_OUT_OF_MEMORY;
912
}
913
MOZ_ASSERT(prefixes.Length() % prefixSize == 0,
914
"PrefixString length must be a multiple of the prefix size.");
915
916
if (LOG_ENABLED()) {
917
PARSER_LOG((" Raw addition (%d-byte prefixes)", prefixSize));
918
PARSER_LOG((" - # of prefixes: %u", prefixes.Length() / prefixSize));
919
if (4 == prefixSize) {
920
uint32_t* fixedLengthPrefixes = (uint32_t*)prefixes.get();
921
PARSER_LOG((" - Memory address: 0x%p", fixedLengthPrefixes));
922
}
923
}
924
925
aTableUpdate.NewPrefixes(prefixSize, prefixes);
926
return NS_OK;
927
}
928
929
nsresult ProtocolParserProtobuf::ProcessRawRemoval(
930
TableUpdateV4& aTableUpdate, const ThreatEntrySet& aRemoval) {
931
if (!aRemoval.has_raw_indices()) {
932
NS_WARNING("A removal has no indices.");
933
return NS_OK;
934
}
935
936
// indices is an array of int32.
937
auto indices = aRemoval.raw_indices().indices();
938
PARSER_LOG(("* Raw removal"));
939
PARSER_LOG((" - # of removal: %d", indices.size()));
940
941
nsresult rv = aTableUpdate.NewRemovalIndices((const uint32_t*)indices.data(),
942
indices.size());
943
if (NS_FAILED(rv)) {
944
PARSER_LOG(("Failed to create new removal indices."));
945
return rv;
946
}
947
948
return NS_OK;
949
}
950
951
static nsresult DoRiceDeltaDecode(const RiceDeltaEncoding& aEncoding,
952
nsTArray<uint32_t>& aDecoded) {
953
if (aEncoding.num_entries() > 0 &&
954
(!aEncoding.has_rice_parameter() || !aEncoding.has_encoded_data())) {
955
PARSER_LOG(("Rice parameter or encoded data is missing."));
956
return NS_ERROR_UC_PARSER_MISSING_PARAM;
957
} else if (aEncoding.num_entries() == 0 && !aEncoding.has_first_value()) {
958
PARSER_LOG(("Missing first_value for an single-integer Rice encoding."));
959
return NS_ERROR_UC_PARSER_MISSING_VALUE;
960
}
961
962
auto first_value = aEncoding.has_first_value() ? aEncoding.first_value() : 0;
963
964
PARSER_LOG(("* Encoding info:"));
965
PARSER_LOG((" - First value: %" PRId64, first_value));
966
PARSER_LOG((" - Num of entries: %d", aEncoding.num_entries()));
967
PARSER_LOG((" - Rice parameter: %d", aEncoding.rice_parameter()));
968
969
// Set up the input buffer. Note that the bits should be read
970
// from LSB to MSB so that we in-place reverse the bits before
971
// feeding to the decoder.
972
auto encoded =
973
const_cast<RiceDeltaEncoding&>(aEncoding).mutable_encoded_data();
974
RiceDeltaDecoder decoder((uint8_t*)encoded->c_str(), encoded->size());
975
976
// Setup the output buffer. The "first value" is included in
977
// the output buffer.
978
if (!aDecoded.SetLength(aEncoding.num_entries() + 1, mozilla::fallible)) {
979
NS_WARNING("Not enough memory to decode the RiceDelta input.");
980
return NS_ERROR_OUT_OF_MEMORY;
981
}
982
983
// Decode!
984
bool rv = decoder.Decode(
985
aEncoding.rice_parameter(), first_value,
986
aEncoding.num_entries(), // # of entries (first value not included).
987
&aDecoded[0]);
988
989
NS_ENSURE_TRUE(rv, NS_ERROR_UC_PARSER_DECODE_FAILURE);
990
991
return NS_OK;
992
}
993
994
nsresult ProtocolParserProtobuf::ProcessEncodedAddition(
995
TableUpdateV4& aTableUpdate, const ThreatEntrySet& aAddition) {
996
if (!aAddition.has_rice_hashes()) {
997
PARSER_LOG(("* No rice encoded addition."));
998
return NS_OK;
999
}
1000
1001
nsTArray<uint32_t> decoded;
1002
nsresult rv = DoRiceDeltaDecode(aAddition.rice_hashes(), decoded);
1003
if (NS_FAILED(rv)) {
1004
PARSER_LOG(("Failed to parse encoded prefixes."));
1005
return rv;
1006
}
1007
1008
// Say we have the following raw prefixes
1009
// BE LE
1010
// 00 00 00 01 1 16777216
1011
// 00 00 02 00 512 131072
1012
// 00 03 00 00 196608 768
1013
// 04 00 00 00 67108864 4
1014
//
1015
// which can be treated as uint32 (big-endian) sorted in increasing order:
1016
//
1017
// [1, 512, 196608, 67108864]
1018
//
1020
// the following should be done prior to compression:
1021
//
1022
// 1) re-interpret in little-endian ==> [16777216, 131072, 768, 4]
1023
// 2) sort in increasing order ==> [4, 768, 131072, 16777216]
1024
//
1025
// In order to get the original byte stream from |decoded|
1026
// ([4, 768, 131072, 16777216] in this case), we have to:
1027
//
1028
// 1) sort in big-endian order ==> [16777216, 131072, 768, 4]
1029
// 2) copy each uint32 in little-endian to the result string
1030
//
1031
1032
// The 4-byte prefixes have to be re-sorted in Big-endian increasing order.
1033
struct CompareBigEndian {
1034
bool Equals(const uint32_t& aA, const uint32_t& aB) const {
1035
return aA == aB;
1036
}
1037
1038
bool LessThan(const uint32_t& aA, const uint32_t& aB) const {
1039
return NativeEndian::swapToBigEndian(aA) <
1040
NativeEndian::swapToBigEndian(aB);
1041
}
1042
};
1043
decoded.Sort(CompareBigEndian());
1044
1045
// The encoded prefixes are always 4 bytes.
1046
nsCString prefixes;
1047
if (!prefixes.SetCapacity(decoded.Length() * 4, mozilla::fallible)) {
1048
return NS_ERROR_OUT_OF_MEMORY;
1049
}
1050
for (size_t i = 0; i < decoded.Length(); i++) {
1051
// Note that the third argument is the number of elements we want
1052
// to copy (and swap) but not the number of bytes we want to copy.
1053
char p[4];
1054
NativeEndian::copyAndSwapToLittleEndian(p, &decoded[i], 1);
1055
prefixes.Append(p, 4);
1056
}
1057
1058
aTableUpdate.NewPrefixes(4, prefixes);
1059
return NS_OK;
1060
}
1061
1062
nsresult ProtocolParserProtobuf::ProcessEncodedRemoval(
1063
TableUpdateV4& aTableUpdate, const ThreatEntrySet& aRemoval) {
1064
if (!aRemoval.has_rice_indices()) {
1065
PARSER_LOG(("* No rice encoded removal."));
1066
return NS_OK;
1067
}
1068
1069
nsTArray<uint32_t> decoded;
1070
nsresult rv = DoRiceDeltaDecode(aRemoval.rice_indices(), decoded);
1071
if (NS_FAILED(rv)) {
1072
PARSER_LOG(("Failed to decode encoded removal indices."));
1073
return rv;
1074
}
1075
1076
// The encoded prefixes are always 4 bytes.
1077
rv = aTableUpdate.NewRemovalIndices(&decoded[0], decoded.Length());
1078
if (NS_FAILED(rv)) {
1079
PARSER_LOG(("Failed to create new removal indices."));
1080
return rv;
1081
}
1082
1083
return NS_OK;
1084
}
1085
1086
} // namespace safebrowsing
1087
} // namespace mozilla