Revision control

1
/* This Source Code Form is subject to the terms of the Mozilla Public
2
* License, v. 2.0. If a copy of the MPL was not distributed with this
3
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5
import Foundation
6
import Account
7
import Shared
8
import XCGLogger
9
import SwiftKeychainWrapper
10
import SwiftyJSON
11
12
private let log = Logger.syncLogger
13
14
/*
15
* This file includes types that manage intra-sync and inter-sync metadata
16
* for the use of synchronizers and the state machine.
17
*
18
* See docs/sync.md for details on what exactly we need to persist.
19
*/
20
21
public struct Fetched<T: Equatable>: Equatable {
22
let value: T
23
let timestamp: Timestamp
24
}
25
26
public func ==<T>(lhs: Fetched<T>, rhs: Fetched<T>) -> Bool {
27
return lhs.timestamp == rhs.timestamp &&
28
lhs.value == rhs.value
29
}
30
31
public enum LocalCommand: CustomStringConvertible, Hashable {
32
// We've seen something (a blank server, a changed global sync ID, a
33
// crypto/keys with a different meta/global) that requires us to reset all
34
// local engine timestamps (save the ones listed) and possibly re-upload.
35
case resetAllEngines(except: Set<String>)
36
37
// We've seen something (a changed engine sync ID, a crypto/keys with a
38
// different per-engine bulk key) that requires us to reset our local engine
39
// timestamp and possibly re-upload.
40
case resetEngine(engine: String)
41
42
// We've seen a change in meta/global: an engine has come or gone.
43
case enableEngine(engine: String)
44
case disableEngine(engine: String)
45
46
public func toJSON() -> JSON {
47
switch self {
48
case let .resetAllEngines(except):
49
return JSON(["type": "ResetAllEngines", "except": Array(except).sorted()])
50
51
case let .resetEngine(engine):
52
return JSON(["type": "ResetEngine", "engine": engine])
53
54
case let .enableEngine(engine):
55
return JSON(["type": "EnableEngine", "engine": engine])
56
57
case let .disableEngine(engine):
58
return JSON(["type": "DisableEngine", "engine": engine])
59
}
60
}
61
62
public static func fromJSON(_ json: JSON) -> LocalCommand? {
63
if json.isError() {
64
return nil
65
}
66
guard let type = json["type"].string else {
67
return nil
68
}
69
switch type {
70
case "ResetAllEngines":
71
if let except = json["except"].array, except.every({$0.isString()}) {
72
return .resetAllEngines(except: Set(except.map({$0.stringValue})))
73
}
74
return nil
75
case "ResetEngine":
76
if let engine = json["engine"].string {
77
return .resetEngine(engine: engine)
78
}
79
return nil
80
case "EnableEngine":
81
if let engine = json["engine"].string {
82
return .enableEngine(engine: engine)
83
}
84
return nil
85
case "DisableEngine":
86
if let engine = json["engine"].string {
87
return .disableEngine(engine: engine)
88
}
89
return nil
90
default:
91
return nil
92
}
93
}
94
95
public var description: String {
96
return self.toJSON().description
97
}
98
99
public func hash(into hasher: inout Hasher) {
100
hasher.combine(description)
101
}
102
}
103
104
public func ==(lhs: LocalCommand, rhs: LocalCommand) -> Bool {
105
switch (lhs, rhs) {
106
case (let .resetAllEngines(exceptL), let .resetAllEngines(exceptR)):
107
return exceptL == exceptR
108
109
case (let .resetEngine(engineL), let .resetEngine(engineR)):
110
return engineL == engineR
111
112
case (let .enableEngine(engineL), let .enableEngine(engineR)):
113
return engineL == engineR
114
115
case (let .disableEngine(engineL), let .disableEngine(engineR)):
116
return engineL == engineR
117
118
default:
119
return false
120
}
121
}
122
123
/*
124
* Persistence pref names.
125
* Note that syncKeyBundle isn't persisted by us.
126
*
127
* Note also that fetched keys aren't kept in prefs: we keep the timestamp ("PrefKeysTS"),
128
* and we keep a 'label'. This label is used to find the real fetched keys in the Keychain.
129
*/
130
131
private let PrefVersion = "_v"
132
private let PrefGlobal = "global"
133
private let PrefGlobalTS = "globalTS"
134
private let PrefKeyLabel = "keyLabel"
135
private let PrefKeysTS = "keysTS"
136
private let PrefLastFetched = "lastFetched"
137
private let PrefLocalCommands = "localCommands"
138
private let PrefClientName = "clientName"
139
private let PrefClientGUID = "clientGUID"
140
private let PrefHashedUID = "hashedUID"
141
private let PrefEngineConfiguration = "engineConfiguration"
142
private let PrefEnginesEnablements = "enginesEnablements"
143
private let PrefDeviceID = "deviceID"
144
145
class PrefsBackoffStorage: BackoffStorage {
146
let prefs: Prefs
147
fileprivate let key = "timestamp"
148
149
init(prefs: Prefs) {
150
self.prefs = prefs
151
}
152
153
var serverBackoffUntilLocalTimestamp: Timestamp? {
154
get {
155
return self.prefs.unsignedLongForKey(self.key)
156
}
157
158
set(value) {
159
if let value = value {
160
self.prefs.setLong(value, forKey: self.key)
161
} else {
162
self.prefs.removeObjectForKey(self.key)
163
}
164
}
165
}
166
167
func clearServerBackoff() {
168
self.prefs.removeObjectForKey(self.key)
169
}
170
171
func isInBackoff(_ now: Timestamp) -> Timestamp? {
172
if let ts = self.serverBackoffUntilLocalTimestamp, now < ts {
173
return ts
174
}
175
return nil
176
}
177
}
178
179
/**
180
* The scratchpad consists of the following:
181
*
182
* 1. Cached records. We cache meta/global and crypto/keys until they change.
183
* 2. Metadata like timestamps, both for cached records and for server fetches.
184
* 3. User preferences -- engine enablement.
185
* 4. Client record state.
186
* 5. Local commands that have yet to be processed.
187
*
188
* Note that the scratchpad itself is immutable, but is a class passed by reference.
189
* Its mutable fields can be mutated, but you can't accidentally e.g., switch out
190
* meta/global and get confused.
191
*
192
* TODO: the Scratchpad needs to be loaded from persistent storage, and written
193
* back at certain points in the state machine (after a replayable action is taken).
194
*/
195
open class Scratchpad {
196
open class Builder {
197
var syncKeyBundle: KeyBundle // For the love of god, if you change this, invalidate keys, too!
198
fileprivate var global: Fetched<MetaGlobal>?
199
fileprivate var keys: Fetched<Keys>?
200
fileprivate var keyLabel: String
201
var localCommands: Set<LocalCommand>
202
var engineConfiguration: EngineConfiguration?
203
// Engines that were manually enabled/disabled by the user since our last sync.
204
var enginesEnablements: [String: Bool]?
205
var clientGUID: String
206
var clientName: String
207
var fxaDeviceId: String
208
var hashedUID: String?
209
var prefs: Prefs
210
211
init(p: Scratchpad) {
212
self.syncKeyBundle = p.syncKeyBundle
213
self.prefs = p.prefs
214
215
self.global = p.global
216
217
self.keys = p.keys
218
self.keyLabel = p.keyLabel
219
self.localCommands = p.localCommands
220
self.engineConfiguration = p.engineConfiguration
221
self.enginesEnablements = p.enginesEnablements
222
self.clientGUID = p.clientGUID
223
self.clientName = p.clientName
224
self.fxaDeviceId = p.fxaDeviceId
225
self.hashedUID = p.hashedUID
226
}
227
228
open func clearLocalCommands() -> Builder {
229
self.localCommands.removeAll()
230
return self
231
}
232
233
open func addLocalCommandsFromKeys(_ keys: Fetched<Keys>?) -> Builder {
234
// Getting new keys can force local collection resets.
235
guard let freshKeys = keys?.value, let staleKeys = self.keys?.value, staleKeys.valid else {
236
// Removing keys, or new keys and either we didn't have old keys or they weren't valid. Everybody gets a reset!
237
self.localCommands.insert(LocalCommand.resetAllEngines(except: []))
238
return self
239
}
240
241
// New keys, and we have valid old keys.
242
if freshKeys.defaultBundle != staleKeys.defaultBundle {
243
// Default bundle has changed. Reset everything but collections that have unchanged bulk keys.
244
var except: Set<String> = Set()
245
// Symmetric difference, like an animal. Swift doesn't allow Hashable tuples; don't fight it.
246
for (collection, keyBundle) in staleKeys.collectionKeys {
247
if keyBundle == freshKeys.forCollection(collection) {
248
except.insert(collection)
249
}
250
}
251
for (collection, keyBundle) in freshKeys.collectionKeys {
252
if keyBundle == staleKeys.forCollection(collection) {
253
except.insert(collection)
254
}
255
}
256
self.localCommands.insert(.resetAllEngines(except: except))
257
} else {
258
// Default bundle is the same. Reset collections that have changed bulk keys.
259
for (collection, keyBundle) in staleKeys.collectionKeys {
260
if keyBundle != freshKeys.forCollection(collection) {
261
self.localCommands.insert(.resetEngine(engine: collection))
262
}
263
}
264
for (collection, keyBundle) in freshKeys.collectionKeys {
265
if keyBundle != staleKeys.forCollection(collection) {
266
self.localCommands.insert(.resetEngine(engine: collection))
267
}
268
}
269
}
270
return self
271
}
272
273
open func setKeys(_ keys: Fetched<Keys>?) -> Builder {
274
self.keys = keys
275
return self
276
}
277
278
open func clearEnginesEnablements() -> Builder {
279
self.enginesEnablements = nil
280
return self
281
}
282
283
open func setGlobal(_ global: Fetched<MetaGlobal>?) -> Builder {
284
self.global = global
285
if let global = global {
286
// We always take the incoming meta/global's engine configuration.
287
self.engineConfiguration = global.value.engineConfiguration()
288
}
289
return self
290
}
291
292
open func setEngineConfiguration(_ engineConfiguration: EngineConfiguration?) -> Builder {
293
self.engineConfiguration = engineConfiguration
294
return self
295
}
296
297
open func build() -> Scratchpad {
298
return Scratchpad(
299
b: self.syncKeyBundle,
300
m: self.global,
301
k: self.keys,
302
keyLabel: self.keyLabel,
303
localCommands: self.localCommands,
304
engines: self.engineConfiguration,
305
enginesEnablements: self.enginesEnablements,
306
clientGUID: self.clientGUID,
307
clientName: self.clientName,
308
fxaDeviceId: self.fxaDeviceId,
309
hashedUID: self.hashedUID,
310
persistingTo: self.prefs
311
)
312
}
313
}
314
315
open lazy var backoffStorage: BackoffStorage = {
316
return PrefsBackoffStorage(prefs: self.prefs.branch("backoff.storage"))
317
}()
318
319
open func evolve() -> Scratchpad.Builder {
320
return Scratchpad.Builder(p: self)
321
}
322
323
// This is never persisted.
324
let syncKeyBundle: KeyBundle
325
326
// Cached records.
327
// This cached meta/global is what we use to add or remove enabled engines. See also
328
// engineConfiguration, below.
329
// We also use it to detect when meta/global hasn't changed -- compare timestamps.
330
//
331
// Note that a Scratchpad held by a Ready state will have the current server meta/global
332
// here. That means we don't need to track syncIDs separately (which is how desktop and
333
// Android are implemented).
334
// If we don't have a meta/global, and thus we don't know syncIDs, it means we haven't
335
// synced with this server before, and we'll do a fresh sync.
336
let global: Fetched<MetaGlobal>?
337
338
// We don't store these keys (so-called "collection keys" or "bulk keys") in Prefs.
339
// Instead, we store a label, which is seeded when you first create a Scratchpad.
340
// This label is used to retrieve the real keys from your Keychain.
341
//
342
// Note that we also don't store the syncKeyBundle here. That's always created from kB,
343
// provided by the Firefox Account.
344
//
345
// Why don't we derive the label from your Sync Key? Firstly, we'd like to be able to
346
// clean up without having your key. Secondly, we don't want to accidentally load keys
347
// from the Keychain just because the Sync Key is the same -- e.g., after a node
348
// reassignment. Randomly generating a label offers all of the benefits with none of the
349
// problems, with only the cost of persisting that label alongside the rest of the state.
350
let keys: Fetched<Keys>?
351
let keyLabel: String
352
353
// Local commands.
354
var localCommands: Set<LocalCommand>
355
356
// Enablement states.
357
let engineConfiguration: EngineConfiguration?
358
let enginesEnablements: [String: Bool]?
359
360
// What's our client name?
361
let clientName: String
362
let clientGUID: String
363
let fxaDeviceId: String
364
let hashedUID: String?
365
366
var hashedDeviceID: String? {
367
guard let hashedUID = hashedUID else {
368
return nil
369
}
370
return (fxaDeviceId + hashedUID).sha256.hexEncodedString
371
}
372
373
// Where do we persist when told?
374
let prefs: Prefs
375
376
init(b: KeyBundle,
377
m: Fetched<MetaGlobal>?,
378
k: Fetched<Keys>?,
379
keyLabel: String,
380
localCommands: Set<LocalCommand>,
381
engines: EngineConfiguration?,
382
enginesEnablements: [String: Bool]?,
383
clientGUID: String,
384
clientName: String,
385
fxaDeviceId: String,
386
hashedUID: String?,
387
persistingTo prefs: Prefs
388
) {
389
self.syncKeyBundle = b
390
self.prefs = prefs
391
392
self.keys = k
393
self.keyLabel = keyLabel
394
self.global = m
395
self.engineConfiguration = engines
396
self.enginesEnablements = enginesEnablements
397
self.localCommands = localCommands
398
self.clientGUID = clientGUID
399
self.clientName = clientName
400
self.fxaDeviceId = fxaDeviceId
401
self.hashedUID = hashedUID
402
}
403
404
// This should never be used in the end; we'll unpickle instead.
405
// This should be a convenience initializer, but... Swift compiler bug?
406
init(b: KeyBundle, persistingTo prefs: Prefs) {
407
self.syncKeyBundle = b
408
self.prefs = prefs
409
410
self.keys = nil
411
self.keyLabel = Bytes.generateGUID()
412
self.global = nil
413
self.engineConfiguration = nil
414
self.enginesEnablements = nil
415
self.localCommands = Set()
416
self.clientGUID = Bytes.generateGUID()
417
self.clientName = DeviceInfo.defaultClientName()
418
419
self.fxaDeviceId = "unknown_fxaDeviceId"
420
421
self.hashedUID = nil
422
}
423
424
func freshStartWithGlobal(_ global: Fetched<MetaGlobal>) -> Scratchpad {
425
// TODO: I *think* a new keyLabel is unnecessary.
426
return self.evolve()
427
.setGlobal(global)
428
.addLocalCommandsFromKeys(nil)
429
.setKeys(nil)
430
.build()
431
}
432
433
fileprivate class func unpickleV1FromPrefs(_ prefs: Prefs, syncKeyBundle: KeyBundle) -> Scratchpad {
434
let b = Scratchpad(b: syncKeyBundle, persistingTo: prefs).evolve()
435
436
if let mg = prefs.stringForKey(PrefGlobal) {
437
if let mgTS = prefs.unsignedLongForKey(PrefGlobalTS) {
438
if let global = MetaGlobal.fromJSON(JSON(parseJSON: mg)) {
439
_ = b.setGlobal(Fetched(value: global, timestamp: mgTS))
440
} else {
441
log.error("Malformed meta/global in prefs. Ignoring.")
442
}
443
} else {
444
// This should never happen.
445
log.error("Found global in prefs, but not globalTS!")
446
}
447
}
448
449
if let keyLabel = prefs.stringForKey(PrefKeyLabel) {
450
b.keyLabel = keyLabel
451
if let ckTS = prefs.unsignedLongForKey(PrefKeysTS) {
452
let key = "keys." + keyLabel
453
KeychainWrapper.sharedAppContainerKeychain.ensureStringItemAccessibility(.afterFirstUnlock, forKey: key)
454
if let keys = KeychainWrapper.sharedAppContainerKeychain.string(forKey: key) {
455
// We serialize as JSON.
456
let keys = Keys(payload: KeysPayload(keys))
457
if keys.valid {
458
log.debug("Read keys from Keychain with label \(keyLabel).")
459
_ = b.setKeys(Fetched(value: keys, timestamp: ckTS))
460
} else {
461
log.error("Invalid keys extracted from Keychain. Discarding.")
462
}
463
} else {
464
log.error("Found keysTS in prefs, but didn't find keys in Keychain!")
465
}
466
}
467
}
468
469
b.clientGUID = prefs.stringForKey(PrefClientGUID) ?? {
470
log.error("No value found in prefs for client GUID! Generating one.")
471
return Bytes.generateGUID()
472
}()
473
474
b.clientName = prefs.stringForKey(PrefClientName) ?? {
475
log.error("No value found in prefs for client name! Using default.")
476
return DeviceInfo.defaultClientName()
477
}()
478
479
b.hashedUID = prefs.stringForKey(PrefHashedUID)
480
481
b.fxaDeviceId = prefs.stringForKey(PrefDeviceID) ?? {
482
// Migrate from previous way of storing device id.
483
// This code will only be run once – the id will be stored
484
// in PrefDeviceID.
485
let PrefDeviceRegistration = "deviceRegistration"
486
if let string = prefs.stringForKey(PrefDeviceRegistration) {
487
let json = JSON(parseJSON: string)
488
if let id = json["id"].string {
489
return id
490
}
491
prefs.removeObjectForKey(PrefDeviceRegistration)
492
}
493
// This is run the first time we sync with a new account.
494
// It will be replaced by a real fxaDeviceId, from account.deviceRegistration?.id.
495
log.warning("No value found in prefs for fxaDeviceId! Will overwrite on first sync")
496
return "unknown_fxaDeviceId"
497
}()
498
499
if let localCommands: [String] = prefs.stringArrayForKey(PrefLocalCommands) {
500
b.localCommands = Set(localCommands.compactMap({LocalCommand.fromJSON(JSON(parseJSON: $0))}))
501
}
502
503
if let engineConfigurationString = prefs.stringForKey(PrefEngineConfiguration) {
504
if let engineConfiguration = EngineConfiguration.fromJSON(JSON(parseJSON: engineConfigurationString)) {
505
b.engineConfiguration = engineConfiguration
506
} else {
507
log.error("Invalid engineConfiguration found in prefs. Discarding.")
508
}
509
}
510
511
if let enginesEnablements = prefs.dictionaryForKey(PrefEnginesEnablements) {
512
b.enginesEnablements = enginesEnablements as? [String: Bool]
513
}
514
515
return b.build()
516
}
517
518
/**
519
* Remove anything that might be left around after prefs is wiped.
520
*/
521
open class func clearFromPrefs(_ prefs: Prefs) {
522
if let keyLabel = prefs.stringForKey(PrefKeyLabel) {
523
log.debug("Removing saved key from keychain.")
524
KeychainWrapper.sharedAppContainerKeychain.removeObject(forKey: keyLabel)
525
} else {
526
log.debug("No key label; nothing to remove from keychain.")
527
}
528
}
529
530
open class func restoreFromPrefs(_ prefs: Prefs, syncKeyBundle: KeyBundle) -> Scratchpad? {
531
if let ver = prefs.intForKey(PrefVersion) {
532
switch ver {
533
case 1:
534
return unpickleV1FromPrefs(prefs, syncKeyBundle: syncKeyBundle)
535
default:
536
return nil
537
}
538
}
539
540
log.debug("No scratchpad found in prefs.")
541
return nil
542
}
543
544
/**
545
* Persist our current state to our origin prefs.
546
* Note that calling this from multiple threads with either mutated or evolved
547
* scratchpads will cause sadness — individual writes are thread-safe, but the
548
* overall pseudo-transaction is not atomic.
549
*/
550
open func checkpoint() -> Scratchpad {
551
return pickle(self.prefs)
552
}
553
554
func pickle(_ prefs: Prefs) -> Scratchpad {
555
prefs.setInt(1, forKey: PrefVersion)
556
if let global = global {
557
prefs.setLong(global.timestamp, forKey: PrefGlobalTS)
558
prefs.setString(global.value.asPayload().json.stringify()!, forKey: PrefGlobal)
559
} else {
560
prefs.removeObjectForKey(PrefGlobal)
561
prefs.removeObjectForKey(PrefGlobalTS)
562
}
563
564
// We store the meat of your keys in the Keychain, using a random identifier that we persist in prefs.
565
prefs.setString(self.keyLabel, forKey: PrefKeyLabel)
566
if let keys = self.keys,
567
let payload = keys.value.asPayload().json.stringify() {
568
let label = "keys." + self.keyLabel
569
log.debug("Storing keys in Keychain with label \(label).")
570
prefs.setString(self.keyLabel, forKey: PrefKeyLabel)
571
prefs.setLong(keys.timestamp, forKey: PrefKeysTS)
572
KeychainWrapper.sharedAppContainerKeychain.set(payload, forKey: label, withAccessibility: .afterFirstUnlock)
573
} else {
574
log.debug("Removing keys from Keychain.")
575
KeychainWrapper.sharedAppContainerKeychain.removeObject(forKey: self.keyLabel)
576
}
577
578
prefs.setString(clientName, forKey: PrefClientName)
579
prefs.setString(clientGUID, forKey: PrefClientGUID)
580
581
if let uid = hashedUID {
582
prefs.setString(uid, forKey: PrefHashedUID)
583
}
584
585
prefs.setString(fxaDeviceId, forKey: PrefDeviceID)
586
587
let localCommands: [String] = Array(self.localCommands).map({$0.toJSON().stringify()!})
588
prefs.setObject(localCommands, forKey: PrefLocalCommands)
589
590
if let engineConfiguration = self.engineConfiguration {
591
prefs.setString(engineConfiguration.toJSON().stringify()!, forKey: PrefEngineConfiguration)
592
} else {
593
prefs.removeObjectForKey(PrefEngineConfiguration)
594
}
595
596
if let enginesEnablements = self.enginesEnablements {
597
prefs.setObject(enginesEnablements, forKey: PrefEnginesEnablements)
598
} else {
599
prefs.removeObjectForKey(PrefEnginesEnablements)
600
}
601
602
return self
603
}
604
}