Revision control

1
/* This Source Code Form is subject to the terms of the Mozilla Public
2
* License, v. 2.0. If a copy of the MPL was not distributed with this
3
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5
import Foundation
6
import Shared
7
import Account
8
import Storage
9
import SwiftyJSON
10
import SyncTelemetry
11
12
fileprivate let log = Logger.syncLogger
13
14
public let PrefKeySyncEvents = "sync.telemetry.events"
15
16
public enum SyncReason: String {
17
case startup = "startup"
18
case scheduled = "scheduled"
19
case backgrounded = "backgrounded"
20
case user = "user"
21
case syncNow = "syncNow"
22
case didLogin = "didLogin"
23
case push = "push"
24
case engineEnabled = "engineEnabled"
25
case clientNameChanged = "clientNameChanged"
26
}
27
28
public enum SyncPingReason: String {
29
case shutdown = "shutdown"
30
case schedule = "schedule"
31
case idChanged = "idchanged"
32
}
33
34
public protocol Stats {
35
func hasData() -> Bool
36
}
37
38
private protocol DictionaryRepresentable {
39
func asDictionary() -> [String: Any]
40
}
41
42
public struct SyncUploadStats: Stats {
43
var sent: Int = 0
44
var sentFailed: Int = 0
45
46
public func hasData() -> Bool {
47
return sent > 0 || sentFailed > 0
48
}
49
}
50
51
extension SyncUploadStats: DictionaryRepresentable {
52
func asDictionary() -> [String: Any] {
53
return [
54
"sent": sent,
55
"failed": sentFailed
56
]
57
}
58
}
59
60
public struct SyncDownloadStats: Stats {
61
var applied: Int = 0
62
var succeeded: Int = 0
63
var failed: Int = 0
64
var newFailed: Int = 0
65
var reconciled: Int = 0
66
67
public func hasData() -> Bool {
68
return applied > 0 ||
69
succeeded > 0 ||
70
failed > 0 ||
71
newFailed > 0 ||
72
reconciled > 0
73
}
74
}
75
76
extension SyncDownloadStats: DictionaryRepresentable {
77
func asDictionary() -> [String: Any] {
78
return [
79
"applied": applied,
80
"succeeded": succeeded,
81
"failed": failed,
82
"newFailed": newFailed,
83
"reconciled": reconciled
84
]
85
}
86
}
87
88
public struct ValidationStats: Stats, DictionaryRepresentable {
89
let problems: [ValidationProblem]
90
let took: Int64
91
let checked: Int?
92
93
public func hasData() -> Bool {
94
return !problems.isEmpty
95
}
96
97
func asDictionary() -> [String: Any] {
98
var dict: [String: Any] = [
99
"problems": problems.map { $0.asDictionary() },
100
"took": took
101
]
102
if let checked = self.checked {
103
dict["checked"] = checked
104
}
105
return dict
106
}
107
}
108
109
public struct ValidationProblem: DictionaryRepresentable {
110
let name: String
111
let count: Int
112
113
func asDictionary() -> [String: Any] {
114
return ["name": name, "count": count]
115
}
116
}
117
118
public class StatsSession {
119
var took: Int64 = 0
120
var when: Timestamp?
121
122
private var startUptimeNanos: UInt64?
123
124
public func start(when: UInt64 = Date.now()) {
125
self.when = when
126
self.startUptimeNanos = DispatchTime.now().uptimeNanoseconds
127
}
128
129
public func hasStarted() -> Bool {
130
return startUptimeNanos != nil
131
}
132
133
public func end() -> Self {
134
guard let startUptime = startUptimeNanos else {
135
assertionFailure("SyncOperationStats called end without first calling start!")
136
return self
137
}
138
139
// Casting to Int64 should be safe since we're using uptime since boot in both cases.
140
// Convert to milliseconds as stated in the sync ping format
141
took = (Int64(DispatchTime.now().uptimeNanoseconds) - Int64(startUptime)) / 1000000
142
return self
143
}
144
}
145
146
// Stats about a single engine's sync.
147
public class SyncEngineStatsSession: StatsSession {
148
public var validationStats: ValidationStats?
149
150
private(set) var uploadStats: SyncUploadStats
151
private(set) var downloadStats: SyncDownloadStats
152
153
public init(collection: String) {
154
self.uploadStats = SyncUploadStats()
155
self.downloadStats = SyncDownloadStats()
156
}
157
158
public func recordDownload(stats: SyncDownloadStats) {
159
self.downloadStats.applied += stats.applied
160
self.downloadStats.succeeded += stats.succeeded
161
self.downloadStats.failed += stats.failed
162
self.downloadStats.newFailed += stats.newFailed
163
self.downloadStats.reconciled += stats.reconciled
164
}
165
166
public func recordUpload(stats: SyncUploadStats) {
167
self.uploadStats.sent += stats.sent
168
self.uploadStats.sentFailed += stats.sentFailed
169
}
170
}
171
172
extension SyncEngineStatsSession: DictionaryRepresentable {
173
func asDictionary() -> [String: Any] {
174
var dict: [String: Any] = [
175
"took": took,
176
]
177
178
if downloadStats.hasData() {
179
dict["incoming"] = downloadStats.asDictionary()
180
}
181
182
if uploadStats.hasData() {
183
dict["outgoing"] = uploadStats.asDictionary()
184
}
185
186
if let validation = self.validationStats, validation.hasData() {
187
dict["validation"] = validation.asDictionary()
188
}
189
190
return dict
191
}
192
}
193
194
// Stats and metadata for a sync operation.
195
public class SyncOperationStatsSession: StatsSession {
196
public let why: SyncReason
197
public var uid: String?
198
public var deviceID: String?
199
200
fileprivate let didLogin: Bool
201
202
public init(why: SyncReason, uid: String, deviceID: String?) {
203
self.why = why
204
self.uid = uid
205
self.deviceID = deviceID
206
self.didLogin = (why == .didLogin)
207
}
208
}
209
210
extension SyncOperationStatsSession: DictionaryRepresentable {
211
func asDictionary() -> [String: Any] {
212
let whenValue = when ?? 0
213
return [
214
"when": whenValue,
215
"took": took,
216
"didLogin": didLogin,
217
"why": why.rawValue
218
]
219
}
220
}
221
222
public enum SyncPingError: MaybeErrorType {
223
case failedToRestoreScratchpad
224
225
public var description: String {
226
switch self {
227
case .failedToRestoreScratchpad: return "Failed to restore Scratchpad from prefs"
228
}
229
}
230
}
231
232
public enum SyncPingFailureReasonName: String {
233
case httpError = "httperror"
234
case unexpectedError = "unexpectederror"
235
case sqlError = "sqlerror"
236
case otherError = "othererror"
237
}
238
239
public protocol SyncPingFailureFormattable {
240
var failureReasonName: SyncPingFailureReasonName { get }
241
}
242
243
public struct SyncPing: SyncTelemetryPing {
244
public private(set) var payload: JSON
245
246
public static func from(result: SyncOperationResult,
247
account: Account.FirefoxAccount,
248
remoteClientsAndTabs: RemoteClientsAndTabs,
249
prefs: Prefs,
250
why: SyncPingReason) -> Deferred<Maybe<SyncPing>> {
251
// Grab our token so we can use the hashed_fxa_uid and clientGUID from our scratchpad for
252
// our ping's identifiers
253
return account.syncAuthState.token(Date.now(), canBeExpired: false) >>== { (token, kSync) in
254
let scratchpadPrefs = prefs.branch("sync.scratchpad")
255
guard let scratchpad = Scratchpad.restoreFromPrefs(scratchpadPrefs, syncKeyBundle: KeyBundle.fromKSync(kSync)) else {
256
return deferMaybe(SyncPingError.failedToRestoreScratchpad)
257
}
258
259
var ping: [String: Any] = pingCommonData(
260
why: why,
261
hashedUID: token.hashedFxAUID,
262
hashedDeviceID: (scratchpad.clientGUID + token.hashedFxAUID).sha256.hexEncodedString
263
)
264
265
// TODO: We don't cache our sync pings so if it fails, it fails. Once we add
266
// some kind of caching we'll want to make sure we don't dump the events if
267
// the ping has failed.
268
let pickledEvents = prefs.arrayForKey(PrefKeySyncEvents) as? [Data] ?? []
269
let events = pickledEvents.compactMap(Event.unpickle).map { $0.toArray() }
270
ping["events"] = events
271
prefs.setObject(nil, forKey: PrefKeySyncEvents)
272
273
return dictionaryFrom(result: result, storage: remoteClientsAndTabs, token: token) >>== { syncDict in
274
// TODO: Split the sync ping metadata from storing a single sync.
275
ping["syncs"] = [syncDict]
276
return deferMaybe(SyncPing(payload: JSON(ping)))
277
}
278
}
279
}
280
281
static func pingCommonData(why: SyncPingReason, hashedUID: String, hashedDeviceID: String) -> [String: Any] {
282
return [
283
"version": 1,
284
"why": why.rawValue,
285
"uid": hashedUID,
286
"deviceID": hashedDeviceID,
287
"os": [
288
"name": "iOS",
289
"version": UIDevice.current.systemVersion,
290
"locale": Locale.current.identifier
291
]
292
]
293
}
294
295
// Generates a single sync ping payload that is stored in the 'syncs' list in the sync ping.
296
private static func dictionaryFrom(result: SyncOperationResult,
297
storage: RemoteClientsAndTabs,
298
token: TokenServerToken) -> Deferred<Maybe<[String: Any]>> {
299
return connectedDevices(fromStorage: storage, token: token) >>== { devices in
300
guard let stats = result.stats else {
301
return deferMaybe([String: Any]())
302
}
303
304
var dict = stats.asDictionary()
305
if let engineResults = result.engineResults.successValue {
306
dict["engines"] = SyncPing.enginePingDataFrom(engineResults: engineResults)
307
} else if let failure = result.engineResults.failureValue {
308
var errorName: SyncPingFailureReasonName
309
if let formattableFailure = failure as? SyncPingFailureFormattable {
310
errorName = formattableFailure.failureReasonName
311
} else {
312
errorName = .unexpectedError
313
}
314
315
dict["failureReason"] = [
316
"name": errorName.rawValue,
317
"error": "\(type(of: failure))",
318
]
319
}
320
321
dict["devices"] = devices
322
return deferMaybe(dict)
323
}
324
}
325
326
// Returns a list of connected devices formatted for use in the 'devices' property in the sync ping.
327
private static func connectedDevices(fromStorage storage: RemoteClientsAndTabs,
328
token: TokenServerToken) -> Deferred<Maybe<[[String: Any]]>> {
329
func dictionaryFrom(client: RemoteClient) -> [String: Any]? {
330
var device = [String: Any]()
331
if let os = client.os {
332
device["os"] = os
333
}
334
if let version = client.version {
335
device["version"] = version
336
}
337
if let guid = client.guid {
338
device["id"] = (guid + token.hashedFxAUID).sha256.hexEncodedString
339
}
340
return device
341
}
342
343
return storage.getClients() >>== { deferMaybe($0.compactMap(dictionaryFrom)) }
344
}
345
346
private static func enginePingDataFrom(engineResults: EngineResults) -> [[String: Any]] {
347
return engineResults.map { result in
348
let (name, status) = result
349
var engine: [String: Any] = [
350
"name": name
351
]
352
353
// For complete/partial results, extract out the collect stats
354
// and add it to engine information. For syncs that were not able to
355
// start, return why and a reason.
356
switch status {
357
case .completed(let stats):
358
engine.merge(with: stats.asDictionary())
359
case .partial(let stats):
360
engine.merge(with: stats.asDictionary())
361
case .notStarted(let reason):
362
engine.merge(with: [
363
"status": reason.telemetryId
364
])
365
}
366
367
return engine
368
}
369
}
370
}