Source code
Revision control
Copy as Markdown
Other Tools
Test Info: Warnings
- This test gets skipped with pattern: os == 'android' OR appname == 'thunderbird'
- Manifest: toolkit/components/nimbus/test/unit/xpcshell.toml
"use strict";
const { TelemetryEvents } = ChromeUtils.importESModule(
);
const { TelemetryEnvironment } = ChromeUtils.importESModule(
);
const { ExperimentAPI } = ChromeUtils.importESModule(
);
const STUDIES_OPT_OUT_PREF = "app.shield.optoutstudies.enabled";
const UPLOAD_ENABLED_PREF = "datareporting.healthreport.uploadEnabled";
const globalSandbox = sinon.createSandbox();
globalSandbox.spy(TelemetryEnvironment, "setExperimentInactive");
globalSandbox.spy(TelemetryEvents, "sendEvent");
registerCleanupFunction(() => {
globalSandbox.restore();
});
/**
* FOG requires a little setup in order to test it
*/
add_setup(function test_setup() {
// FOG needs a profile directory to put its data in.
do_get_profile();
// FOG needs to be initialized in order for data to flow.
Services.fog.initializeFOG();
});
/**
* Normal unenrollment for experiments:
* - set .active to false
* - set experiment inactive in telemetry
* - send unrollment event
*/
add_task(async function test_set_inactive() {
const manager = ExperimentFakes.manager();
await manager.onStartup();
await manager.store.addEnrollment(ExperimentFakes.experiment("foo"));
manager.unenroll("foo", "some-reason");
Assert.equal(
manager.store.get("foo").active,
false,
"should set .active to false"
);
});
add_task(async function test_unenroll_opt_out() {
globalSandbox.reset();
Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, true);
const manager = ExperimentFakes.manager();
const experiment = ExperimentFakes.experiment("foo");
// Clear any pre-existing data in Glean
Services.fog.testResetFOG();
await manager.onStartup();
await manager.store.addEnrollment(experiment);
// Check that there aren't any Glean unenrollment events yet
var unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue();
Assert.equal(
undefined,
unenrollmentEvents,
"no Glean unenrollment events before unenrollment"
);
Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, false);
Assert.equal(
manager.store.get(experiment.slug).active,
false,
"should set .active to false"
);
Assert.ok(TelemetryEvents.sendEvent.calledOnce);
Assert.deepEqual(
TelemetryEvents.sendEvent.firstCall.args,
[
"unenroll",
"nimbus_experiment",
experiment.slug,
{
reason: "studies-opt-out",
branch: experiment.branch.slug,
},
],
"should send an unenrollment ping with the slug, reason, and branch slug"
);
// Check that the Glean unenrollment event was recorded.
unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue();
// We expect only one event
Assert.equal(1, unenrollmentEvents.length);
// And that one event matches the expected enrolled experiment
Assert.equal(
experiment.slug,
unenrollmentEvents[0].extra.experiment,
"Glean.nimbusEvents.unenrollment recorded with correct experiment slug"
);
Assert.equal(
experiment.branch.slug,
unenrollmentEvents[0].extra.branch,
"Glean.nimbusEvents.unenrollment recorded with correct branch slug"
);
Assert.equal(
"studies-opt-out",
unenrollmentEvents[0].extra.reason,
"Glean.nimbusEvents.unenrollment recorded with correct reason"
);
// reset pref
Services.prefs.clearUserPref(STUDIES_OPT_OUT_PREF);
});
add_task(async function test_unenroll_rollout_opt_out() {
globalSandbox.reset();
Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, true);
const manager = ExperimentFakes.manager();
const rollout = ExperimentFakes.rollout("foo");
// Clear any pre-existing data in Glean
Services.fog.testResetFOG();
await manager.onStartup();
await manager.store.addEnrollment(rollout);
// Check that there aren't any Glean unenrollment events yet
var unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue();
Assert.equal(
undefined,
unenrollmentEvents,
"no Glean unenrollment events before unenrollment"
);
Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, false);
Assert.equal(
manager.store.get(rollout.slug).active,
false,
"should set .active to false"
);
Assert.ok(TelemetryEvents.sendEvent.calledOnce);
Assert.deepEqual(
TelemetryEvents.sendEvent.firstCall.args,
[
"unenroll",
"nimbus_experiment",
rollout.slug,
{
reason: "studies-opt-out",
branch: rollout.branch.slug,
},
],
"should send an unenrollment ping with the slug, reason, and branch slug"
);
// Check that the Glean unenrollment event was recorded.
unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue();
// We expect only one event
Assert.equal(1, unenrollmentEvents.length);
// And that one event matches the expected enrolled experiment
Assert.equal(
rollout.slug,
unenrollmentEvents[0].extra.experiment,
"Glean.nimbusEvents.unenrollment recorded with correct rollout slug"
);
Assert.equal(
rollout.branch.slug,
unenrollmentEvents[0].extra.branch,
"Glean.nimbusEvents.unenrollment recorded with correct branch slug"
);
Assert.equal(
"studies-opt-out",
unenrollmentEvents[0].extra.reason,
"Glean.nimbusEvents.unenrollment recorded with correct reason"
);
// reset pref
Services.prefs.clearUserPref(STUDIES_OPT_OUT_PREF);
});
add_task(async function test_unenroll_uploadPref() {
globalSandbox.reset();
const manager = ExperimentFakes.manager();
const recipe = ExperimentFakes.recipe("foo");
await manager.onStartup();
await ExperimentFakes.enrollmentHelper(recipe, { manager }).enrollmentPromise;
Assert.equal(
manager.store.get(recipe.slug).active,
true,
"Should set .active to true"
);
Services.prefs.setBoolPref(UPLOAD_ENABLED_PREF, false);
Assert.equal(
manager.store.get(recipe.slug).active,
false,
"Should set .active to false"
);
Services.prefs.clearUserPref(UPLOAD_ENABLED_PREF);
});
add_task(async function test_setExperimentInactive_called() {
globalSandbox.reset();
const manager = ExperimentFakes.manager();
const experiment = ExperimentFakes.experiment("foo");
// Clear any pre-existing data in Glean
Services.fog.testResetFOG();
await manager.onStartup();
await manager.store.addEnrollment(experiment);
// Because `manager.store.addEnrollment()` sidesteps telemetry recording
// we will also call on the Glean experiment API directly to test that
// `manager.unenroll()` does in fact call `Glean.setExperimentActive()`
Services.fog.setExperimentActive(
experiment.slug,
experiment.branch.slug,
null
);
// Test Glean experiment API interaction
Assert.notEqual(
undefined,
Services.fog.testGetExperimentData(experiment.slug),
"experiment should be active before unenroll"
);
manager.unenroll("foo", "some-reason");
Assert.ok(
TelemetryEnvironment.setExperimentInactive.calledWith("foo"),
"should call TelemetryEnvironment.setExperimentInactive with slug"
);
// Test Glean experiment API interaction
Assert.equal(
undefined,
Services.fog.testGetExperimentData(experiment.slug),
"experiment should be inactive after unenroll"
);
});
add_task(async function test_send_unenroll_event() {
globalSandbox.reset();
const manager = ExperimentFakes.manager();
const experiment = ExperimentFakes.experiment("foo");
// Clear any pre-existing data in Glean
Services.fog.testResetFOG();
await manager.onStartup();
await manager.store.addEnrollment(experiment);
// Check that there aren't any Glean unenrollment events yet
var unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue();
Assert.equal(
undefined,
unenrollmentEvents,
"no Glean unenrollment events before unenrollment"
);
manager.unenroll("foo", "some-reason");
Assert.ok(TelemetryEvents.sendEvent.calledOnce);
Assert.deepEqual(
TelemetryEvents.sendEvent.firstCall.args,
[
"unenroll",
"nimbus_experiment",
"foo", // slug
{
reason: "some-reason",
branch: experiment.branch.slug,
},
],
"should send an unenrollment ping with the slug, reason, and branch slug"
);
// Check that the Glean unenrollment event was recorded.
unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue();
// We expect only one event
Assert.equal(1, unenrollmentEvents.length);
// And that one event matches the expected enrolled experiment
Assert.equal(
experiment.slug,
unenrollmentEvents[0].extra.experiment,
"Glean.nimbusEvents.unenrollment recorded with correct experiment slug"
);
Assert.equal(
experiment.branch.slug,
unenrollmentEvents[0].extra.branch,
"Glean.nimbusEvents.unenrollment recorded with correct branch slug"
);
Assert.equal(
"some-reason",
unenrollmentEvents[0].extra.reason,
"Glean.nimbusEvents.unenrollment recorded with correct reason"
);
});
add_task(async function test_undefined_reason() {
globalSandbox.reset();
const manager = ExperimentFakes.manager();
const experiment = ExperimentFakes.experiment("foo");
// Clear any pre-existing data in Glean
Services.fog.testResetFOG();
await manager.onStartup();
await manager.store.addEnrollment(experiment);
manager.unenroll("foo");
const options = TelemetryEvents.sendEvent.firstCall?.args[3];
Assert.ok(
"reason" in options,
"options object with .reason should be the fourth param"
);
Assert.equal(
options.reason,
"unknown",
"should include unknown as the reason if none was supplied"
);
// Check that the Glean unenrollment event was recorded.
let unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue();
// We expect only one event
Assert.equal(1, unenrollmentEvents.length);
// And that one event reason matches the expected reason
Assert.equal(
"unknown",
unenrollmentEvents[0].extra.reason,
"Glean.nimbusEvents.unenrollment recorded with correct (unknown) reason"
);
});
/**
* Normal unenrollment for rollouts:
* - remove stored enrollment and synced data (prefs)
* - set rollout inactive in telemetry
* - send unrollment event
*/
add_task(async function test_remove_rollouts() {
const store = ExperimentFakes.store();
const manager = ExperimentFakes.manager(store);
const rollout = ExperimentFakes.rollout("foo");
sinon.stub(store, "get").returns(rollout);
sinon.spy(store, "updateExperiment");
await manager.onStartup();
manager.unenroll("foo", "some-reason");
Assert.ok(
manager.store.updateExperiment.calledOnce,
"Called to set the rollout as !active"
);
Assert.ok(
manager.store.updateExperiment.calledWith(rollout.slug, {
active: false,
unenrollReason: "some-reason",
}),
"Called with expected parameters"
);
});
add_task(async function test_remove_rollout_onFinalize() {
const store = ExperimentFakes.store();
const manager = ExperimentFakes.manager(store);
const rollout = ExperimentFakes.rollout("foo");
sinon.stub(store, "getAllActiveRollouts").returns([rollout]);
sinon.stub(store, "get").returns(rollout);
sinon.spy(manager, "unenroll");
sinon.spy(manager, "sendFailureTelemetry");
// Clear any pre-existing data in Glean
Services.fog.testResetFOG();
await manager.onStartup();
manager.onFinalize("NimbusTestUtils");
// Check that there aren't any Glean unenroll_failed events
var unenrollFailedEvents = Glean.nimbusEvents.unenrollFailed.testGetValue();
Assert.equal(
undefined,
unenrollFailedEvents,
"no Glean unenroll_failed events when removing rollout"
);
Assert.ok(manager.sendFailureTelemetry.notCalled, "Nothing should fail");
Assert.ok(manager.unenroll.calledOnce, "Should unenroll recipe not seen");
Assert.ok(manager.unenroll.calledWith(rollout.slug, "recipe-not-seen"));
});
add_task(async function test_rollout_telemetry_events() {
globalSandbox.restore();
const store = ExperimentFakes.store();
const manager = ExperimentFakes.manager(store);
const rollout = ExperimentFakes.rollout("foo");
globalSandbox.spy(TelemetryEnvironment, "setExperimentInactive");
globalSandbox.spy(TelemetryEvents, "sendEvent");
sinon.stub(store, "getAllActiveRollouts").returns([rollout]);
sinon.stub(store, "get").returns(rollout);
sinon.spy(manager, "sendFailureTelemetry");
// Clear any pre-existing data in Glean
Services.fog.testResetFOG();
await manager.onStartup();
// Check that there aren't any Glean unenrollment events yet
var unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue();
Assert.equal(
undefined,
unenrollmentEvents,
"no Glean unenrollment events before unenrollment"
);
manager.onFinalize("NimbusTestUtils");
// Check that there aren't any Glean unenroll_failed events
var unenrollFailedEvents = Glean.nimbusEvents.unenrollFailed.testGetValue();
Assert.equal(
undefined,
unenrollFailedEvents,
"no Glean unenroll_failed events when removing rollout"
);
Assert.ok(manager.sendFailureTelemetry.notCalled, "Nothing should fail");
Assert.ok(
TelemetryEnvironment.setExperimentInactive.calledOnce,
"Should unenroll recipe not seen"
);
Assert.ok(
TelemetryEnvironment.setExperimentInactive.calledWith(rollout.slug),
"Should set rollout to inactive."
);
// Test Glean experiment API interaction
Assert.equal(
undefined,
Services.fog.testGetExperimentData(rollout.slug),
"Should set rollout to inactive"
);
Assert.ok(
TelemetryEvents.sendEvent.calledWith(
"unenroll",
sinon.match.string,
rollout.slug,
sinon.match.object
),
"Should send unenroll event for rollout."
);
// Check that the Glean unenrollment event was recorded.
unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue();
// We expect only one event
Assert.equal(1, unenrollmentEvents.length);
// And that one event matches the expected enrolled experiment
Assert.equal(
rollout.slug,
unenrollmentEvents[0].extra.experiment,
"Glean.nimbusEvents.unenrollment recorded with correct rollout slug"
);
Assert.equal(
rollout.branch.slug,
unenrollmentEvents[0].extra.branch,
"Glean.nimbusEvents.unenrollment recorded with correct branch slug"
);
Assert.equal(
"recipe-not-seen",
unenrollmentEvents[0].extra.reason,
"Glean.nimbusEvents.unenrollment recorded with correct reason"
);
globalSandbox.restore();
});
add_task(async function test_check_unseen_enrollments_telemetry_events() {
globalSandbox.restore();
const store = ExperimentFakes.store();
const manager = ExperimentFakes.manager(store);
const sandbox = sinon.createSandbox();
sandbox.stub(manager, "unenroll").returns();
sandbox.stub(ExperimentAPI, "_store").get(() => manager.store);
sandbox.stub(ExperimentAPI, "_manager").get(() => manager);
await manager.onStartup();
await manager.store.ready();
const experiment = ExperimentFakes.recipe("foo", {
branches: [
{
slug: "wsup",
ratio: 1,
features: [
{
featureId: "nimbusTelemetry",
value: {
gleanMetricConfiguration: {
metrics_enabled: {
"nimbus_events.enrollment_status": true,
},
},
},
},
],
},
],
bucketConfig: {
...ExperimentFakes.recipe.bucketConfig,
count: 1000,
},
});
await manager.enroll(experiment, "aaa");
const source = "test";
const slugs = [],
experiments = [];
for (let i = 0; i < 7; i++) {
slugs.push(`slug-${i}`);
experiments.push({
slug: slugs[i],
source,
branch: {
slug: "control",
},
});
}
manager.sessions.set(source, new Set([slugs[0]]));
manager._checkUnseenEnrollments(
experiments,
source,
[slugs[1]],
[slugs[2]],
new Map([]),
new Map([[slugs[3], experiments[3]]]),
[slugs[4]],
new Map([[slugs[5], experiments[5]]])
);
const events = Glean.nimbusEvents.enrollmentStatus.testGetValue();
Assert.equal(events?.length, 7);
Assert.equal(events[0].extra.status, "Enrolled");
Assert.equal(events[0].extra.reason, "Qualified");
Assert.equal(events[0].extra.branch, "control");
Assert.equal(events[0].extra.slug, slugs[0]);
Assert.equal(events[1].extra.status, "Disqualified");
Assert.equal(events[1].extra.reason, "NotTargeted");
Assert.equal(events[1].extra.branch, "control");
Assert.equal(events[1].extra.slug, slugs[1]);
Assert.equal(events[2].extra.status, "Disqualified");
Assert.equal(events[2].extra.reason, "Error");
Assert.equal(events[2].extra.error_string, "invalid-recipe");
Assert.equal(events[2].extra.branch, "control");
Assert.equal(events[2].extra.slug, slugs[2]);
Assert.equal(events[3].extra.status, "Disqualified");
Assert.equal(events[3].extra.reason, "Error");
Assert.equal(events[3].extra.error_string, "invalid-branch");
Assert.equal(events[3].extra.branch, "control");
Assert.equal(events[3].extra.slug, slugs[3]);
Assert.equal(events[4].extra.status, "Disqualified");
Assert.equal(events[4].extra.reason, "Error");
Assert.equal(events[4].extra.error_string, "l10n-missing-locale");
Assert.equal(events[4].extra.branch, "control");
Assert.equal(events[4].extra.slug, slugs[4]);
Assert.equal(events[5].extra.status, "Disqualified");
Assert.equal(events[5].extra.reason, "Error");
Assert.equal(events[5].extra.error_string, "l10n-missing-entry");
Assert.equal(events[5].extra.branch, "control");
Assert.equal(events[5].extra.slug, slugs[5]);
Assert.equal(events[6].extra.status, "WasEnrolled");
Assert.equal(events[6].extra.branch, "control");
Assert.equal(events[6].extra.slug, slugs[6]);
sandbox.restore();
});