Source code
Revision control
Copy as Markdown
Other Tools
/* Any copyright is dedicated to the Public Domain.
"use strict";
const DEFAULT_CONTEXT = { telemetry: { location: "home" } };
async function setupEvaluation({ url, waitForLoad = true }) {
await SpecialPowers.pushPrefEnv({
set: [["services.settings.server", "data:,#remote-settings-dummy/v1"]],
});
const { RemoteSettingsClient } = ChromeUtils.importESModule(
);
const originalValidateCollectionSignature =
RemoteSettingsClient.prototype.validateCollectionSignature;
RemoteSettingsClient.prototype.validateCollectionSignature = async () => {};
const tab = await BrowserTestUtils.openNewForegroundTab(
gBrowser,
url,
waitForLoad
);
return {
tab,
async cleanup() {
info("Cleaning up");
RemoteSettingsClient.prototype.validateCollectionSignature =
originalValidateCollectionSignature;
await BrowserTestUtils.removeTab(tab);
await SpecialPowers.popPrefEnv();
},
};
}
/**
* Collect the full text response and tool calls from Chat.fetchWithHistory.
*
* @param {ChatConversation} conversation
* @param {object} engineInstance
* @returns {Promise<{ responseText: string, toolCalls: Array<{id: string, type: string, function: {name: string, arguments: string}}> }>}
*/
async function collectChatResponse(conversation, engineInstance) {
const { Chat } = ChromeUtils.importESModule(
"moz-src:///browser/components/aiwindow/models/Chat.sys.mjs"
);
await Chat.fetchWithHistory(conversation, engineInstance, DEFAULT_CONTEXT);
const messages = conversation.getMessagesInOpenAiFormat();
const lastAssistant = messages.findLast(msg => msg.role === "assistant");
const responseText = lastAssistant?.content ?? "";
const toolCalls = messages
.filter(msg => msg.tool_calls)
.flatMap(msg => msg.tool_calls);
return { responseText, toolCalls };
}
/**
* Report eval data out to stdout, which will be picked up by the test harness for
* analysis.
*
* @param {any} data - JSON serializable data.
*/
function reportEvalResult(data) {
info("evalDataPayload | " + JSON.stringify(data));
dump("-------------------------------------\n");
dump("Eval result:\n");
dump(JSON.stringify(data, null, 2));
dump("\n");
}
/**
* Renders a prompt from a string into a messages array, splitting on !role:[role]
* markers and replacing {placeholder} tokens with provided values.
*
* @param {string} rawPromptContent The raw prompt as a string
* @param {object} stringsToReplace A map of placeholder strings to their replacements
* @returns {Array<{role: string, content: string}>}
*/
function renderPrompt(rawPromptContent, stringsToReplace = {}) {
const roleRegex = /!role:\[(\w+)\]/g;
const messages = [];
let lastIndex = 0;
let lastRole = null;
let match;
while ((match = roleRegex.exec(rawPromptContent)) !== null) {
if (lastRole !== null) {
messages.push({
role: lastRole,
content: rawPromptContent.slice(lastIndex, match.index).trim(),
});
}
lastRole = match[1];
lastIndex = match.index + match[0].length;
}
if (lastRole !== null) {
messages.push({
role: lastRole,
content: rawPromptContent.slice(lastIndex).trim(),
});
}
for (const message of messages) {
for (const [orig, repl] of Object.entries(stringsToReplace)) {
message.content = message.content.replace(
new RegExp(`\\{${orig}\\}`, "g"),
() => repl
);
}
}
return messages;
}