Source code
Revision control
Copy as Markdown
Other Tools
Test Info:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import json
import multiprocessing
import os
import subprocess
import sys
import tempfile
import time
import unittest
from unittest import mock
import mozunit
import pytest
try:
import psutil
except ImportError:
psutil = None
from mozsystemmonitor.resourcemonitor import (
FORWARD_ENV,
SystemResourceMonitor,
SystemResourceUsage,
_parse_hg_source_url,
_process_sampling_interval,
)
UBSAN_FILE = "/builds/worker/checkouts/gecko/third_party/llama.cpp/ggml/src/ggml-c.c"
UBSAN_MESSAGE = "applying non-zero offset 96 to null pointer"
UBSAN_LINES = [
f"{UBSAN_FILE}:7106:33: runtime error: {UBSAN_MESSAGE}",
f" #0 0x7b417858d37c in incr_ptr_aligned {UBSAN_FILE}:7106:33",
" #1 0x7f42401b0a33 in clone misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:100",
f"SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior {UBSAN_FILE}:7106:33",
]
ASAN_FILE = "/builds/worker/checkouts/gecko/dom/base/SpeculationRules.cpp"
ASAN_FUNCTION = "mozilla::dom::SpeculationRules::RemoveLink(mozilla::dom::Element*)"
ASAN_MESSAGE = (
"SEGV on unknown address 0x00000000030b "
"(pc 0x7b03e9dd40af bp 0x7fff627fb7f0 sp 0x7fff627fb7e0 T0)"
)
ASAN_LINES = [
f"==28130==ERROR: AddressSanitizer: {ASAN_MESSAGE}",
"==28130==The signal is caused by a READ memory access.",
f" #0 0x7b03e9dd40af in {ASAN_FUNCTION} {ASAN_FILE}:385:7",
" #1 0x561bbfeecf08 in _start (/builds/worker/workspace/build/application/firefox/firefox+0xc1f08)",
f"SUMMARY: AddressSanitizer: SEGV {ASAN_FILE}:385:7 in {ASAN_FUNCTION}",
"==28130==ABORTING",
]
# Monitors created by the tests shouldn't forward to the one running the tests
# in automation.
@pytest.fixture(autouse=True)
def no_forwarding(monkeypatch):
monkeypatch.delenv(FORWARD_ENV, raising=False)
def _wait_for_measurement(monitor, timeout=30):
"""Wait for the first measurement, which takes a while when the
collecting process is spawned rather than forked, e.g. on macOS."""
deadline = time.monotonic() + timeout
while not monitor.measurements:
assert time.monotonic() < deadline, "no measurement received"
monitor._drain_pipe()
time.sleep(0.05)
@unittest.skipIf(psutil is None, "Resource monitor requires psutil.")
class TestResourceMonitor(unittest.TestCase):
def test_basic(self):
monitor = SystemResourceMonitor(poll_interval=0.5)
monitor.start()
time.sleep(3)
monitor.stop()
data = list(monitor.range_usage())
self.assertGreater(len(data), 3)
self.assertIsInstance(data[0], SystemResourceUsage)
def test_empty(self):
monitor = SystemResourceMonitor(poll_interval=2.0)
monitor.start()
monitor.stop()
data = list(monitor.range_usage())
self.assertEqual(len(data), 0)
def test_phases(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
time.sleep(1)
with monitor.phase("phase1"):
time.sleep(1)
with monitor.phase("phase2"):
time.sleep(1)
monitor.stop()
self.assertEqual(len(monitor.phases), 2)
self.assertEqual(["phase2", "phase1"], list(monitor.phases.keys()))
all = list(monitor.range_usage())
data1 = list(monitor.phase_usage("phase1"))
data2 = list(monitor.phase_usage("phase2"))
self.assertGreater(len(all), len(data1))
self.assertGreater(len(data1), len(data2))
# This could fail if time.monotonic() takes more than 0.1s. It really
# shouldn't.
self.assertAlmostEqual(data1[-1].end, data2[-1].end, delta=0.25)
def test_no_data(self):
monitor = SystemResourceMonitor()
data = list(monitor.range_usage())
self.assertEqual(len(data), 0)
def test_events(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
time.sleep(0.5)
t0 = time.monotonic()
monitor.record_event("t0")
time.sleep(2)
monitor.record_event("t1")
time.sleep(0.5)
monitor.stop()
events = monitor.events
self.assertEqual(len(events), 2)
event = events[0]
self.assertEqual(event[1], "t0")
self.assertAlmostEqual(event[0], t0, delta=0.25)
data = list(monitor.between_events_usage("t0", "t1"))
self.assertGreater(len(data), 0)
def test_aggregate_cpu(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
_wait_for_measurement(monitor)
time.sleep(1)
monitor.stop()
values = monitor.aggregate_cpu_percent()
self.assertIsInstance(values, list)
self.assertEqual(len(values), multiprocessing.cpu_count())
for v in values:
self.assertIsInstance(v, float)
value = monitor.aggregate_cpu_percent(per_cpu=False)
self.assertIsInstance(value, float)
values = monitor.aggregate_cpu_times()
self.assertIsInstance(values, list)
self.assertGreater(len(values), 0)
self.assertTrue(hasattr(values[0], "user"))
t = type(values[0])
value = monitor.aggregate_cpu_times(per_cpu=False)
self.assertIsInstance(value, t)
def test_aggregate_io(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
# There's really no easy way to ensure I/O occurs. For all we know
# reads and writes will all be serviced by the page cache.
monitor.start()
time.sleep(1.0)
monitor.stop()
values = monitor.aggregate_io()
self.assertTrue(hasattr(values, "read_count"))
def test_memory(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
time.sleep(1.0)
monitor.stop()
v = monitor.min_memory_available()
self.assertIsInstance(v, int)
v = monitor.max_memory_percent()
self.assertIsInstance(v, float)
def test_lsan_events(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
time.sleep(0.1)
# Direct leak with a stack and an allow-list match -> yellow.
SystemResourceMonitor.lsan_leak({
"time": int(time.time() * 1000),
"kind": "Direct",
"bytes": 128,
"objects": 2,
"stack": [{"function": "Foo::Bar"}, {"function": "main"}],
"scope": "browser/foo",
"allowed_match": "Foo::Bar",
})
# Indirect leak without allow-list match -> orange, no stack/scope.
SystemResourceMonitor.lsan_leak({
"time": int(time.time() * 1000),
"kind": "Indirect",
"bytes": 32,
"objects": 1,
})
# Allow-listed summary -> yellow.
SystemResourceMonitor.lsan_summary({
"time": int(time.time() * 1000),
"bytes": 160,
"allocations": 3,
"allowed": True,
})
time.sleep(0.1)
monitor.stop()
leak_events = [e for e in monitor.events if e[1] == "LSan Leak"]
summary_events = [e for e in monitor.events if e[1] == "LSan Summary"]
self.assertEqual(len(leak_events), 2)
self.assertEqual(len(summary_events), 1)
direct, indirect = leak_events[0][2], leak_events[1][2]
self.assertEqual(direct["type"], "LSanLeak")
self.assertEqual(direct["kind"], "Direct")
self.assertEqual(direct["bytes"], 128)
self.assertEqual(direct["objects"], 2)
self.assertEqual(direct["scope"], "browser/foo")
self.assertEqual(direct["allowed_match"], "Foo::Bar")
self.assertEqual(direct["color"], "yellow")
self.assertEqual(direct["stack"][0]["function"], "Foo::Bar")
self.assertEqual(indirect["kind"], "Indirect")
self.assertEqual(indirect["color"], "orange")
self.assertNotIn("stack", indirect)
self.assertNotIn("scope", indirect)
self.assertNotIn("allowed_match", indirect)
summary = summary_events[0][2]
self.assertEqual(summary["type"], "LSanSummary")
self.assertEqual(summary["bytes"], 160)
self.assertEqual(summary["allocations"], 3)
self.assertEqual(summary["color"], "yellow")
self.assertTrue(summary["allowed"])
markers = monitor.as_profile()["threads"][0]["markers"]["data"]
self.assertTrue(any(m.get("type") == "LSanLeak" for m in markers))
self.assertTrue(any(m.get("type") == "LSanSummary" for m in markers))
def test_tsan_error(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
time.sleep(0.1)
now = int(time.time() * 1000)
# A lock-order inversion with two labeled acquisition stacks: it should
# produce two markers sharing a report_index, each carrying one stack.
SystemResourceMonitor.tsan_error({
"time": now,
"kind": "lock-order-inversion (potential deadlock)",
"signature": "Mutex_posix.cpp:91:3 in mutexLock",
"pid": 1234,
"description": "Cycle in lock order graph: M0 => M1 => M0",
"scope": "browser/foo",
"stacks": [
{
"label": "Mutex M1 acquired here while holding mutex M0",
"stack": [
{
"function": "mutexLock",
"module": "firefox",
"file": "/builds/worker/checkouts/gecko/mozglue/misc/Mutex_posix.cpp",
"line": 91,
},
{"function": "main", "module": "firefox"},
],
},
{
"label": "Mutex M0 acquired here while holding mutex M1",
"stack": [{"function": "other", "module": "libxul.so"}],
},
],
})
# A report with no stacks should still produce a single marker.
SystemResourceMonitor.tsan_error({
"time": now,
"kind": "data race",
"signature": "Activation.cpp:16 in registerProfiling",
"stacks": [],
})
time.sleep(0.1)
monitor.stop()
events = [e[2] for e in monitor.events if e[1] == "TSan Error"]
# 2 markers for the lock-order report + 1 for the stackless data race.
self.assertEqual(len(events), 3)
lock_order = [e for e in events if e["kind"].startswith("lock-order-inversion")]
self.assertEqual(len(lock_order), 2)
first, second = lock_order
self.assertEqual(first["type"], "TSanError")
self.assertEqual(first["color"], "orange")
self.assertEqual(first["pid"], 1234)
self.assertEqual(first["scope"], "browser/foo")
# Both markers of one report share a report_index.
self.assertEqual(first["report_index"], second["report_index"])
self.assertNotEqual(first["label"], second["label"])
self.assertEqual(first["stack"][0]["function"], "mutexLock")
# checkouts/gecko paths are rewritten to a repo-relative/hg view.
self.assertNotIn("/builds/worker/checkouts/gecko/", first["stack"][0]["file"])
data_race = [e for e in events if e["kind"] == "data race"]
self.assertEqual(len(data_race), 1)
self.assertNotIn("stack", data_race[0])
# The two reports get distinct report_index values.
self.assertNotEqual(first["report_index"], data_race[0]["report_index"])
markers = monitor.as_profile()["threads"][0]["markers"]["data"]
self.assertTrue(any(m.get("type") == "TSanError" for m in markers))
def _ubsan_action(self, message=UBSAN_MESSAGE, stack=None, **extra):
data = {
"time": int(time.time() * 1000),
"kind": "undefined-behavior",
"message": message,
"file": UBSAN_FILE,
"lineno": 7106,
"column": 33,
}
if stack is not None:
data["stack"] = stack
data.update(extra)
SystemResourceMonitor.ubsan_error(data)
def test_ubsan_error(self):
rev = "56b3cc68b5e7557a3e13fca984f0f8aebc60dd22"
monitor = SystemResourceMonitor(
poll_interval=0.25,
)
monitor.start()
self._ubsan_action(
scope="browser/foo",
test="test_foo",
stack=[
{
"function": "incr_ptr_aligned",
"file": UBSAN_FILE,
"line": 7106,
"column": 33,
},
{"function": "clone", "module": "libc.so.6", "module_offset": 0x1000},
],
)
# A report without location or stack still produces a marker.
SystemResourceMonitor.ubsan_error({
"time": int(time.time() * 1000),
"kind": "undefined-behavior",
"message": "division by zero",
})
monitor.stop()
events = self._events(monitor, "UBSan Error")
self.assertEqual(len(events), 2)
first, second = events[0][2], events[1][2]
self.assertEqual(first["type"], "UBSanError")
self.assertEqual(first["color"], "orange")
self.assertEqual(first["kind"], "undefined-behavior")
self.assertEqual(first["message"], UBSAN_MESSAGE)
self.assertEqual(first["file"], "third_party/llama.cpp/ggml/src/ggml-c.c")
self.assertEqual(first["line"], 7106)
self.assertEqual(first["column"], 33)
self.assertEqual(first["scope"], "browser/foo")
self.assertEqual(first["test"], "test_foo")
self.assertEqual(
first["stack"][0]["file"],
f"hg:hg.mozilla.org/try:third_party/llama.cpp/ggml/src/ggml-c.c:{rev}",
)
self.assertNotIn("file", first["stack"][1])
self.assertEqual(second["message"], "division by zero")
self.assertNotIn("file", second)
self.assertNotIn("stack", second)
markers = monitor.as_profile()["threads"][0]["markers"]["data"]
self.assertEqual(len([m for m in markers if m.get("type") == "UBSanError"]), 2)
def test_ubsan_error_without_stack_points_at_the_location(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._ubsan_action()
monitor.stop()
events = self._events(monitor, "UBSan Error")
self.assertEqual(len(events), 1)
self.assertEqual(
events[0][2]["stack"],
[{"file": "third_party/llama.cpp/ggml/src/ggml-c.c", "line": 7106}],
)
def test_process_output_ubsan_report(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
for line in UBSAN_LINES:
self._process_output(monitor, line, process="GECKO(1717)")
self._process_output(monitor, "unrelated output", process="GECKO(1717)")
monitor.stop()
events = self._events(monitor, "UBSan Error")
self.assertEqual(len(events), 1)
data = events[0][2]
self.assertEqual(data["type"], "UBSanError")
self.assertEqual(data["message"], UBSAN_MESSAGE)
self.assertEqual(data["file"], "third_party/llama.cpp/ggml/src/ggml-c.c")
self.assertEqual(data["line"], 7106)
self.assertEqual(data["column"], 33)
self.assertEqual(
[f["function"] for f in data["stack"]], ["incr_ptr_aligned", "clone"]
)
self.assertEqual(
data["stack"][0]["file"], "third_party/llama.cpp/ggml/src/ggml-c.c"
)
# The report lines are folded into the marker; only the unrelated
# line becomes a generic output marker.
self.assertEqual(len(self._events(monitor, "output")), 1)
def test_ubsan_error_action_supersedes_line_report(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
# Harnesses feed their parser before logging each line, so the
# action arrives after the header and frames but before SUMMARY.
for line in UBSAN_LINES[:3]:
self._process_output(monitor, line, process="GECKO(1717)")
action_stack = [
{"function": "incr_ptr_aligned", "file": UBSAN_FILE, "line": 7106},
{"function": "ggml_graph_nbytes", "file": UBSAN_FILE, "line": 7113},
{"function": "clone"},
]
self._ubsan_action(stack=action_stack)
self._process_output(monitor, UBSAN_LINES[3], process="GECKO(1717)")
self._process_output(monitor, "unrelated output", process="GECKO(1717)")
monitor.stop()
events = self._events(monitor, "UBSan Error")
self.assertEqual(len(events), 1)
self.assertEqual(len(events[0][2]["stack"]), 3)
self.assertEqual(len(self._events(monitor, "output")), 1)
def test_ubsan_line_report_completing_after_action_is_dropped(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._ubsan_action()
for line in UBSAN_LINES:
self._process_output(monitor, line, process="GECKO(1717)")
self.assertEqual(len(self._events(monitor, "UBSan Error")), 1)
# A second identical report is a new one.
for line in UBSAN_LINES:
self._process_output(monitor, line, process="GECKO(1717)")
monitor.stop()
self.assertEqual(len(self._events(monitor, "UBSan Error")), 2)
self.assertEqual(len(self._events(monitor, "output")), 0)
def test_process_output_ubsan_flushed_on_stop(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
for line in UBSAN_LINES[:2]:
self._process_output(monitor, line, process="GECKO(1717)")
self.assertEqual(len(self._events(monitor, "UBSan Error")), 0)
monitor.stop()
events = self._events(monitor, "UBSan Error")
self.assertEqual(len(events), 1)
self.assertEqual(len(events[0][2]["stack"]), 1)
def test_process_output_ubsan_keyed_by_process(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
other = [
"/src/a.c:12:3: runtime error: division by zero",
"SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /src/a.c:12:3",
]
self._process_output(monitor, UBSAN_LINES[0], process="A")
self._process_output(monitor, other[0], process="B")
self._process_output(monitor, UBSAN_LINES[1], process="A")
self._process_output(monitor, other[1], process="B")
for line in UBSAN_LINES[2:]:
self._process_output(monitor, line, process="A")
monitor.stop()
events = self._events(monitor, "UBSan Error")
self.assertEqual(
[e[2]["message"] for e in events], ["division by zero", UBSAN_MESSAGE]
)
self.assertEqual(len(events[1][2]["stack"]), 2)
self.assertEqual(len(self._events(monitor, "output")), 0)
def test_process_output_ubsan_without_moztest(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor._ubsan_parser = None
monitor.start()
for line in UBSAN_LINES:
self._process_output(monitor, line, process="GECKO(1717)")
monitor.stop()
self.assertEqual(len(self._events(monitor, "UBSan Error")), 0)
self.assertEqual(len(self._events(monitor, "output")), 4)
def _asan_action(self, message=ASAN_MESSAGE, stack=None, **extra):
data = {
"time": int(time.time() * 1000),
"kind": "SEGV",
"message": message,
"pid": 28130,
"file": ASAN_FILE,
"lineno": 385,
"column": 7,
"function": ASAN_FUNCTION,
}
if stack is not None:
data["stack"] = stack
data.update(extra)
SystemResourceMonitor.asan_error(data)
def test_asan_error(self):
rev = "56b3cc68b5e7557a3e13fca984f0f8aebc60dd22"
monitor = SystemResourceMonitor(
poll_interval=0.25,
)
monitor.start()
self._asan_action(
scope="/css/selectors",
test="/css/selectors/a.html",
stack=[
{
"function": ASAN_FUNCTION,
"file": ASAN_FILE,
"line": 385,
"column": 7,
},
{"function": "_start", "module": "firefox", "module_offset": 0xC1F08},
],
stacks=[
{
"label": "freed by thread T0 here",
"stack": [{"function": "free", "file": ASAN_FILE, "line": 12}],
}
],
)
# A report of a run that will be retried is a warning; one without
# location or stack still produces a marker.
SystemResourceMonitor.asan_error({
"time": int(time.time() * 1000),
"kind": "double-free",
"message": "attempting double-free on 0x1 in thread T0:",
"quiet": True,
})
monitor.stop()
events = self._events(monitor, "ASan Error")
self.assertEqual(len(events), 2)
first, second = events[0][2], events[1][2]
self.assertEqual(first["type"], "ASanError")
self.assertEqual(first["color"], "red")
self.assertEqual(first["kind"], "SEGV")
self.assertEqual(first["message"], ASAN_MESSAGE)
self.assertEqual(first["pid"], 28130)
self.assertEqual(first["file"], "dom/base/SpeculationRules.cpp")
self.assertEqual(first["line"], 385)
self.assertEqual(first["column"], 7)
self.assertEqual(first["function"], ASAN_FUNCTION)
self.assertEqual(first["location"], "dom/base/SpeculationRules.cpp:385")
self.assertEqual(first["scope"], "/css/selectors")
self.assertEqual(first["test"], "/css/selectors/a.html")
self.assertEqual(
first["stack"][0]["file"],
f"hg:hg.mozilla.org/try:dom/base/SpeculationRules.cpp:{rev}",
)
self.assertNotIn("file", first["stack"][1])
self.assertEqual(first["stacks"][0]["label"], "freed by thread T0 here")
self.assertEqual(
first["stacks"][0]["stack"],
[
{
"function": "free",
"file": f"hg:hg.mozilla.org/try:dom/base/SpeculationRules.cpp:{rev}",
"line": 12,
}
],
)
self.assertEqual(second["color"], "orange")
self.assertEqual(second["kind"], "double-free")
self.assertEqual(
second["location"], "attempting double-free on 0x1 in thread T0:"
)
self.assertNotIn("pid", second)
self.assertNotIn("file", second)
self.assertNotIn("stack", second)
self.assertNotIn("stacks", second)
markers = monitor.as_profile()["threads"][0]["markers"]["data"]
self.assertEqual(len([m for m in markers if m.get("type") == "ASanError"]), 2)
def test_asan_error_without_stack_points_at_the_location(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._asan_action()
monitor.stop()
events = self._events(monitor, "ASan Error")
self.assertEqual(len(events), 1)
self.assertEqual(
events[0][2]["stack"],
[{"file": "dom/base/SpeculationRules.cpp", "line": 385}],
)
def test_process_output_asan_report(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(
monitor, "AddressSanitizer:DEADLYSIGNAL", process="GECKO(19529)"
)
for line in ASAN_LINES[:2]:
self._process_output(monitor, line, process="GECKO(19529)")
# Another process's output interleaves while the stack is symbolized.
self._process_output(
monitor,
"TEST-OK | /css/selectors/a.html | took 1ms",
process="GECKO(19529)",
)
for line in ASAN_LINES[2:]:
self._process_output(monitor, line, process="GECKO(19529)")
monitor.stop()
events = self._events(monitor, "ASan Error")
self.assertEqual(len(events), 1)
data = events[0][2]
self.assertEqual(data["type"], "ASanError")
self.assertEqual(data["color"], "red")
self.assertEqual(data["kind"], "SEGV")
self.assertEqual(data["message"], ASAN_MESSAGE)
self.assertEqual(data["pid"], 28130)
self.assertEqual(data["file"], "dom/base/SpeculationRules.cpp")
self.assertEqual(data["line"], 385)
self.assertEqual(data["column"], 7)
self.assertEqual(data["function"], ASAN_FUNCTION)
self.assertEqual(
[f["function"] for f in data["stack"]], [ASAN_FUNCTION, "_start"]
)
self.assertEqual(data["stack"][0]["file"], "dom/base/SpeculationRules.cpp")
# The report lines are folded into the marker; only the interleaved
# line becomes a generic output marker.
self.assertEqual(len(self._events(monitor, "output")), 1)
def test_asan_error_action_supersedes_line_report(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
# Harnesses feed their parser before logging each line, so the
# action arrives after the header and frames but before SUMMARY.
for line in ASAN_LINES[:4]:
self._process_output(monitor, line, process="GECKO(19529)")
action_stack = [
{"function": "GetInnerWindow", "file": ASAN_FILE, "line": 1942},
{"function": ASAN_FUNCTION, "file": ASAN_FILE, "line": 385},
{"function": "_start"},
]
self._asan_action(stack=action_stack)
for line in ASAN_LINES[4:]:
self._process_output(monitor, line, process="GECKO(19529)")
self._process_output(monitor, "unrelated output", process="GECKO(19529)")
monitor.stop()
events = self._events(monitor, "ASan Error")
self.assertEqual(len(events), 1)
self.assertEqual(len(events[0][2]["stack"]), 3)
self.assertEqual(len(self._events(monitor, "output")), 1)
def test_asan_line_report_completing_after_action_is_dropped(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._asan_action()
for line in ASAN_LINES:
self._process_output(monitor, line, process="GECKO(19529)")
self.assertEqual(len(self._events(monitor, "ASan Error")), 1)
# A second identical report is a new one.
for line in ASAN_LINES:
self._process_output(monitor, line, process="GECKO(19529)")
monitor.stop()
self.assertEqual(len(self._events(monitor, "ASan Error")), 2)
self.assertEqual(len(self._events(monitor, "output")), 0)
def test_process_output_asan_flushed_on_stop(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
for line in ASAN_LINES[:3]:
self._process_output(monitor, line, process="GECKO(19529)")
self.assertEqual(len(self._events(monitor, "ASan Error")), 0)
monitor.stop()
events = self._events(monitor, "ASan Error")
self.assertEqual(len(events), 1)
self.assertEqual(len(events[0][2]["stack"]), 1)
self.assertNotIn("file", events[0][2])
self.assertEqual(events[0][2]["location"], ASAN_FUNCTION)
def test_process_output_asan_without_mozleak(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor._asan_parser = None
monitor.start()
for line in ASAN_LINES:
self._process_output(monitor, line, process="GECKO(19529)")
monitor.stop()
self.assertEqual(len(self._events(monitor, "ASan Error")), 0)
self.assertEqual(len(self._events(monitor, "output")), len(ASAN_LINES))
def test_process_output_lsan_summary_is_not_an_asan_error(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
for line in (
"==1689==ERROR: LeakSanitizer: detected memory leaks",
"SUMMARY: AddressSanitizer: 488 byte(s) leaked in 3 allocation(s).",
):
self._process_output(monitor, line, process="GECKO(1689)")
monitor.stop()
self.assertEqual(len(self._events(monitor, "ASan Error")), 0)
self.assertEqual(len(self._events(monitor, "output")), 2)
def test_as_profile(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
time.sleep(0.1)
monitor.begin_phase("phase1")
monitor.record_event("foo")
time.sleep(0.1)
monitor.begin_phase("phase2")
monitor.record_event("bar")
time.sleep(0.2)
monitor.finish_phase("phase1")
time.sleep(0.2)
monitor.finish_phase("phase2")
time.sleep(0.4)
monitor.stop()
d = monitor.as_profile()
self.assertEqual(len(d["threads"]), 1)
self.assertIn("markers", d["threads"][0])
self.assertIn("data", d["threads"][0]["markers"])
markers = d["threads"][0]["markers"]["data"]
self.assertTrue(
any(m["type"] == "Phase" and m["phase"] == "phase1" for m in markers)
)
self.assertTrue(
any(m["type"] == "Phase" and m["phase"] == "phase2" for m in markers)
)
self.assertIn({"type": "Text", "text": "foo"}, markers)
self.assertIn({"type": "Text", "text": "bar"}, markers)
def _test_status(self, monitor, **kwargs):
data = {
"action": "test_status",
"test": "test_foo.js",
"time": (time.monotonic() - monitor.start_time) * 1000
+ monitor.start_timestamp * 1000,
}
data.update(kwargs)
SystemResourceMonitor.test_status(data)
def test_status_expectations(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
# mozlog drops "expected" when it matches "status", so an expected
# failure arrives without it.
self._test_status(monitor, status="FAIL", message="a todo")
self._test_status(monitor, status="FAIL", expected="PASS", message="a failure")
self._test_status(monitor, status="PASS", message="a pass")
self._test_status(
monitor, status="PASS", expected="FAIL", message="a todo that passed"
)
self._test_status(
monitor,
status="FAIL",
message="a failure that will be retried",
extra={"will_retry": True},
)
monitor.stop()
by_message = {
data["message"]: (name, data)
for _, name, data in monitor.events
if data and data.get("type") == "TestStatus"
}
self.assertEqual(len(by_message), 5)
name, data = by_message["a todo"]
self.assertEqual(name, "KNOWN-FAIL")
self.assertEqual(data["color"], "yellow")
name, data = by_message["a failure"]
self.assertEqual(name, "FAIL")
self.assertEqual(data["color"], "orange")
name, data = by_message["a pass"]
self.assertEqual(name, "PASS")
self.assertEqual(data["color"], "green")
name, data = by_message["a todo that passed"]
self.assertEqual(name, "UNEXPECTED-PASS")
self.assertEqual(data["color"], "orange")
name, data = by_message["a failure that will be retried"]
self.assertEqual(name, "FAIL")
self.assertEqual(data["color"], "orange")
def _process_output(self, monitor, line, process=None):
data = {
"action": "process_output",
"data": line,
"time": (time.monotonic() - monitor.start_time) * 1000
+ monitor.start_timestamp * 1000,
}
if process is not None:
data["process"] = process
SystemResourceMonitor.test_status(data)
def _events(self, monitor, name):
return [e for e in monitor.events if len(e) == 3 and e[1] == name]
def test_process_output_docshell(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(
monitor,
"[Child 4208: Main Thread]: I/DocShellAndDOMWindowLeak "
"++DOCSHELL 2f804b00 == 2 [pid = 4208] [id = 37]",
)
time.sleep(0.05)
self._process_output(
monitor,
"[Child 4208: Main Thread]: I/DocShellAndDOMWindowLeak "
"--DOCSHELL 2f804b00 == 0 [pid = 4208] [id = 37] "
"[url = about:aichatcontent]",
)
monitor.stop()
docshell = [m for m in monitor.markers if m[0] == "DocShell"]
self.assertEqual(len(docshell), 1)
name, start, end, data, _ = docshell[0]
self.assertEqual(data["type"], "DocShell")
self.assertEqual(data["url"], "about:aichatcontent")
self.assertEqual(data["id"], 37)
self.assertEqual(data["pid"], 4208)
self.assertEqual(data["process"], "Child")
self.assertEqual(data["thread"], "Main Thread")
self.assertEqual(data["pointer"], "2f804b00")
self.assertLess(start, end)
def test_process_output_domwindow(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(
monitor,
"[Child 3444: Main Thread]: I/DocShellAndDOMWindowLeak "
"++DOMWINDOW == 2 (b3bc400) [pid = 3444] [serial = 2] "
"[outer = 4f60940]",
)
time.sleep(0.05)
self._process_output(
monitor,
"[Child 3444: Main Thread]: I/DocShellAndDOMWindowLeak "
"--DOMWINDOW == 1 (b3bc400) [pid = 3444] [serial = 2] "
"[outer = 4f60940] [url = about:blank]",
)
monitor.stop()
windows = [m for m in monitor.markers if m[0] == "DOMWindow"]
self.assertEqual(len(windows), 1)
_, start, end, data, _ = windows[0]
self.assertEqual(data["type"], "DOMWindow")
self.assertEqual(data["url"], "about:blank")
self.assertEqual(data["serial"], 2)
self.assertEqual(data["pointer"], "b3bc400")
self.assertEqual(data["outer"], "4f60940")
self.assertLess(start, end)
def test_process_output_javascript_error(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(
monitor,
"JavaScript error: chrome://browser/content/places/browserPlacesViews.js,"
" line 118: Error: No DOM node set for aPlacesNode.",
)
self._process_output(
monitor,
)
monitor.stop()
errors = [
e for e in monitor.events if len(e) == 3 and e[1] == "JavaScript error"
]
warns = [
e for e in monitor.events if len(e) == 3 and e[1] == "JavaScript warning"
]
self.assertEqual(len(errors), 1)
self.assertEqual(len(warns), 1)
_, _, data = errors[0]
self.assertEqual(data["type"], "jsError")
self.assertNotIn("level", data)
self.assertNotIn("color", data)
self.assertEqual(
data["file"], "chrome://browser/content/places/browserPlacesViews.js"
)
self.assertEqual(data["line"], 118)
self.assertIn("No DOM node set", data["message"])
self.assertEqual(data["stack"][0]["is_js"], True)
def test_process_output_cpp_warning(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
# Both ": file" and ", file" separator forms occur in practice.
self._process_output(
monitor,
"[Parent 72612, Main Thread] WARNING: NS_ENSURE_TRUE(uri) failed: "
"file caps/BasePrincipal.cpp:1511",
)
self._process_output(
monitor,
"[Child 19675, Main Thread] WARNING: 'NS_FAILED(rv)', "
"file checkouts/gecko/xpcom/threads/nsThreadUtils.cpp:238",
)
monitor.stop()
warnings_ = [e for e in monitor.events if len(e) == 3 and e[1] == "C++ warning"]
self.assertEqual(len(warnings_), 2)
_, _, data = warnings_[0]
self.assertEqual(data["type"], "cppDebug")
self.assertEqual(data["message"], "NS_ENSURE_TRUE(uri) failed")
self.assertEqual(data["file"], "caps/BasePrincipal.cpp")
self.assertEqual(data["line"], 1511)
self.assertEqual(data["pid"], 72612)
self.assertEqual(data["process"], "Parent")
self.assertEqual(data["thread"], "Main Thread")
self.assertNotIn("color", data)
# Comma-separator form preserves the quoted message verbatim.
_, _, data = warnings_[1]
self.assertEqual(data["message"], "'NS_FAILED(rv)'")
# NS_DebugBreak rings the bell before the assertion text.
NS_ASSERTION_LINE = (
"\x07[Parent 5900, Main Thread] ###!!! ASSERTION: Out-of-flow frame got "
"reflowed before its placeholder: 'Error', file "
"layout/generic/nsPlaceholderFrame.cpp:131"
)
NS_ASSERTION_FRAMES = (
"#01: NS_DebugBreak(unsigned int, char const*, char const*, char const*, int) "
"[xpcom/base/nsDebugImpl.cpp:516]",
"#02: nsPlaceholderFrame::Reflow(nsPresContext*) "
"[layout/generic/nsPlaceholderFrame.cpp:131]",
)
MOZ_ASSERT_LINE = (
"[6136] Assertion failure: false (unreachable), at "
"/builds/worker/checkouts/gecko/dom/workers/RuntimeService.cpp:2229"
)
def test_process_output_cpp_assertion(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(monitor, self.NS_ASSERTION_LINE)
for frame in self.NS_ASSERTION_FRAMES:
self._process_output(monitor, frame)
self._process_output(monitor, "totally random output line that we don't parse")
monitor.stop()
asserts = self._events(monitor, "NS_ASSERTION")
self.assertEqual(len(asserts), 1)
_, _, data = asserts[0]
self.assertEqual(data["type"], "AssertionFailure")
self.assertEqual(data["kind"], "NS_ASSERTION")
self.assertEqual(data["color"], "orange")
self.assertNotIn("fatal", data)
self.assertIn("Out-of-flow frame", data["message"])
self.assertEqual(data["file"], "layout/generic/nsPlaceholderFrame.cpp")
self.assertEqual(data["line"], 131)
self.assertEqual(data["pid"], 5900)
self.assertEqual(data["process"], "Parent")
self.assertEqual(data["thread"], "Main Thread")
self.assertEqual(
[f["function"] for f in data["stack"]],
[
"NS_DebugBreak(unsigned int, char const*, char const*, char const*, int)",
"nsPlaceholderFrame::Reflow(nsPresContext*)",
],
)
self.assertEqual(data["stack"][1]["line"], 131)
# The frames were folded into the assertion, only the trailing line
# is a generic output marker.
self.assertEqual(len(self._events(monitor, "output")), 1)
def test_process_output_assertion_flushed_on_stop(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(monitor, self.MOZ_ASSERT_LINE)
self._process_output(monitor, "#01: f() [dom/workers/RuntimeService.cpp:2229]")
self.assertEqual(self._events(monitor, "MOZ_ASSERT"), [])
monitor.stop()
asserts = self._events(monitor, "MOZ_ASSERT")
self.assertEqual(len(asserts), 1)
_, _, data = asserts[0]
self.assertEqual(data["color"], "red")
self.assertTrue(data["fatal"])
self.assertEqual(data["file"], "dom/workers/RuntimeService.cpp")
self.assertEqual(data["pid"], 6136)
self.assertNotIn("process", data)
self.assertEqual(
data["stack"],
[
{
"function": "f()",
"file": "dom/workers/RuntimeService.cpp",
"line": 2229,
}
],
)
def test_process_output_xpcshell_assertion_without_frames(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(
monitor,
"[123, Main Thread] ###!!! ASSERTION: boom: 'expr', file xpcom/a.cpp:7",
)
self._process_output(monitor, "unrelated")
monitor.stop()
asserts = self._events(monitor, "NS_ASSERTION")
self.assertEqual(len(asserts), 1)
_, _, data = asserts[0]
self.assertEqual(data["pid"], 123)
self.assertNotIn("process", data)
# Without frames the source location still gives the marker a cause.
self.assertEqual(data["stack"], [{"file": "xpcom/a.cpp", "line": 7}])
def test_process_output_assertions_keyed_by_process(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(monitor, self.MOZ_ASSERT_LINE, process="A")
self._process_output(monitor, self.NS_ASSERTION_LINE, process="B")
self._process_output(monitor, "#01: a() [a.cpp:1]", process="A")
self._process_output(monitor, "#01: b() [b.cpp:1]", process="B")
monitor.stop()
moz_assert = self._events(monitor, "MOZ_ASSERT")[0][2]
ns_assertion = self._events(monitor, "NS_ASSERTION")[0][2]
self.assertEqual([f["function"] for f in moz_assert["stack"]], ["a()"])
self.assertEqual([f["function"] for f in ns_assertion["stack"]], ["b()"])
def test_process_output_cpp_assertion_without_moztest(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor._assertion_parser = None
monitor.start()
self._process_output(monitor, self.NS_ASSERTION_LINE)
monitor.stop()
asserts = self._events(monitor, "C++ assertion")
self.assertEqual(len(asserts), 1)
_, _, data = asserts[0]
self.assertEqual(data["type"], "cppDebug")
self.assertEqual(data["color"], "red")
self.assertEqual(data["line"], 131)
def _assertion_failure(self, message="x", stack=None, **extra):
SystemResourceMonitor.assertion_failure({
"time": int(time.time() * 1000),
"kind": "MOZ_ASSERT",
"message": message,
"file": "/builds/worker/checkouts/gecko/dom/workers/RuntimeService.cpp",
"lineno": 2229,
"fatal": True,
"stack": stack,
**extra,
})
def test_assertion_failure_action(self):
rev = "56b3cc68b5e7557a3e13fca984f0f8aebc60dd22"
monitor = SystemResourceMonitor(
poll_interval=0.25,
)
monitor.start()
self._assertion_failure(
message="false (unreachable)",
pid=6136,
process_type="Child",
thread="DOM Worker",
test="dom/workers/test/test_bar.html",
stack=[
{
"function": "Run()",
"file": "/builds/worker/checkouts/gecko/dom/workers/RuntimeService.cpp",
"line": 2229,
},
{"module": "libxul.so", "module_offset": 0x43A0},
],
)
monitor.stop()
asserts = self._events(monitor, "MOZ_ASSERT")
self.assertEqual(len(asserts), 1)
_, _, data = asserts[0]
self.assertEqual(data["type"], "AssertionFailure")
self.assertEqual(data["kind"], "MOZ_ASSERT")
self.assertEqual(data["color"], "red")
self.assertTrue(data["fatal"])
self.assertEqual(data["message"], "false (unreachable)")
self.assertEqual(data["file"], "dom/workers/RuntimeService.cpp")
self.assertEqual(data["line"], 2229)
self.assertEqual(data["pid"], 6136)
self.assertEqual(data["process"], "Child")
self.assertEqual(data["thread"], "DOM Worker")
self.assertEqual(data["test"], "dom/workers/test/test_bar.html")
self.assertEqual(
data["stack"][0]["file"],
f"hg:hg.mozilla.org/try:dom/workers/RuntimeService.cpp:{rev}",
)
self.assertEqual(
data["stack"][1], {"module": "libxul.so", "module_offset": 0x43A0}
)
markers = monitor.as_profile()["threads"][0]["markers"]["data"]
marker = next(m for m in markers if m.get("type") == "AssertionFailure")
self.assertIn("cause", marker)
def test_assertion_failure_action_supersedes_line_report(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(monitor, self.MOZ_ASSERT_LINE)
self._process_output(monitor, "#01: f() [dom/workers/RuntimeService.cpp:2229]")
# The harness parser emits the action when it sees the terminating
# line, before that line is logged.
self._assertion_failure(
message="false (unreachable)",
stack=[{"function": "g()", "file": "dom/a.cpp", "line": 1}],
)
self._process_output(monitor, "terminating line")
monitor.stop()
asserts = self._events(monitor, "MOZ_ASSERT")
self.assertEqual(len(asserts), 1)
self.assertEqual([f["function"] for f in asserts[0][2]["stack"]], ["g()"])
self.assertEqual(len(self._events(monitor, "output")), 1)
def test_assertion_failure_action_with_other_key_keeps_both(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(monitor, self.MOZ_ASSERT_LINE)
self._process_output(monitor, "#01: f() [dom/workers/RuntimeService.cpp:2229]")
self._assertion_failure(message="something else")
self._process_output(monitor, "terminating line")
monitor.stop()
messages = sorted(e[2]["message"] for e in self._events(monitor, "MOZ_ASSERT"))
self.assertEqual(messages, ["false (unreachable)", "something else"])
def test_line_report_completing_after_action_is_dropped(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._assertion_failure(message="false (unreachable)")
self._process_output(monitor, self.MOZ_ASSERT_LINE)
self._process_output(monitor, "#01: f() [dom/workers/RuntimeService.cpp:2229]")
self._process_output(monitor, "terminating line")
# The same assertion firing again is a new event.
self._process_output(monitor, self.MOZ_ASSERT_LINE)
self._process_output(monitor, "terminating line")
monitor.stop()
asserts = self._events(monitor, "MOZ_ASSERT")
self.assertEqual(len(asserts), 2)
self.assertIsNone(asserts[0][2].get("stack", [{}])[0].get("function"))
self.assertEqual(
asserts[1][2]["stack"],
[{"file": "dom/workers/RuntimeService.cpp", "line": 2229}],
)
def test_process_output_console(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(
monitor,
'console.error: (new Error("Unable to retrieve the translation models.",'
' "resource://gre/actors/TranslationsParent.sys.mjs", 2674))',
)
self._process_output(
monitor, 'console.warn: "No view for invalid view, switching to default"'
)
self._process_output(
monitor, "console.log: Downloads: Closing the downloads panel."
)
monitor.stop()
names = sorted(e[1] for e in monitor.events if len(e) == 3)
self.assertEqual(names, ["console.error", "console.log", "console.warn"])
log_event = next(
e for e in monitor.events if len(e) == 3 and e[1] == "console.log"
)
self.assertEqual(log_event[2]["type"], "console")
self.assertEqual(
log_event[2]["message"], "Downloads: Closing the downloads panel."
)
def test_process_output_docshell_unmatched(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(
monitor,
"[Child 4208: Main Thread]: I/DocShellAndDOMWindowLeak "
"++DOCSHELL 2f804b00 == 2 [pid = 4208] [id = 37]",
)
# No matching --DOCSHELL: simulates a process that crashed or
# never tore down its docshells before profiling stopped.
monitor.stop()
docshell = [m for m in monitor.markers if m[0] == "DocShell"]
self.assertEqual(len(docshell), 1)
_, start, end, data, _ = docshell[0]
self.assertIsNone(end)
self.assertEqual(data["id"], 37)
self.assertEqual(data["pid"], 4208)
self.assertEqual(data["pointer"], "2f804b00")
self.assertNotIn("url", data)
def test_process_output_console_trace_with_stack(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(
monitor, 'console.trace: AboutHomeStartupCache: "Preloaded was updated."'
)
self._process_output(
monitor,
"resource:///modules/AboutHomeStartupCache.sys.mjs 775 onPreloadedNewTabMessage",
)
self._process_output(
monitor,
)
# A second console.trace flushes the first.
self._process_output(
monitor, 'console.trace: AboutHomeStartupCache: "Preloaded was updated."'
)
self._process_output(
monitor, "resource:///modules/AboutHomeStartupCache.sys.mjs 775 onFoo"
)
monitor.stop()
traces = [e for e in monitor.events if len(e) == 3 and e[1] == "console.trace"]
self.assertEqual(len(traces), 2)
_, _, data = traces[0]
self.assertEqual(data["type"], "console")
self.assertEqual(len(data["stack"]), 2)
self.assertEqual(
data["stack"][0]["file"],
"resource:///modules/AboutHomeStartupCache.sys.mjs",
)
self.assertEqual(data["stack"][0]["line"], 775)
self.assertEqual(data["stack"][0]["function"], "onPreloadedNewTabMessage")
self.assertEqual(data["stack"][1]["function"], "middleware/</<")
# The second trace was flushed at stop with one frame collected.
self.assertEqual(len(traces[1][2]["stack"]), 1)
def test_process_output_console_trace_flushed_by_other_line(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(monitor, "console.trace: foo")
self._process_output(monitor, "some/file.mjs 42 doStuff")
# Non-frame line should flush the pending trace and be processed normally.
self._process_output(monitor, "totally random output line")
monitor.stop()
traces = [e for e in monitor.events if len(e) == 3 and e[1] == "console.trace"]
self.assertEqual(len(traces), 1)
self.assertEqual(len(traces[0][2]["stack"]), 1)
outputs = [e for e in monitor.events if len(e) == 3 and e[1] == "output"]
self.assertEqual(len(outputs), 1)
def test_process_output_console_multiline_error(self):
"""console.error from Console.sys.mjs's createMultiLineDumper produces an
empty header followed by indented Message:/Stack: lines and JS frames.
"""
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(monitor, "console.error: ")
self._process_output(
monitor, " Message: Error: Cannot attach ID to a tab in a closed window."
)
self._process_output(monitor, " Stack:")
self._process_output(
)
self._process_output(
)
self._process_output(
)
# An unrelated line flushes the pending marker and is processed normally.
self._process_output(monitor, "totally random output line")
monitor.stop()
errors = [e for e in monitor.events if len(e) == 3 and e[1] == "console.error"]
self.assertEqual(len(errors), 1)
data = errors[0][2]
self.assertEqual(data["type"], "console")
self.assertEqual(
data["message"], "Error: Cannot attach ID to a tab in a closed window."
)
self.assertEqual(len(data["stack"]), 3)
self.assertEqual(data["stack"][0]["function"], "setId")
self.assertEqual(
data["stack"][0]["file"],
"chrome://browser/content/parent/ext-browser.js",
)
self.assertEqual(data["stack"][0]["line"], 387)
self.assertEqual(data["stack"][0]["column"], 13)
self.assertTrue(data["stack"][0]["is_js"])
self.assertEqual(data["stack"][2]["function"], "wrapTab")
outputs = [e for e in monitor.events if len(e) == 3 and e[1] == "output"]
self.assertEqual(len(outputs), 1)
def test_process_output_console_multiline_no_trailing_space(self):
"""The header line may or may not carry a trailing space depending on
how the harness rstrips lines; both shapes start the multi-line body.
"""
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(monitor, "console.error:")
self._process_output(monitor, " Message: Boom")
self._process_output(monitor, " Stack:")
monitor.stop()
errors = [e for e in monitor.events if len(e) == 3 and e[1] == "console.error"]
self.assertEqual(len(errors), 1)
data = errors[0][2]
self.assertEqual(data["message"], "Boom")
self.assertEqual(len(data["stack"]), 1)
def test_process_output_console_multiline_flushed_at_stop(self):
"""A multi-line console body without follow-up still gets flushed."""
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(monitor, "console.error: ")
self._process_output(monitor, " Message: Boom")
# No Stack: follow-up; stop() must still emit the marker.
monitor.stop()
errors = [e for e in monitor.events if len(e) == 3 and e[1] == "console.error"]
self.assertEqual(len(errors), 1)
self.assertEqual(errors[0][2]["message"], "Boom")
self.assertNotIn("stack", errors[0][2])
def test_process_output_console_multiline_no_message(self):
"""When the line after the header isn't 'Message:', we flush an empty
marker and let the next line be processed normally.
"""
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(monitor, "console.error: ")
self._process_output(monitor, "totally random output line")
monitor.stop()
errors = [e for e in monitor.events if len(e) == 3 and e[1] == "console.error"]
self.assertEqual(len(errors), 1)
outputs = [e for e in monitor.events if len(e) == 3 and e[1] == "output"]
self.assertEqual(len(outputs), 1)
def test_process_output_console_multiline_with_prefix(self):
"""Console.sys.mjs's createConsole({prefix}) prepends "<prefix>: " to
the header, so the multi-line body opens with "console.<m>: <prefix>:"
rather than just "console.<m>: ".
"""
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(monitor, "console.error: services.settings:")
self._process_output(
monitor,
' Message: EmptyDatabaseError: "main/nimbus-desktop-experiments"'
" has not been synced yet",
)
self._process_output(monitor, " Stack:")
self._process_output(
monitor,
)
self._process_output(
monitor,
)
monitor.stop()
errors = [e for e in monitor.events if len(e) == 3 and e[1] == "console.error"]
self.assertEqual(len(errors), 1)
data = errors[0][2]
# The prefix is stitched back in front of the Message: text.
self.assertEqual(
data["message"],
'services.settings: EmptyDatabaseError: "main/nimbus-desktop-experiments"'
" has not been synced yet",
)
self.assertEqual(len(data["stack"]), 2)
self.assertEqual(data["stack"][0]["function"], "EmptyDatabaseError")
self.assertEqual(data["stack"][1]["function"], "async*get")
def test_process_output_console_speculative_falls_back_to_single_line(self):
"""A multi-line method whose body happens to end with ":" but isn't a
multi-line header still emits a single-line marker, and the next line
is processed normally.
"""
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(monitor, "console.error: oops, ends in colon:")
self._process_output(monitor, "totally random output line")
monitor.stop()
errors = [e for e in monitor.events if len(e) == 3 and e[1] == "console.error"]
self.assertEqual(len(errors), 1)
self.assertEqual(errors[0][2]["message"], "oops, ends in colon:")
outputs = [e for e in monitor.events if len(e) == 3 and e[1] == "output"]
self.assertEqual(len(outputs), 1)
def test_process_output_console_warn_empty_is_single_line(self):
"""console.warn uses createDumper(), not the multi-line dumper, so an
empty body must NOT trigger the multi-line state machine.
"""
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(monitor, "console.warn: ")
self._process_output(monitor, " Message: not a follow-up")
monitor.stop()
warns = [e for e in monitor.events if len(e) == 3 and e[1] == "console.warn"]
self.assertEqual(len(warns), 1)
# The " Message: ..." line wasn't consumed by a multi-line body and
# should fall back to a generic output marker.
outputs = [e for e in monitor.events if len(e) == 3 and e[1] == "output"]
self.assertEqual(len(outputs), 1)
def test_process_output_unrecognized_falls_back(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(monitor, "totally random output line that we don't parse")
monitor.stop()
outputs = [e for e in monitor.events if len(e) == 3 and e[1] == "output"]
self.assertEqual(len(outputs), 1)
def test_cpp_warning_frame_uses_hg_source_url(self):
rev = "56b3cc68b5e7557a3e13fca984f0f8aebc60dd22"
monitor = SystemResourceMonitor(
poll_interval=0.25,
)
monitor.start()
self._process_output(
monitor,
"[Parent 100, Main Thread] WARNING: oops: file "
"/builds/worker/workspace/obj-build/foo/./../../../../../checkouts/gecko/"
"netwerk/protocol/http/Http2Compression.cpp:42",
)
monitor.stop()
warnings_ = [e for e in monitor.events if len(e) == 3 and e[1] == "C++ warning"]
self.assertEqual(len(warnings_), 1)
_, _, data = warnings_[0]
self.assertEqual(data["file"], "netwerk/protocol/http/Http2Compression.cpp")
self.assertEqual(
data["stack"][0]["file"],
f"hg:hg.mozilla.org/try:netwerk/protocol/http/Http2Compression.cpp:{rev}",
)
def test_cpp_warning_frame_no_source_url(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._process_output(
monitor,
"[Parent 100, Main Thread] WARNING: oops: file "
"/builds/worker/workspace/obj-build/foo/./../../../../../checkouts/gecko/"
"netwerk/protocol/http/Http2Compression.cpp:42",
)
monitor.stop()
warnings_ = [e for e in monitor.events if len(e) == 3 and e[1] == "C++ warning"]
self.assertEqual(len(warnings_), 1)
_, _, data = warnings_[0]
self.assertEqual(data["file"], "netwerk/protocol/http/Http2Compression.cpp")
self.assertEqual(
data["stack"][0]["file"], "netwerk/protocol/http/Http2Compression.cpp"
)
def test_cpp_warning_frame_sysroot_path_not_wrapped(self):
# Even with a sourceURL set, a path that isn't repo-relative (sysroot,
# fetches, rust stdlib) must pass through unchanged so the source view
# doesn't try to fetch it from hg.mozilla.org and 404.
rev = "56b3cc68b5e7557a3e13fca984f0f8aebc60dd22"
monitor = SystemResourceMonitor(
poll_interval=0.25,
)
monitor.start()
sysroot = (
"/builds/worker/fetches/sysroot-x86_64-linux-gnu/usr/lib/gcc/"
"x86_64-linux-gnu/10/../../../../include/c++/10/bits/std_function.h"
)
self._process_output(
monitor,
f"[Parent 100, Main Thread] WARNING: oops: file {sysroot}:42",
)
monitor.stop()
warnings_ = [e for e in monitor.events if len(e) == 3 and e[1] == "C++ warning"]
self.assertEqual(len(warnings_), 1)
_, _, data = warnings_[0]
self.assertEqual(data["file"], sysroot)
self.assertEqual(data["stack"][0]["file"], sysroot)
@unittest.skipIf(psutil is None, "Resource monitor requires psutil.")
class TestCleanFrameFile(unittest.TestCase):
def _monitor(self, source_url=None):
metadata = {"sourceURL": source_url} if source_url else {}
return SystemResourceMonitor(poll_interval=0.25, metadata=metadata)
def test_ci_path_no_hg_prefix(self):
self.assertEqual(
self._monitor()._clean_frame_file(
"/builds/worker/workspace/obj-build/netwerk/protocol/http/./../../../"
"../../checkouts/gecko/netwerk/protocol/http/Http2Compression.cpp"
),
(
"netwerk/protocol/http/Http2Compression.cpp",
"netwerk/protocol/http/Http2Compression.cpp",
),
)
def test_ci_path_with_hg_prefix(self):
rev = "56b3cc68b5e7557a3e13fca984f0f8aebc60dd22"
self.assertEqual(
"/builds/worker/workspace/obj-build/netwerk/protocol/http/./../../../"
"../../checkouts/gecko/netwerk/protocol/http/Http2Compression.cpp"
),
(
"netwerk/protocol/http/Http2Compression.cpp",
f"hg:hg.mozilla.org/try:netwerk/protocol/http/Http2Compression.cpp:{rev}",
),
)
def test_path_without_marker_unchanged(self):
self.assertEqual(
self._monitor()._clean_frame_file("/home/user/firefox/foo/bar.cpp"),
("/home/user/firefox/foo/bar.cpp", "/home/user/firefox/foo/bar.cpp"),
)
def test_sysroot_path_not_wrapped_even_with_hg_prefix(self):
# Sysroot/fetches paths must not be wrapped as "hg:..." URLs; the
# source view would 404 trying to fetch them from hg.mozilla.org.
rev = "deadbeef"
sysroot = (
"/builds/worker/fetches/sysroot-x86_64-linux-gnu/usr/lib/gcc/"
"x86_64-linux-gnu/10/../../../../include/c++/10/bits/std_function.h"
)
self.assertEqual(monitor._clean_frame_file(sysroot), (sysroot, sysroot))
def test_last_occurrence_wins(self):
self.assertEqual(
self._monitor()._clean_frame_file(
"/x/checkouts/gecko/y/../../checkouts/gecko/foo/bar.cpp"
),
("foo/bar.cpp", "foo/bar.cpp"),
)
def test_falsy_passthrough(self):
monitor = self._monitor()
self.assertEqual(monitor._clean_frame_file(""), ("", ""))
self.assertEqual(monitor._clean_frame_file(None), (None, None))
class TestParseHgSourceUrl(unittest.TestCase):
def test_try(self):
self.assertEqual(
_parse_hg_source_url(
),
(
"hg:hg.mozilla.org/try:",
"56b3cc68b5e7557a3e13fca984f0f8aebc60dd22",
),
)
def test_mozilla_central(self):
self.assertEqual(
_parse_hg_source_url(
),
("hg:hg.mozilla.org/mozilla-central:", "abcdef0123456789"),
)
def test_multi_segment_repo(self):
self.assertEqual(
_parse_hg_source_url(
),
("hg:hg.mozilla.org/integration/autoland:", "abcdef0123456789"),
)
self.assertEqual(
_parse_hg_source_url(
),
("hg:hg.mozilla.org/releases/mozilla-beta:", "abcdef0123456789"),
)
def test_non_hg_returns_none(self):
self.assertEqual(
(None, None),
)
def test_empty_returns_none(self):
self.assertEqual(_parse_hg_source_url(None), (None, None))
self.assertEqual(_parse_hg_source_url(""), (None, None))
@unittest.skipIf(psutil is None, "Resource monitor requires psutil.")
class TestLsanLeakFrameRewrite(unittest.TestCase):
def _record_lsan_leak(self, monitor, stack):
SystemResourceMonitor.lsan_leak({
"kind": "Direct",
"bytes": 16,
"objects": 1,
"stack": stack,
"time": int((time.time() - monitor.start_timestamp) * 1000),
})
def test_lsan_leak_rewrites_frame_files_with_source_url(self):
rev = "56b3cc68b5e7557a3e13fca984f0f8aebc60dd22"
monitor = SystemResourceMonitor(
poll_interval=0.25,
)
monitor.start()
original_stack = [
{
"function": "Foo",
"file": "/builds/worker/workspace/obj-build/./../../../../../"
"checkouts/gecko/xpcom/ds/nsAtomTable.cpp",
"line": 90,
},
{"function": "Bar", "module": "libxul.so"},
]
self._record_lsan_leak(monitor, original_stack)
monitor.stop()
leaks = [e for e in monitor.events if len(e) == 3 and e[1] == "LSan Leak"]
self.assertEqual(len(leaks), 1)
_, _, data = leaks[0]
self.assertEqual(
data["stack"][0]["file"],
f"hg:hg.mozilla.org/try:xpcom/ds/nsAtomTable.cpp:{rev}",
)
self.assertNotIn("file", data["stack"][1])
# Original frame dicts must not be mutated.
self.assertEqual(
original_stack[0]["file"],
"/builds/worker/workspace/obj-build/./../../../../../"
"checkouts/gecko/xpcom/ds/nsAtomTable.cpp",
)
def test_lsan_leak_empty_stack(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
self._record_lsan_leak(monitor, [])
monitor.stop()
leaks = [e for e in monitor.events if len(e) == 3 and e[1] == "LSan Leak"]
self.assertEqual(len(leaks), 1)
_, _, data = leaks[0]
self.assertNotIn("stack", data)
@unittest.skipIf(psutil is None, "Resource monitor requires psutil.")
class TestForwarding(unittest.TestCase):
def setUp(self):
fd, self.forward_path = tempfile.mkstemp()
os.close(fd)
self.addCleanup(os.unlink, self.forward_path)
def _child(self):
with mock.patch.dict(os.environ, {FORWARD_ENV: self.forward_path}):
return SystemResourceMonitor(poll_interval=0.1)
def test_forwarding(self):
parent = SystemResourceMonitor(poll_interval=0.1)
self.assertFalse(parent.forwarding)
parent.start()
parent.accept_forwarded(self.forward_path)
child = self._child()
self.assertTrue(child.forwarding)
child.start()
self.assertEqual(child.measurements, [])
child.record_event("event", data={"type": "Text", "text": "data"})
start = time.monotonic()
child.record_marker("marker", start, start + 0.1, "text")
with child.phase("phase"):
time.sleep(0.3)
child.forward_perfherder("suite", ["option"], "/upload")
child.stop()
parent.stop()
self.assertEqual(len(parent.events), 1)
self.assertEqual(
parent.events[0][1:], ("event", {"type": "Text", "text": "data"})
)
self.assertEqual(
parent.markers[0],
("marker", start, start + 0.1, "text", SystemResourceMonitor.TASK_CATEGORY),
)
self.assertEqual(list(parent.phases), ["phase"])
self.assertEqual(len(parent.perfherder_requests), 1)
name, options, req_start, req_end, upload_dir, source = (
parent.perfherder_requests[0]
)
self.assertEqual((name, options, upload_dir), ("suite", ["option"], "/upload"))
self.assertEqual(req_start, child.start_time)
self.assertEqual(source, os.getpid())
data = parent.perfherder_data(name, options, req_start, req_end, source)
self.assertEqual(data["framework"], {"name": "job_resource_usage"})
self.assertEqual(
[suite["name"] for suite in data["suites"]],
["suite.overall", "suite.phase"],
)
self.assertEqual(data["suites"][0]["extraOptions"], ["option"])
def test_forwarded_phase_name_conflict(self):
monitor = SystemResourceMonitor(poll_interval=0.1)
monitor.start()
with monitor.phase("phase"):
pass
monitor._add_phase("phase", 1.0, 2.0, 1)
monitor._add_phase("phase", 3.0, 4.0, 2)
monitor.stop()
self.assertEqual(monitor.phases["phase (2)"], (1.0, 2.0))
self.assertEqual(monitor.phases["phase (3)"], (3.0, 4.0))
data = monitor.perfherder_data("suite", [], 1.0, 2.0, 1)
self.assertEqual(
[suite["name"] for suite in data["suites"]],
["suite.overall", "suite.phase"],
)
def test_forwarded_phase_before_local_phase(self):
monitor = SystemResourceMonitor(poll_interval=0.1)
monitor.start()
monitor._add_phase("phase", 1.0, 2.0, 1)
with monitor.phase("phase"):
pass
monitor.stop()
self.assertEqual(monitor.phases["phase"], (1.0, 2.0))
self.assertIn("phase (2)", monitor.phases)
data = monitor.perfherder_data("suite", [])
self.assertEqual(
[suite["name"] for suite in data["suites"]],
["suite.overall", "suite.phase"],
)
self.assertNotEqual(data["suites"][1]["subtests"][0]["value"], 1.0)
def test_perfherder_data_source_phases(self):
monitor = SystemResourceMonitor(poll_interval=0.1)
monitor.start()
start = time.monotonic()
monitor._add_phase("outer", start, start + 0.3, 1)
monitor._add_phase("inner", start + 0.1, start + 0.2, 2)
monitor.stop()
for source, phase in ((1, "outer"), (2, "inner")):
data = monitor.perfherder_data("suite", [], start, start + 0.3, source)
self.assertEqual(
[suite["name"] for suite in data["suites"]],
["suite.overall", f"suite.{phase}"],
)
data = monitor.perfherder_data("suite", [])
self.assertEqual([suite["name"] for suite in data["suites"]], ["suite.overall"])
def test_concurrent_forwarding(self):
parent = SystemResourceMonitor(poll_interval=0.1)
parent.start()
parent.accept_forwarded(self.forward_path)
_run_forwarding_children(self.forward_path)
parent.stop()
self.assertEqual(len([m for m in parent.markers if m[0] == "marker"]), 2000)
def test_accept_forwarded_not_running(self):
monitor = SystemResourceMonitor(poll_interval=0.1)
monitor.accept_forwarded(self.forward_path)
self.assertIsNone(monitor._from_children_path)
def test_forward_after_stop(self):
child = self._child()
child.start()
child.stop()
with self.assertWarns(UserWarning):
child.forward_perfherder("suite", [])
def test_children_forward_path(self):
parent = SystemResourceMonitor(poll_interval=0.1)
self.assertIsNone(parent.children_forward_path)
parent.start()
parent.accept_forwarded(self.forward_path)
self.assertEqual(parent.children_forward_path, self.forward_path)
parent.stop()
self.assertEqual(self._child().children_forward_path, self.forward_path)
def test_forward_to(self):
parent = SystemResourceMonitor(poll_interval=0.1)
parent.start()
parent.accept_forwarded(self.forward_path)
child = SystemResourceMonitor(forward_to=self.forward_path)
self.assertTrue(child.forwarding)
child.start()
start = time.monotonic()
child.record_marker("marker", start, start + 0.1, "text")
child.stop()
parent.stop()
self.assertEqual(
parent.markers[0],
("marker", start, start + 0.1, "text", SystemResourceMonitor.TASK_CATEGORY),
)
@unittest.skipUnless(hasattr(os, "fork"), "Requires fork.")
def test_forked_child_forgets_monitor(self):
monitor = SystemResourceMonitor(poll_interval=0.1)
monitor.start()
_wait_for_measurement(monitor)
pid = os.fork()
if pid == 0:
forgotten = SystemResourceMonitor.instance is None and monitor._pipe.closed
monitor._atexit_stop()
# Hold the lock as the parent's drain timer could have when forking.
monitor._pipe_lock.acquire()
monitor.stop()
os._exit(0 if forgotten else 1)
_, status = os.waitpid(pid, 0)
self.assertEqual(os.waitstatus_to_exitcode(status), 0)
child_exit = time.monotonic()
time.sleep(0.5)
self.assertIs(SystemResourceMonitor.instance, monitor)
monitor.stop()
self.assertGreater(monitor.measurements[-1].end, child_exit)
def _artifact_filenames(self, monitor):
return sorted(
data["filename"] for _, name, data in monitor.events if name == "artifact"
)
def test_forwarded_artifacts_in_parent_upload_dir(self):
with tempfile.TemporaryDirectory() as upload_dir:
open(os.path.join(upload_dir, "file"), "w").close()
parent = SystemResourceMonitor(poll_interval=0.1)
parent.start()
parent.accept_forwarded(self.forward_path)
child = self._child()
child.start()
child.stop(upload_dir=upload_dir)
parent.stop(upload_dir=upload_dir)
self.assertEqual(self._artifact_filenames(parent), ["file"])
def test_forwarded_artifacts_in_other_upload_dir(self):
with tempfile.TemporaryDirectory() as upload_dir:
with tempfile.TemporaryDirectory() as child_upload_dir:
open(os.path.join(upload_dir, "parent"), "w").close()
open(os.path.join(child_upload_dir, "child"), "w").close()
parent = SystemResourceMonitor(poll_interval=0.1)
parent.start()
parent.accept_forwarded(self.forward_path)
child = self._child()
child.start()
child.stop(upload_dir=child_upload_dir)
parent.stop(upload_dir=upload_dir)
self.assertEqual(self._artifact_filenames(parent), ["child", "parent"])
def test_forwarded_artifacts_moved_out_of_parent_upload_dir(self):
with tempfile.TemporaryDirectory() as upload_dir:
path = os.path.join(upload_dir, "file")
open(path, "w").close()
parent = SystemResourceMonitor(poll_interval=0.1)
parent.start()
parent.accept_forwarded(self.forward_path)
child = self._child()
child.start()
child.stop(upload_dir=upload_dir)
os.unlink(path)
parent.stop(upload_dir=upload_dir)
self.assertEqual(self._artifact_filenames(parent), ["file"])
def test_forwarded_sccache_log_parsed_once(self):
with tempfile.TemporaryDirectory() as upload_dir:
open(os.path.join(upload_dir, "sccache.log"), "w").close()
parent = SystemResourceMonitor(poll_interval=0.1)
parent.start()
parent.accept_forwarded(self.forward_path)
with mock.patch.object(
SystemResourceMonitor, "_parse_sccache_log"
) as parse_sccache_log:
child = self._child()
child.start()
child.stop(upload_dir=upload_dir)
parent.stop(upload_dir=upload_dir)
self.assertEqual(parse_sccache_log.call_count, 1)
self.assertEqual(self._artifact_filenames(parent), ["sccache.log"])
def test_partial_line(self):
parent = SystemResourceMonitor(poll_interval=0.1)
parent.start()
parent.accept_forwarded(self.forward_path)
with open(self.forward_path, "a") as fh:
fh.write('["event", "complete", 1.0, null]\n["event", "incompl')
parent._read_forwarded()
self.assertEqual(parent.events, [(1.0, "complete")])
with open(self.forward_path, "a") as fh:
fh.write('ete", 2.0, null]\n')
parent.stop()
self.assertEqual(parent.events, [(1.0, "complete"), (2.0, "incomplete")])
def test_perfherder_data_all_phases(self):
monitor = SystemResourceMonitor(poll_interval=0.1)
monitor.start()
with monitor.phase("phase1"):
time.sleep(0.3)
monitor.stop()
data = monitor.perfherder_data("suite", [])
self.assertEqual(
[suite["name"] for suite in data["suites"]],
["suite.overall", "suite.phase1"],
)
overall = {s["name"] for s in data["suites"][0]["subtests"]}
self.assertLessEqual(
{"io_write_bytes", "io.read_bytes", "io_write_time", "io_read_time"},
overall,
)
def _run_forwarding_children(forward_path):
script = (
"import time\n"
"from mozsystemmonitor.resourcemonitor import SystemResourceMonitor\n"
"m = SystemResourceMonitor()\n"
"m.start()\n"
"for i in range(500):\n"
" t = time.monotonic()\n"
" m.record_marker('marker', t, t, 'x' * 200)\n"
"m.stop()\n"
)
env = dict(os.environ, **{FORWARD_ENV: forward_path})
env["PYTHONPATH"] = os.pathsep.join(sys.path)
children = [
subprocess.Popen([sys.executable, "-c", script], env=env) for _ in range(4)
]
for child in children:
assert child.wait() == 0
def test_concurrent_forwarding_lines(tmp_path):
# Forwarding children don't need psutil, so this also runs where the
# parent can't monitor.
forward_path = tmp_path / "forward.jsonl"
forward_path.touch()
_run_forwarding_children(str(forward_path))
data = forward_path.read_bytes()
assert data.endswith(b"\n")
records = [json.loads(line) for line in data.splitlines()]
assert len(records) == 2000
assert all(record[0] == "marker" for record in records)
@pytest.mark.parametrize(
"value,expected",
[
(None, None),
("", None),
("0", None),
("2.5", 2.5),
("yes", 0.1),
],
)
def test_process_sampling_interval(value, expected):
env = {} if value is None else {"MOZ_PROCESS_SAMPLING": value}
with mock.patch.dict(os.environ, env):
if value is None:
os.environ.pop("MOZ_PROCESS_SAMPLING", None)
assert _process_sampling_interval(0.1) == expected
@pytest.mark.skipif(psutil is None, reason="Resource monitor requires psutil.")
def test_process_sampling_with_interval():
with mock.patch.dict(os.environ, {"MOZ_PROCESS_SAMPLING": "0.5"}):
monitor = SystemResourceMonitor(poll_interval=0.1)
monitor.start()
_wait_for_measurement(monitor)
subprocess.run([sys.executable, "-c", "import time; time.sleep(2)"], check=True)
time.sleep(0.6)
monitor.stop()
cmds = [cmd for pid, start, end, cmd, ppid in monitor.processes]
assert any("time.sleep(2)" in cmd for cmd in cmds), cmds
@pytest.mark.skipif(psutil is None, reason="Resource monitor requires psutil.")
def test_process_sampling_still_running():
sleep = [sys.executable, "-c", "import time; time.sleep(60)"]
before = subprocess.Popen(sleep + ["before"])
try:
with mock.patch.dict(os.environ, {"MOZ_PROCESS_SAMPLING": "0.1"}):
monitor = SystemResourceMonitor(poll_interval=0.1)
monitor.start()
# Let the monitor take its first sample, which is what it
# considers to be running before it started.
_wait_for_measurement(monitor)
during = subprocess.Popen(sleep + ["during"])
try:
time.sleep(1)
monitor.stop()
finally:
during.kill()
during.wait()
finally:
before.kill()
before.wait()
cmds = [cmd for pid, start, end, cmd, ppid in monitor.processes]
assert any(cmd.endswith(" during") for cmd in cmds), cmds
assert not any(cmd.endswith(" before") for cmd in cmds), cmds
@pytest.mark.skipif(psutil is None, reason="Resource monitor requires psutil.")
@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="Linux only")
def test_process_sampling_start_time():
with mock.patch.dict(os.environ, {"MOZ_PROCESS_SAMPLING": "0.1"}):
monitor = SystemResourceMonitor(poll_interval=0.1)
monitor.start()
_wait_for_measurement(monitor)
before = time.monotonic()
child = subprocess.Popen([
sys.executable,
"-c",
"import time; time.sleep(60)",
"start-time",
])
after = time.monotonic()
try:
time.sleep(1)
monitor.stop()
finally:
child.kill()
child.wait()
starts = [
start for pid, start, end, cmd, ppid in monitor.processes if pid == child.pid
]
assert len(starts) == 1, monitor.processes
# /proc/<pid>/stat has a resolution of one clock tick, usually 10ms, and
# converting between clocks adds a little imprecision.
assert before - 0.02 <= starts[0] <= after + 0.01, (before, starts[0], after)
if __name__ == "__main__":
mozunit.main()