mirror of
https://github.com/Quad4-Software/MeshChatX.git
synced 2026-08-18 09:49:09 -04:00
feat: update backend benchmarks with new conversation handling and missed call metrics
This commit is contained in:
Ivan
parent
9c913bdb3c
commit
d7b66171bf
5 changed files with 242 additions and 8 deletions
|
|
@ -28,6 +28,8 @@ All notable changes to this project will be documented in this file.
|
|||
- Relay Chat: denser hub UI, announce interval, collapsed system lines, reconnect notices
|
||||
- Low-memory cleanup and SQLite pragmas under memory pressure
|
||||
- CI benches use median-of-medians and quieter regression gates
|
||||
- Backend benchmarks cover slim conversation list, mark-as-read, call history, and missed-call notification paths
|
||||
- Benchmark gate fails when a full-suite run drops required benches or loses coverage vs baseline
|
||||
- Backend tests can run sharded in CI
|
||||
- Plugin strings live in plugin bundles, not main locale files
|
||||
- Docker frontend build installs Go, builds visualiser WASM, and fails if WASM artifacts are missing
|
||||
|
|
|
|||
BIN
meshchatx.rsm
BIN
meshchatx.rsm
Binary file not shown.
|
|
@ -29,6 +29,32 @@ from tests.backend.benchmarking_utils import ( # noqa: E402
|
|||
should_alert_regression,
|
||||
)
|
||||
|
||||
# Must stay present in comprehensive suite runs. Gate fails if a full-suite
|
||||
# baseline is missing any of these from the current JSON.
|
||||
REQUIRED_BENCHES = frozenset(
|
||||
{
|
||||
"Database Initialization",
|
||||
"Message Upsert (Batch of 100)",
|
||||
"Get 100 Conversations List",
|
||||
"Get Conversations Slim List (handler)",
|
||||
"Get Conversations Unread Filter (handler)",
|
||||
"Mark Conversation As Read",
|
||||
"Get Messages for Conversation (offset 500)",
|
||||
"Log Telephone Call",
|
||||
"Get Call History List",
|
||||
"Notification Add + Unread Count",
|
||||
"Missed Call Unread Count by Type",
|
||||
"Dismiss Missed Call Notifications",
|
||||
"Config Get (50 keys)",
|
||||
"Get Contacts List",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _is_full_suite_baseline(previous):
|
||||
"""True when previous looks like the comprehensive suite, not a unit fixture."""
|
||||
return "Database Initialization" in previous
|
||||
|
||||
|
||||
def _load_entries(path):
|
||||
with open(path, encoding="utf-8") as f:
|
||||
|
|
@ -91,6 +117,8 @@ def compare(
|
|||
alerts = []
|
||||
improvements = []
|
||||
skipped = []
|
||||
coverage_alerts = []
|
||||
enforce_coverage = _is_full_suite_baseline(previous)
|
||||
|
||||
for name in sorted(current):
|
||||
cur = current[name]
|
||||
|
|
@ -153,23 +181,55 @@ def compare(
|
|||
)
|
||||
|
||||
missing = sorted(set(previous) - set(current))
|
||||
coverage_seen = set()
|
||||
for name in missing:
|
||||
detail = "present in baseline only"
|
||||
status = "removed"
|
||||
if enforce_coverage:
|
||||
coverage_alerts.append(name)
|
||||
coverage_seen.add(name)
|
||||
if name in REQUIRED_BENCHES:
|
||||
status = "MISSING"
|
||||
detail = "required bench absent from current suite"
|
||||
else:
|
||||
status = "REMOVED"
|
||||
detail = "dropped from suite vs full baseline"
|
||||
rows.append(
|
||||
{
|
||||
"name": name,
|
||||
"current": None,
|
||||
"previous": _value_ms(previous[name]),
|
||||
"ratio": None,
|
||||
"status": "removed",
|
||||
"detail": "present in baseline only",
|
||||
"status": status,
|
||||
"detail": detail,
|
||||
},
|
||||
)
|
||||
|
||||
check_required = enforce_coverage or (
|
||||
not previous and "Database Initialization" in current
|
||||
)
|
||||
if check_required:
|
||||
for name in sorted(REQUIRED_BENCHES - set(current) - coverage_seen):
|
||||
coverage_alerts.append(name)
|
||||
rows.append(
|
||||
{
|
||||
"name": name,
|
||||
"current": None,
|
||||
"previous": (
|
||||
_value_ms(previous[name]) if name in previous else None
|
||||
),
|
||||
"ratio": None,
|
||||
"status": "MISSING",
|
||||
"detail": "required bench absent from current suite",
|
||||
},
|
||||
)
|
||||
|
||||
lines = [
|
||||
"MeshChatX Backend Benchmark Gate",
|
||||
f"Current: {current_path}",
|
||||
f"Previous: {previous_path or '(none)'}",
|
||||
f"Noise floor: {noise_floor_ms} ms | Min abs delta: {min_abs_delta_ms} ms",
|
||||
f"Coverage enforce: {'yes' if check_required else 'no'}",
|
||||
"",
|
||||
f"{'Benchmark':42} {'Curr':>10} {'Prev':>10} {'Ratio':>8} Status",
|
||||
"-" * 90,
|
||||
|
|
@ -182,17 +242,23 @@ def compare(
|
|||
f"{row['name'][:42]:42} {cur_s:>10} {prev_s:>10} {ratio_s:>8} "
|
||||
f"{row['status']}",
|
||||
)
|
||||
if row["status"] == "REGRESSION":
|
||||
if row["status"] in {"REGRESSION", "REMOVED", "MISSING"}:
|
||||
lines.append(f" -> {row['detail']}")
|
||||
|
||||
lines.append("-" * 90)
|
||||
lines.append(
|
||||
f"Regressions: {len(alerts)} | Improvements: {len(improvements)} | "
|
||||
f"Noise-skipped: {len(skipped)} | New: "
|
||||
f"{sum(1 for r in rows if r['status'] == 'new')}",
|
||||
f"{sum(1 for r in rows if r['status'] == 'new')} | "
|
||||
f"Coverage: {len(coverage_alerts)}",
|
||||
)
|
||||
if alerts:
|
||||
lines.append("ALERT: " + ", ".join(alerts))
|
||||
if alerts or coverage_alerts:
|
||||
parts = []
|
||||
if alerts:
|
||||
parts.append("regressions: " + ", ".join(alerts))
|
||||
if coverage_alerts:
|
||||
parts.append("coverage: " + ", ".join(coverage_alerts))
|
||||
lines.append("ALERT: " + " | ".join(parts))
|
||||
else:
|
||||
lines.append("No actionable regressions.")
|
||||
|
||||
|
|
@ -210,7 +276,7 @@ def compare(
|
|||
f.write(text)
|
||||
f.write("```\n")
|
||||
|
||||
return 1 if alerts else 0, rows
|
||||
return 1 if (alerts or coverage_alerts) else 0, rows
|
||||
|
||||
|
||||
def update_baseline(
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from meshchatx.src.backend.database.map_drawings import MapDrawingsDAO # noqa:
|
|||
from meshchatx.src.backend.database.telephone import TelephoneDAO # noqa: E402
|
||||
from meshchatx.src.backend.database.voicemails import VoicemailDAO # noqa: E402
|
||||
from meshchatx.src.backend.identity_manager import IdentityManager # noqa: E402
|
||||
from meshchatx.src.backend.message_handler import MessageHandler # noqa: E402
|
||||
from tests.backend.benchmarking_utils import ( # noqa: E402
|
||||
BenchmarkResult,
|
||||
aggregate_run_results,
|
||||
|
|
@ -259,6 +260,24 @@ class BackendBenchmarker:
|
|||
def get_convs():
|
||||
return self.db.messages.get_conversations()
|
||||
|
||||
@benchmark("Get Conversations Slim List (handler)", iterations=10)
|
||||
def get_convs_handler():
|
||||
handler = MessageHandler(self.db)
|
||||
return handler.get_conversations(self.my_hash, limit=100)
|
||||
|
||||
@benchmark("Get Conversations Unread Filter (handler)", iterations=10)
|
||||
def get_convs_unread():
|
||||
handler = MessageHandler(self.db)
|
||||
return handler.get_conversations(
|
||||
self.my_hash,
|
||||
filter_unread=True,
|
||||
limit=100,
|
||||
)
|
||||
|
||||
@benchmark("Mark Conversation As Read", iterations=20)
|
||||
def mark_read():
|
||||
self.db.messages.mark_conversation_as_read(random.choice(peer_hashes))
|
||||
|
||||
@benchmark("Get Messages for Conversation (offset 500)", iterations=20)
|
||||
def get_messages():
|
||||
return self.db.messages.get_conversation_messages(
|
||||
|
|
@ -276,7 +295,12 @@ class BackendBenchmarker:
|
|||
|
||||
_, res = get_convs()
|
||||
self.results.append(res)
|
||||
|
||||
_, res = get_convs_handler()
|
||||
self.results.append(res)
|
||||
_, res = get_convs_unread()
|
||||
self.results.append(res)
|
||||
_, res = mark_read()
|
||||
self.results.append(res)
|
||||
_, res = get_messages()
|
||||
self.results.append(res)
|
||||
|
||||
|
|
@ -385,8 +409,24 @@ class BackendBenchmarker:
|
|||
timestamp=time.time(),
|
||||
)
|
||||
|
||||
@benchmark("Get Call History List", iterations=20)
|
||||
def get_history():
|
||||
return dao.get_call_history(limit=50)
|
||||
|
||||
for _ in range(40):
|
||||
dao.add_call_history(
|
||||
remote_identity_hash=secrets.token_hex(16),
|
||||
remote_identity_name="Seed Peer",
|
||||
is_incoming=True,
|
||||
status="Busy",
|
||||
duration_seconds=0,
|
||||
timestamp=time.time(),
|
||||
)
|
||||
|
||||
_, res = log_call()
|
||||
self.results.append(res)
|
||||
_, res = get_history()
|
||||
self.results.append(res)
|
||||
|
||||
def bench_contact_operations(self):
|
||||
dao = ContactsDAO(self.db.provider)
|
||||
|
|
@ -662,6 +702,27 @@ class BackendBenchmarker:
|
|||
)
|
||||
return self.db.misc.get_unread_notification_count()
|
||||
|
||||
@benchmark("Missed Call Unread Count by Type", iterations=20)
|
||||
def missed_call_count():
|
||||
return self.db.misc.get_unread_notification_count_by_type(
|
||||
"telephone_missed_call",
|
||||
)
|
||||
|
||||
@benchmark("Dismiss Missed Call Notifications", iterations=10)
|
||||
def dismiss_missed_calls():
|
||||
with self.db.provider:
|
||||
for _ in range(5):
|
||||
self.db.misc.add_notification(
|
||||
notification_type="telephone_missed_call",
|
||||
remote_hash=random.choice(dest_hashes),
|
||||
title="Missed Call",
|
||||
content="bench missed call",
|
||||
)
|
||||
self.db.misc.dismiss_unviewed_notifications("telephone_missed_call")
|
||||
return self.db.misc.get_unread_notification_count_by_type(
|
||||
"telephone_missed_call",
|
||||
)
|
||||
|
||||
for _ in range(50):
|
||||
with self.db.provider:
|
||||
self.db.misc.add_blocked_destination(random.choice(dest_hashes))
|
||||
|
|
@ -672,6 +733,13 @@ class BackendBenchmarker:
|
|||
"#fff",
|
||||
"#000",
|
||||
)
|
||||
with self.db.provider:
|
||||
self.db.misc.add_notification(
|
||||
notification_type="telephone_missed_call",
|
||||
remote_hash=random.choice(dest_hashes),
|
||||
title="Missed Call",
|
||||
content="seed missed call",
|
||||
)
|
||||
|
||||
_, res = blocked_dest_roundtrip()
|
||||
self.results.append(res)
|
||||
|
|
@ -683,6 +751,10 @@ class BackendBenchmarker:
|
|||
self.results.append(res)
|
||||
_, res = notification_roundtrip()
|
||||
self.results.append(res)
|
||||
_, res = missed_call_count()
|
||||
self.results.append(res)
|
||||
_, res = dismiss_missed_calls()
|
||||
self.results.append(res)
|
||||
|
||||
def print_summary(self, json_output_path=None):
|
||||
suite_runs = getattr(self, "_suite_runs", 1)
|
||||
|
|
|
|||
|
|
@ -233,6 +233,100 @@ class TestCompareBenchmarks(unittest.TestCase):
|
|||
)
|
||||
self.assertEqual(code, 1)
|
||||
|
||||
def test_full_suite_fails_when_required_bench_missing(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
current = os.path.join(tmp, "current.json")
|
||||
previous = os.path.join(tmp, "previous.json")
|
||||
# Full-suite signal without the new required conversation benches.
|
||||
self._write(
|
||||
previous,
|
||||
[
|
||||
{
|
||||
"name": "Database Initialization",
|
||||
"unit": "ms",
|
||||
"value": 20.0,
|
||||
},
|
||||
{
|
||||
"name": "Get 100 Conversations List",
|
||||
"unit": "ms",
|
||||
"value": 5.0,
|
||||
},
|
||||
],
|
||||
)
|
||||
self._write(
|
||||
current,
|
||||
[
|
||||
{
|
||||
"name": "Database Initialization",
|
||||
"unit": "ms",
|
||||
"value": 21.0,
|
||||
},
|
||||
{
|
||||
"name": "Get 100 Conversations List",
|
||||
"unit": "ms",
|
||||
"value": 5.1,
|
||||
},
|
||||
],
|
||||
)
|
||||
code, rows = compare(current, previous)
|
||||
by_name = {r["name"]: r for r in rows}
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(
|
||||
by_name["Get Conversations Slim List (handler)"]["status"],
|
||||
"MISSING",
|
||||
)
|
||||
|
||||
def test_full_suite_fails_when_bench_removed(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
current = os.path.join(tmp, "current.json")
|
||||
previous = os.path.join(tmp, "previous.json")
|
||||
self._write(
|
||||
previous,
|
||||
[
|
||||
{
|
||||
"name": "Database Initialization",
|
||||
"unit": "ms",
|
||||
"value": 20.0,
|
||||
},
|
||||
{
|
||||
"name": "Extra Optional Bench",
|
||||
"unit": "ms",
|
||||
"value": 1.0,
|
||||
},
|
||||
],
|
||||
)
|
||||
# Current only has init: required benches missing + optional removed.
|
||||
self._write(
|
||||
current,
|
||||
[
|
||||
{
|
||||
"name": "Database Initialization",
|
||||
"unit": "ms",
|
||||
"value": 20.0,
|
||||
},
|
||||
],
|
||||
)
|
||||
code, rows = compare(current, previous)
|
||||
by_name = {r["name"]: r for r in rows}
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(by_name["Extra Optional Bench"]["status"], "REMOVED")
|
||||
|
||||
def test_toy_fixture_does_not_enforce_required_coverage(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
current = os.path.join(tmp, "current.json")
|
||||
previous = os.path.join(tmp, "previous.json")
|
||||
self._write(
|
||||
current,
|
||||
[{"name": "Toy Bench", "unit": "ms", "value": 1.0}],
|
||||
)
|
||||
self._write(
|
||||
previous,
|
||||
[{"name": "Toy Bench", "unit": "ms", "value": 1.0}],
|
||||
)
|
||||
code, rows = compare(current, previous)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(rows[0]["status"], "ok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue