From cfa8cddd3b85bdf0b713aa943ca75cd0d97cc204 Mon Sep 17 00:00:00 2001
From: Ivan <e318cbc04468bd574db2b4523dddd710>
Date: Thu, 13 Aug 2026 00:15:40 -0500
Subject: [PATCH] feat: improve manifest generation by implementing threading
 for improved performance and preventing deadlocks

---
 meshchatx.rsm                        | Bin 188784 -> 188784 bytes
 scripts/ci/tree_manifest_generate.py |  54 ++++++++++++++++++---------
 2 files changed, 37 insertions(+), 17 deletions(-)

diff --git a/meshchatx.rsm b/meshchatx.rsm
index 187c2c7860f5f490ed37b885693376beb6fff89f..1db69a9a1e6e22d60d61f04afe4b31322575ec0b 100644
GIT binary patch
delta 189
zcmV;u07C!p!VB=i3qS>^&AC<h@O06>At7CvA?`(Fh&usXj-!<?hi**_&~P>t!Srsi
z4kPq*)lxpC)oihld{tJVxv8;npQA$JfC-U67$75V<A@%<54-^4IX4FL&t~3&h|&!D
zrsPdtwE>jTq$q)SwRi#OuRvm9W??lqVKFo?Wn^MwH(@noFf%k_F*q?~Gcqz_V>CE2
rWie!AIW;w8VPZEiVPiF7H)LfnVK!koWHK~3VlaoGumQKAumaF9J)cjb

delta 189
zcmV;u07C!p!VB=i3qWF}C9;`->sqdY-!$aH|0|!SN_~E3@jp)+80RV)(CaSyp5&6g
zo%zplHCU;}+CBEk+GVm;X5IAp@h5fbv<#6z7$C83KECRa1}8ltNBc&z9c!858Wqh!
zO<nv1C*c?Xkt>0CwRi#OuRvxnWic=?Ght(4GGSz4Gc#s5WiT^iWiVtoW@2V!IW{;n
rI5aq8G-ER`VKrniIb>oqFlAw8G&y2qF*IU1Wj2SPumQKAumaF9Zj(}i

diff --git a/scripts/ci/tree_manifest_generate.py b/scripts/ci/tree_manifest_generate.py
index 1d9b7ea2..3dadd3bf 100644
--- a/scripts/ci/tree_manifest_generate.py
+++ b/scripts/ci/tree_manifest_generate.py
@@ -13,6 +13,7 @@ import hashlib
 import os
 import subprocess
 import sys
+import threading
 from pathlib import Path
 
 MANIFEST_HEADER = "# meshchatx tree manifest v1"
@@ -72,6 +73,15 @@ def _read_batch_blob(stdout, header: bytes) -> bytes | None:
     return data
 
 
+def _feed_cat_file_stdin(stdin, oids: list[str]) -> None:
+    try:
+        for oid in oids:
+            stdin.write(f"{oid}\n".encode("ascii"))
+        stdin.close()
+    except BrokenPipeError:
+        pass
+
+
 def generate_manifest(root: Path) -> str:
     env = os.environ.copy()
     env["LC_ALL"] = "C"
@@ -103,23 +113,33 @@ def generate_manifest(root: Path) -> str:
     assert proc.stdin is not None
     assert proc.stdout is not None
 
-    stdin_buf = "".join(f"{oid}\n" for _path, _mode, oid in rows).encode("ascii")
-    proc.stdin.write(stdin_buf)
-    proc.stdin.close()
-
-    for path, _mode, _oid in rows:
-        header = proc.stdout.readline()
-        if not header:
-            raise RuntimeError("git cat-file --batch closed stdout early")
-        blob = _read_batch_blob(proc.stdout, header)
-        if blob is None:
-            continue
-        digest = hashlib.sha256(blob).hexdigest()
-        lines.append(f"{digest}  {path}")
-
-    proc.wait()
-    if proc.returncode not in (0, None):
-        raise RuntimeError(f"git cat-file --batch exited {proc.returncode}")
+    # Writer thread feeds oids while this thread reads blobs. Writing the full
+    # oid list first deadlocks once cat-file fills the stdout pipe (about 64KiB).
+    writer = threading.Thread(
+        target=_feed_cat_file_stdin,
+        args=(proc.stdin, [oid for _path, _mode, oid in rows]),
+        daemon=True,
+    )
+    writer.start()
+    try:
+        for path, _mode, _oid in rows:
+            header = proc.stdout.readline()
+            if not header:
+                raise RuntimeError("git cat-file --batch closed stdout early")
+            blob = _read_batch_blob(proc.stdout, header)
+            if blob is None:
+                continue
+            digest = hashlib.sha256(blob).hexdigest()
+            lines.append(f"{digest}  {path}")
+        writer.join(timeout=30)
+        proc.wait()
+        if proc.returncode not in (0, None):
+            raise RuntimeError(f"git cat-file --batch exited {proc.returncode}")
+    except Exception:
+        proc.kill()
+        writer.join(timeout=5)
+        proc.wait()
+        raise
 
     return "\n".join(lines) + "\n"