From a058a2856dec7d3e837ca5034fe892c83bc8077e Mon Sep 17 00:00:00 2001 From: Ayden Meng Date: Thu, 27 Nov 2025 20:57:59 +0800 Subject: [PATCH] BaseTools: Cap AutoGen thread count to avoid file descriptor exhaustion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the number of build threads multiplied by per-thread file descriptor usage exceeds the system's open file descriptor limit, some threads may fail to acquire necessary resources (e.g., pipes or semaphores), leading to deadlocks or hangs during parallel builds. To prevent this situation, calculate the safety upper limit of concurrency by dividing the system's maximum file descriptor limit by 3 (An empirical value derived from balancing performance overhead against the theoretical number of file descriptors consumed per thread). The actual thread count is then clamped to this safe value. Other usages of ThreadNum()—such as during actual compilation or log queue creation—do not significantly contribute to file descriptor consumption. Therefore, adjusting ThreadNum() globally would be unwarranted, as it could unnecessarily restrict parallelism in stages that are not FD-bound. This ensures stable parallel builds even under constrained resource limits. Signed-off-by: Ayden Meng --- BaseTools/Source/Python/build/build.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 947eeab745..d403851d51 100755 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -838,13 +838,22 @@ class Build(): try: if SkipAutoGen: return True,0 + if sys.platform == "win32": + SafeThreadNumber = self.ThreadNumber + else: + import resource + soft = resource.getrlimit(resource.RLIMIT_NOFILE)[0] + SafeThreadNumber = min(self.ThreadNumber, soft // 3) + if SafeThreadNumber < self.ThreadNumber: + EdkLogger.verbose("AutoGen workers limited to %d to avoid file descriptor exhaustion" % SafeThreadNumber) feedback_q = mp.Queue() error_event = mp.Event() FfsCmd = DataPipe.Get("FfsCommand") if FfsCmd is None: FfsCmd = {} GlobalData.FfsCmd = FfsCmd - auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,cqueue,self.log_q,error_event) for _ in range(self.ThreadNumber)] + + auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,cqueue,self.log_q,error_event) for _ in range(SafeThreadNumber)] self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event) self.AutoGenMgr.start() for w in auto_workers: