mirror of
https://github.com/maziggy/bambuddy.git
synced 2026-08-11 00:30:12 -04:00
fix(restore): pause timer-based DB writers before swap (Postgres deadlock)
close_all_connections() only disposes the engine's connection pool —
asyncio tasks like print_scheduler.run() and the smart-plug snapshot
loop wake on their 30 s cadence and lazily reopen pool connections
holding RowExclusiveLock on print_queue / smart_plug_energy_snapshots.
The restore's DROP TABLE ... CASCADE pass needs AccessExclusiveLock on
every public table, producing an AB/BA deadlock that rolls back the
entire restore transaction.
Reproduced 2026-06-09 restoring a native install's backup into a fresh
Docker+Postgres deploy:
asyncpg.exceptions.DeadlockDetectedError: deadlock detected
Process X waits for AccessExclusiveLock on relation 109940
Process Y waits for RowExclusiveLock on relation 110182
Fix:
- Layer 1: pause print_scheduler / smart_plug_manager /
notification_service / background_dispatch via their existing stop
affordances before close_all_connections(), with a 1.0 s sleep for
in-flight loop iterations to release sessions. Restore handler
already requires a container restart on success, so the paused
services come back via the next lifespan startup.
- Layer 2: prepend SET LOCAL lock_timeout = '10s' to the begin-block
in _import_sqlite_to_postgres so any reactive writer (per-printer
MQTT, hourly AMS history recorder) that slips through the pause
window fails fast and visibly instead of producing a new deadlock.
This commit is contained in:
parent
ecdf577cfc
commit
047bb16c4c
2 changed files with 40 additions and 0 deletions
File diff suppressed because one or more lines are too long
|
|
@ -694,6 +694,15 @@ async def _import_sqlite_to_postgres(sqlite_path: Path, postgres_url: str):
|
|||
table.constraints.discard(fk)
|
||||
|
||||
async with pg_engine.begin() as conn:
|
||||
# Cap how long DROP TABLE will wait for AccessExclusiveLock so
|
||||
# any residual concurrent writer (per-printer MQTT clients
|
||||
# writing reactively, an AMS history recorder firing on its
|
||||
# hourly cadence) surfaces a fast `lock_timeout` error instead
|
||||
# of blocking the restore for 30 s or producing a deadlock.
|
||||
# SET LOCAL scopes to this transaction only; outside this
|
||||
# restore path the global default (no timeout) applies.
|
||||
await conn.execute(text("SET LOCAL lock_timeout = '10s'"))
|
||||
|
||||
# Drop every existing table in the public schema with CASCADE
|
||||
# rather than `metadata.drop_all`. Two reasons:
|
||||
# 1. The user's live DB may carry orphan tables from removed
|
||||
|
|
@ -910,6 +919,35 @@ async def restore_backup(
|
|||
except Exception as e:
|
||||
logger.warning("Failed to stop virtual printer: %s", e)
|
||||
|
||||
# 3b. Pause timer-based background services BEFORE the DB swap.
|
||||
# close_all_connections() below only disposes the engine's pool,
|
||||
# not the asyncio tasks that opened sessions from it. The print
|
||||
# scheduler (30 s cadence), smart-plug snapshot loop (30 s),
|
||||
# notification digest loop, and background dispatch worker all
|
||||
# wake up and call async_session(), which lazily re-creates a
|
||||
# pool connection holding RowExclusiveLock on print_queue /
|
||||
# smart_plug_energy_snapshots / etc. The DROP TABLE CASCADE
|
||||
# pass in the PostgreSQL restore path needs AccessExclusiveLock
|
||||
# on every public table, producing an AB/BA deadlock and a
|
||||
# full restore rollback. Successful restore already requires a
|
||||
# container restart, so we don't restart the services here.
|
||||
try:
|
||||
from backend.app.services.background_dispatch import background_dispatch
|
||||
from backend.app.services.notification_service import notification_service
|
||||
from backend.app.services.print_scheduler import scheduler as print_scheduler
|
||||
from backend.app.services.smart_plug_manager import smart_plug_manager
|
||||
|
||||
logger.info("Pausing background services for restore...")
|
||||
print_scheduler.stop()
|
||||
smart_plug_manager.stop_scheduler()
|
||||
notification_service.stop_digest_scheduler()
|
||||
await background_dispatch.stop()
|
||||
# In-flight loop iterations need a moment to commit + release
|
||||
# their DB sessions before we dispose() the engine pool.
|
||||
await asyncio.sleep(1.0)
|
||||
except Exception as e:
|
||||
logger.warning("Could not cleanly pause background services: %s", e)
|
||||
|
||||
# 4. Close current database connections
|
||||
logger.info("Closing database connections...")
|
||||
await close_all_connections()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue