fix(cron): prevent replay after persistence failure
This commit is contained in:
+23
-1
@@ -170,6 +170,7 @@ class CronService:
|
||||
self._timer_task: asyncio.Task[None] | None = None
|
||||
self._running = False
|
||||
self._active_executions = 0
|
||||
self._store_dirty = False
|
||||
self.max_sleep_ms = max_sleep_ms
|
||||
|
||||
def _should_persist_store(self) -> bool:
|
||||
@@ -305,6 +306,11 @@ class CronService:
|
||||
load (during ``start``) can return ``None`` to signal an unrecoverable
|
||||
state to the caller.
|
||||
"""
|
||||
# Never replace state that a previous save failed to persist. Reloading
|
||||
# the older on-disk snapshot here could make an already executed job due
|
||||
# again and repeat its side effect.
|
||||
if self._store_dirty and self._store:
|
||||
return self._store
|
||||
if self._active_executions > 0 and self._store and not reload_during_execution:
|
||||
return self._store
|
||||
loaded = self._load_jobs()
|
||||
@@ -347,6 +353,9 @@ class CronService:
|
||||
if not self._store:
|
||||
return
|
||||
|
||||
# Set this before serialization/write so every exceptional exit keeps
|
||||
# the in-memory snapshot authoritative until a later save succeeds.
|
||||
self._store_dirty = True
|
||||
self.store_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
data = {
|
||||
@@ -399,6 +408,7 @@ class CronService:
|
||||
}
|
||||
|
||||
self._atomic_write(self.store_path, json.dumps(data, indent=2, ensure_ascii=False))
|
||||
self._store_dirty = False
|
||||
|
||||
@staticmethod
|
||||
def _atomic_write(path: Path, content: str) -> None:
|
||||
@@ -514,10 +524,17 @@ class CronService:
|
||||
reload_store = self._active_executions == 0
|
||||
self._active_executions += 1
|
||||
try:
|
||||
# A prior tick may have completed external side effects but failed
|
||||
# to persist their advanced schedule. Persist that exact snapshot
|
||||
# before reloading or executing anything else; otherwise the older
|
||||
# disk state can replay the same job.
|
||||
if self._store_dirty:
|
||||
self._save_store()
|
||||
return
|
||||
|
||||
store = self._load_store(reload_during_execution=reload_store)
|
||||
# If a hot reload found a corrupt store on disk, ``self._store``
|
||||
# may still hold the previous, known-good in-memory snapshot.
|
||||
# Keep using it rather than crashing the timer or wiping live jobs.
|
||||
if store is None:
|
||||
return
|
||||
|
||||
@@ -808,6 +825,11 @@ class CronService:
|
||||
reload_store = self._active_executions == 0
|
||||
self._active_executions += 1
|
||||
try:
|
||||
# A manual run is another side-effecting entrypoint. Do not start
|
||||
# it while the result of a previous timer execution is still only
|
||||
# in memory.
|
||||
if self._store_dirty:
|
||||
self._save_store()
|
||||
store = self._require_store(reload_during_execution=reload_store)
|
||||
for job in store.jobs:
|
||||
if job.id == job_id:
|
||||
|
||||
Reference in New Issue
Block a user