fix(cron): keep scheduler alive when job-store persistence fails
A single OSError from _save_store() (disk full, permission change, locked file) escaped _on_timer's try/finally and killed the asyncio timer task, because _arm_timer() sits outside the block. All scheduled jobs silently stopped until restart or a manual re-arm via add_job/update_job/remove_job. Move _arm_timer() into the finally block and guard the whole tick body (including _load_store, which can persist during agent-binding migrations) so a transient persistence failure is logged and retried on the next tick instead of killing the scheduler. Add test_save_store_failure_does_not_kill_scheduler to cover the failure path that existing tests (which mock _arm_timer) never exercised.
This commit is contained in:
+16
-5
@@ -515,11 +515,10 @@ class CronService:
|
||||
self._active_executions += 1
|
||||
try:
|
||||
store = self._load_store(reload_during_execution=reload_store)
|
||||
# If a hot reload found a corrupt store on disk, ``self._store`` may
|
||||
# still hold the previous, known-good in-memory snapshot. Keep using
|
||||
# it rather than crashing the timer or wiping live jobs.
|
||||
# If a hot reload found a corrupt store on disk, ``self._store``
|
||||
# may still hold the previous, known-good in-memory snapshot.
|
||||
# Keep using it rather than crashing the timer or wiping live jobs.
|
||||
if store is None:
|
||||
self._arm_timer()
|
||||
return
|
||||
|
||||
now = _now_ms()
|
||||
@@ -532,9 +531,21 @@ class CronService:
|
||||
await self._execute_job(job)
|
||||
|
||||
self._save_store()
|
||||
except Exception:
|
||||
# A load/persist failure must not kill the scheduler: keep the
|
||||
# in-memory store and retry on the next tick. This mirrors the
|
||||
# read-path defense in ``_load_jobs`` (``.corrupt-<ts>`` backups);
|
||||
# ``_load_store`` may also persist (agent-binding migrations).
|
||||
logger.exception(
|
||||
"Cron: tick failed ({}); "
|
||||
"keeping in-memory state and retrying on next tick",
|
||||
self.store_path,
|
||||
)
|
||||
finally:
|
||||
self._active_executions -= 1
|
||||
self._arm_timer()
|
||||
# Always re-arm the timer, even on unexpected failures, so a
|
||||
# single bad tick cannot silently stop all future jobs.
|
||||
self._arm_timer()
|
||||
|
||||
async def _execute_job(self, job: CronJob) -> None:
|
||||
"""Execute a single job."""
|
||||
|
||||
Reference in New Issue
Block a user