feat: retry channel startup on transient failures (max 5 attempts, exponential backoff)
Test Suite / Detect changes (push) Has been cancelled
Test Suite / Python (Windows, 3.14) (push) Has been cancelled
Test Suite / Python (minimum, 3.11) (push) Has been cancelled
Test Suite / Python (latest, 3.14 + coverage) (push) Has been cancelled
Test Suite / webui (push) Has been cancelled
Test Suite / docker (push) Has been cancelled
Test Suite / Detect changes (push) Has been cancelled
Test Suite / Python (Windows, 3.14) (push) Has been cancelled
Test Suite / Python (minimum, 3.11) (push) Has been cancelled
Test Suite / Python (latest, 3.14 + coverage) (push) Has been cancelled
Test Suite / webui (push) Has been cancelled
Test Suite / docker (push) Has been cancelled
Previously, a single transient network error during channel startup (e.g., proxy blip causing Telegram get_me() to fail) would leave the channel permanently dead until the next gateway restart. Now _start_channel retries up to 5 times with exponential backoff starting at 2s (2s, 4s, 8s, 16s, 32s = ~62s total max wait).
This commit is contained in:
@@ -330,19 +330,40 @@ class ChannelManager:
|
|||||||
value = getattr(section, key, None)
|
value = getattr(section, key, None)
|
||||||
return value if isinstance(value, bool) else default
|
return value if isinstance(value, bool) else default
|
||||||
|
|
||||||
|
# Retry settings for channel startup (transient failures like network blips)
|
||||||
|
_CHANNEL_START_MAX_RETRIES: int = 5
|
||||||
|
_CHANNEL_START_BASE_DELAY: float = 2.0 # seconds, doubled each retry
|
||||||
|
|
||||||
async def _start_channel(self, name: str, channel: BaseChannel) -> None:
|
async def _start_channel(self, name: str, channel: BaseChannel) -> None:
|
||||||
"""Start a channel and log any exceptions."""
|
"""Start a channel with retries for transient failures."""
|
||||||
errors = getattr(self, "_channel_errors", None)
|
errors = getattr(self, "_channel_errors", None)
|
||||||
if errors is None:
|
if errors is None:
|
||||||
errors = self._channel_errors = {}
|
errors = self._channel_errors = {}
|
||||||
errors.pop(name, None)
|
errors.pop(name, None)
|
||||||
try:
|
|
||||||
await channel.start()
|
last_exc: Exception | None = None
|
||||||
except asyncio.CancelledError:
|
for attempt in range(self._CHANNEL_START_MAX_RETRIES + 1):
|
||||||
raise
|
try:
|
||||||
except Exception:
|
await channel.start()
|
||||||
errors[name] = "Channel failed to start. Check gateway logs."
|
if attempt > 0:
|
||||||
logger.exception("Failed to start channel {}", name)
|
logger.info("Channel {} started on attempt {}", name, attempt + 1)
|
||||||
|
return
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
last_exc = exc
|
||||||
|
if attempt < self._CHANNEL_START_MAX_RETRIES:
|
||||||
|
delay = self._CHANNEL_START_BASE_DELAY * (2 ** attempt)
|
||||||
|
logger.warning(
|
||||||
|
"Channel {} start attempt {}/{} failed: {}. Retrying in {:.0f}s...",
|
||||||
|
name, attempt + 1, self._CHANNEL_START_MAX_RETRIES + 1,
|
||||||
|
exc, delay,
|
||||||
|
)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
|
||||||
|
errors[name] = "Channel failed to start. Check gateway logs."
|
||||||
|
logger.exception("Failed to start channel {} after {} attempts",
|
||||||
|
name, self._CHANNEL_START_MAX_RETRIES + 1)
|
||||||
|
|
||||||
def _start_channel_task(self, name: str, channel: BaseChannel) -> asyncio.Task:
|
def _start_channel_task(self, name: str, channel: BaseChannel) -> asyncio.Task:
|
||||||
logger.info("Starting {} channel...", name)
|
logger.info("Starting {} channel...", name)
|
||||||
|
|||||||
Reference in New Issue
Block a user