feat: retry channel startup on transient failures (max 5 attempts, exponential backoff)
Test Suite / Detect changes (push) Has been cancelled
Test Suite / Python (Windows, 3.14) (push) Has been cancelled
Test Suite / Python (minimum, 3.11) (push) Has been cancelled
Test Suite / Python (latest, 3.14 + coverage) (push) Has been cancelled
Test Suite / webui (push) Has been cancelled
Test Suite / docker (push) Has been cancelled
Test Suite / Detect changes (push) Has been cancelled
Test Suite / Python (Windows, 3.14) (push) Has been cancelled
Test Suite / Python (minimum, 3.11) (push) Has been cancelled
Test Suite / Python (latest, 3.14 + coverage) (push) Has been cancelled
Test Suite / webui (push) Has been cancelled
Test Suite / docker (push) Has been cancelled
Previously, a single transient network error during channel startup (e.g., proxy blip causing Telegram get_me() to fail) would leave the channel permanently dead until the next gateway restart. Now _start_channel retries up to 5 times with exponential backoff starting at 2s (2s, 4s, 8s, 16s, 32s = ~62s total max wait).
This commit is contained in:
@@ -330,19 +330,40 @@ class ChannelManager:
|
||||
value = getattr(section, key, None)
|
||||
return value if isinstance(value, bool) else default
|
||||
|
||||
# Retry settings for channel startup (transient failures like network blips)
|
||||
_CHANNEL_START_MAX_RETRIES: int = 5
|
||||
_CHANNEL_START_BASE_DELAY: float = 2.0 # seconds, doubled each retry
|
||||
|
||||
async def _start_channel(self, name: str, channel: BaseChannel) -> None:
|
||||
"""Start a channel and log any exceptions."""
|
||||
"""Start a channel with retries for transient failures."""
|
||||
errors = getattr(self, "_channel_errors", None)
|
||||
if errors is None:
|
||||
errors = self._channel_errors = {}
|
||||
errors.pop(name, None)
|
||||
try:
|
||||
await channel.start()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
errors[name] = "Channel failed to start. Check gateway logs."
|
||||
logger.exception("Failed to start channel {}", name)
|
||||
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(self._CHANNEL_START_MAX_RETRIES + 1):
|
||||
try:
|
||||
await channel.start()
|
||||
if attempt > 0:
|
||||
logger.info("Channel {} started on attempt {}", name, attempt + 1)
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
last_exc = exc
|
||||
if attempt < self._CHANNEL_START_MAX_RETRIES:
|
||||
delay = self._CHANNEL_START_BASE_DELAY * (2 ** attempt)
|
||||
logger.warning(
|
||||
"Channel {} start attempt {}/{} failed: {}. Retrying in {:.0f}s...",
|
||||
name, attempt + 1, self._CHANNEL_START_MAX_RETRIES + 1,
|
||||
exc, delay,
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
errors[name] = "Channel failed to start. Check gateway logs."
|
||||
logger.exception("Failed to start channel {} after {} attempts",
|
||||
name, self._CHANNEL_START_MAX_RETRIES + 1)
|
||||
|
||||
def _start_channel_task(self, name: str, channel: BaseChannel) -> asyncio.Task:
|
||||
logger.info("Starting {} channel...", name)
|
||||
|
||||
Reference in New Issue
Block a user