fix(channels): prevent retry amplification and silent message loss across channels
Audited all channel implementations for overly broad exception handling that causes retry amplification or silent message loss during network errors. This is the same class of bug as #3050 (Telegram _send_text). Fixes by channel: Telegram (send_delta): - _stream_end path used except Exception for HTML edit fallback - Network errors (TimedOut, NetworkError) triggered redundant plain text edit, doubling connection demand during pool exhaustion - Changed to except BadRequest, matching the _send_text fix Discord: - send() caught all exceptions without re-raising - ChannelManager._send_with_retry() saw successful return, never retried - Messages silently dropped on any send failure - Added raise after error logging DingTalk: - _send_batch_message() returned False on all exceptions including network errors — no retry, fallback text sent unnecessarily - _read_media_bytes() and _upload_media() swallowed transport errors, causing _send_media_ref() to cascade through doomed fallback attempts - Added except httpx.TransportError handlers that re-raise immediately WeChat: - Media send failure triggered text fallback even for network errors - During network issues: 3×(media + text) = 6 API calls per message - Added specific catches: TimeoutException/TransportError re-raise, 5xx HTTPStatusError re-raises, 4xx falls back to text QQ: - _send_media() returned False on all exceptions - Network errors triggered fallback text instead of retry - Added except (aiohttp.ClientError, OSError) that re-raises Tests: 331 passed (283 existing + 48 new across 5 channel test files) Fixes: #3054 Related: #3050, #3053
This commit is contained in:
@@ -337,6 +337,9 @@ class DingTalkChannel(BaseChannel):
|
||||
content_type = (resp.headers.get("content-type") or "").split(";")[0].strip()
|
||||
filename = self._guess_filename(media_ref, self._guess_upload_type(media_ref))
|
||||
return resp.content, filename, content_type or None
|
||||
except httpx.TransportError as e:
|
||||
logger.error("DingTalk media download network error ref={} err={}", media_ref, e)
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("DingTalk media download error ref={} err={}", media_ref, e)
|
||||
return None, None, None
|
||||
@@ -388,6 +391,9 @@ class DingTalkChannel(BaseChannel):
|
||||
logger.error("DingTalk media upload missing media_id body={}", text[:500])
|
||||
return None
|
||||
return str(media_id)
|
||||
except httpx.TransportError as e:
|
||||
logger.error("DingTalk media upload network error type={} err={}", media_type, e)
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("DingTalk media upload error type={} err={}", media_type, e)
|
||||
return None
|
||||
@@ -437,6 +443,9 @@ class DingTalkChannel(BaseChannel):
|
||||
return False
|
||||
logger.debug("DingTalk message sent to {} with msgKey={}", chat_id, msg_key)
|
||||
return True
|
||||
except httpx.TransportError as e:
|
||||
logger.error("DingTalk network error sending message msgKey={} err={}", msg_key, e)
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Error sending DingTalk message msgKey={} err={}", msg_key, e)
|
||||
return False
|
||||
|
||||
@@ -366,6 +366,7 @@ class DiscordChannel(BaseChannel):
|
||||
await client.send_outbound(msg)
|
||||
except Exception as e:
|
||||
logger.error("Error sending Discord message: {}", e)
|
||||
raise
|
||||
finally:
|
||||
if not is_progress:
|
||||
await self._stop_typing(msg.chat_id)
|
||||
|
||||
@@ -362,7 +362,12 @@ class QQChannel(BaseChannel):
|
||||
|
||||
logger.info("QQ media sent: {}", filename)
|
||||
return True
|
||||
except (aiohttp.ClientError, OSError) as e:
|
||||
# Network / transport errors — propagate for retry by caller
|
||||
logger.warning("QQ send media network error filename={} err={}", filename, e)
|
||||
raise
|
||||
except Exception as e:
|
||||
# API-level or other non-network errors — return False so send() can fallback
|
||||
logger.error("QQ send media failed filename={} err={}", filename, e)
|
||||
return False
|
||||
|
||||
|
||||
@@ -570,7 +570,10 @@ class TelegramChannel(BaseChannel):
|
||||
chat_id=int_chat_id, message_id=buf.message_id,
|
||||
text=html, parse_mode="HTML",
|
||||
)
|
||||
except Exception as e:
|
||||
except BadRequest as e:
|
||||
# Only fall back to plain text on actual HTML parse/format errors.
|
||||
# Network errors (TimedOut, NetworkError) should propagate immediately
|
||||
# to avoid doubling connection demand during pool exhaustion.
|
||||
if self._is_not_modified_error(e):
|
||||
logger.debug("Final stream edit already applied for {}", chat_id)
|
||||
self._stream_bufs.pop(chat_id, None)
|
||||
|
||||
@@ -985,7 +985,43 @@ class WeixinChannel(BaseChannel):
|
||||
for media_path in (msg.media or []):
|
||||
try:
|
||||
await self._send_media_file(msg.chat_id, media_path, ctx_token)
|
||||
except (httpx.TimeoutException, httpx.TransportError) as net_err:
|
||||
# Network/transport errors: do NOT fall back to text —
|
||||
# the text send would also likely fail, and the outer
|
||||
# except will re-raise so ChannelManager retries properly.
|
||||
logger.error(
|
||||
"Network error sending WeChat media {}: {}",
|
||||
media_path,
|
||||
net_err,
|
||||
)
|
||||
raise
|
||||
except httpx.HTTPStatusError as http_err:
|
||||
status_code = (
|
||||
http_err.response.status_code
|
||||
if http_err.response is not None
|
||||
else 0
|
||||
)
|
||||
if status_code >= 500:
|
||||
# Server-side / retryable HTTP error — same as network.
|
||||
logger.error(
|
||||
"Server error ({} {}) sending WeChat media {}: {}",
|
||||
status_code,
|
||||
http_err.response.reason_phrase
|
||||
if http_err.response is not None
|
||||
else "",
|
||||
media_path,
|
||||
http_err,
|
||||
)
|
||||
raise
|
||||
# 4xx client errors are NOT retryable — fall back to text.
|
||||
filename = Path(media_path).name
|
||||
logger.error("Failed to send WeChat media {}: {}", media_path, http_err)
|
||||
await self._send_text(
|
||||
msg.chat_id, f"[Failed to send: {filename}]", ctx_token,
|
||||
)
|
||||
except Exception as e:
|
||||
# Non-network errors (format, file-not-found, etc.):
|
||||
# notify the user via text fallback.
|
||||
filename = Path(media_path).name
|
||||
logger.error("Failed to send WeChat media {}: {}", media_path, e)
|
||||
# Notify user about failure via text
|
||||
|
||||
Reference in New Issue
Block a user