Grayscale/nanobot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(msteams): split ref storage into main+meta sidecar files

Browse Source 
- Separate updated_at into a meta sidecar file (msteams_conversations_meta.json)
    to keep backward compatibility with legacy data that never had updated_at.
    On first upgrade, legacy refs are kept alive by initializing updated_at to now
    instead of purging them immediately.
  - Add cross-process locking via fcntl (with Windows fallback) to prevent
    concurrent writes from different gateway processes overwriting each other.
  - Add ref_touch_interval_s config (default 300s) to throttle how often
    successful sends refresh updated_at, preventing unnecessary I/O.
  - Touch active refs on send success to prevent them from expiring while in use.
  - Add _safe_float and _normalize_ref_record for robust schema migration.
  - All refs operations now use threading.RLock within a process.
This commit is contained in:
zhuzhh
2026-04-25 15:39:43 +08:00
parent 15e9d0471f
commit fe928a0d94
3 changed files with 283 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_msteams.py
+91 -9
View File
@@ -115,7 +115,10 @@ async def test_handle_activity_personal_message_publishes_and_stores_ref(make_ch
saved = json.loads((tmp_path / "state" / "msteams_conversations.json").read_text(encoding="utf-8"))
assert saved["conv-123"]["conversation_id"] == "conv-123"
assert saved["conv-123"]["tenant_id"] == "tenant-id"
assert float(saved["conv-123"]["updated_at"]) > 0
saved_meta = json.loads(
(tmp_path / "state" / msteams_module.MSTEAMS_REF_META_FILENAME).read_text(encoding="utf-8"),
)
assert float(saved_meta["conv-123"]["updated_at"]) > 0
def test_init_prunes_stale_and_unsupported_conversation_refs(make_channel, tmp_path, monkeypatch):
@@ -125,6 +128,7 @@ def test_init_prunes_stale_and_unsupported_conversation_refs(make_channel, tmp_p
state_dir = tmp_path / "state"
state_dir.mkdir(parents=True, exist_ok=True)
refs_path = state_dir / "msteams_conversations.json"
refs_meta_path = state_dir / msteams_module.MSTEAMS_REF_META_FILENAME
refs_path.write_text(
json.dumps(
{
@@ -132,25 +136,21 @@ def test_init_prunes_stale_and_unsupported_conversation_refs(make_channel, tmp_p
"service_url": "https://smba.trafficmanager.net/amer/",
"conversation_id": "conv-valid",
"conversation_type": "personal",
"updated_at": now - 60,
},
"conv-webchat": {
"service_url": "https://webchat.botframework.com/",
"conversation_id": "conv-webchat",
"conversation_type": "personal",
"updated_at": now - 60,
},
"conv-group": {
"service_url": "https://smba.trafficmanager.net/amer/",
"conversation_id": "conv-group",
"conversation_type": "channel",
"updated_at": now - 60,
},
"conv-stale": {
"service_url": "https://smba.trafficmanager.net/amer/",
"conversation_id": "conv-stale",
"conversation_type": "personal",
"updated_at": now - msteams_module.MSTEAMS_REF_TTL_S - 1,
},
"conv-missing-ts": {
"service_url": "https://smba.trafficmanager.net/amer/",
@@ -162,14 +162,27 @@ def test_init_prunes_stale_and_unsupported_conversation_refs(make_channel, tmp_p
),
encoding="utf-8",
)
refs_meta_path.write_text(
json.dumps(
{
"conv-valid": {"updated_at": now - 60},
"conv-webchat": {"updated_at": now - 60},
"conv-group": {"updated_at": now - 60},
"conv-stale": {"updated_at": now - msteams_module.MSTEAMS_REF_TTL_S - 1},
},
indent=2,
),
encoding="utf-8",
)
ch = make_channel()
assert set(ch._conversation_refs.keys()) == {"conv-valid"}
assert set(ch._conversation_refs.keys()) == {"conv-valid", "conv-missing-ts"}
assert ch._conversation_refs["conv-valid"].conversation_id == "conv-valid"
assert ch._conversation_refs["conv-missing-ts"].conversation_id == "conv-missing-ts"
persisted = json.loads(refs_path.read_text(encoding="utf-8"))
assert set(persisted.keys()) == {"conv-valid"}
assert set(persisted.keys()) == {"conv-valid", "conv-missing-ts"}
def test_save_prunes_unsupported_conversation_refs(make_channel, tmp_path, monkeypatch):
@@ -204,6 +217,10 @@ def test_save_prunes_unsupported_conversation_refs(make_channel, tmp_path, monke
saved = json.loads((tmp_path / "state" / "msteams_conversations.json").read_text(encoding="utf-8"))
assert set(saved.keys()) == {"conv-valid"}
saved_meta = json.loads(
(tmp_path / "state" / msteams_module.MSTEAMS_REF_META_FILENAME).read_text(encoding="utf-8"),
)
assert set(saved_meta.keys()) == {"conv-valid"}
def test_init_respects_prune_toggle_flags(make_channel, tmp_path, monkeypatch):
@@ -248,6 +265,7 @@ def test_init_respects_custom_ref_ttl_days(make_channel, tmp_path, monkeypatch):
state_dir = tmp_path / "state"
state_dir.mkdir(parents=True, exist_ok=True)
refs_path = state_dir / "msteams_conversations.json"
refs_meta_path = state_dir / msteams_module.MSTEAMS_REF_META_FILENAME
refs_path.write_text(
json.dumps(
{
@@ -255,19 +273,27 @@ def test_init_respects_custom_ref_ttl_days(make_channel, tmp_path, monkeypatch):
"service_url": "https://smba.trafficmanager.net/amer/",
"conversation_id": "conv-fresh",
"conversation_type": "personal",
"updated_at": now - 12 * 60 * 60,
},
"conv-old": {
"service_url": "https://smba.trafficmanager.net/amer/",
"conversation_id": "conv-old",
"conversation_type": "personal",
"updated_at": now - 10 * 24 * 60 * 60,
},
},
indent=2,
),
encoding="utf-8",
)
refs_meta_path.write_text(
json.dumps(
{
"conv-fresh": {"updated_at": now - 12 * 60 * 60},
"conv-old": {"updated_at": now - 10 * 24 * 60 * 60},
},
indent=2,
),
encoding="utf-8",
)
ch = make_channel(refTtlDays=1)
@@ -276,6 +302,34 @@ def test_init_respects_custom_ref_ttl_days(make_channel, tmp_path, monkeypatch):
assert set(persisted.keys()) == {"conv-fresh"}
def test_init_without_meta_keeps_legacy_refs_alive(make_channel, tmp_path, monkeypatch):
now = 1_800_000_000.0
monkeypatch.setattr(msteams_module.time, "time", lambda: now)
state_dir = tmp_path / "state"
state_dir.mkdir(parents=True, exist_ok=True)
refs_path = state_dir / "msteams_conversations.json"
refs_path.write_text(
json.dumps(
{
"conv-legacy": {
"service_url": "https://smba.trafficmanager.net/amer/",
"conversation_id": "conv-legacy",
"conversation_type": "personal",
}
},
indent=2,
),
encoding="utf-8",
)
ch = make_channel(refTtlDays=1)
assert set(ch._conversation_refs.keys()) == {"conv-legacy"}
assert ch._conversation_refs["conv-legacy"].updated_at == now
assert not (state_dir / msteams_module.MSTEAMS_REF_META_FILENAME).exists()
def test_save_uses_atomic_replace_and_keeps_existing_file_on_replace_error(make_channel, tmp_path, monkeypatch):
ch = make_channel()
refs_path = tmp_path / "state" / "msteams_conversations.json"
@@ -591,6 +645,33 @@ async def test_send_replies_to_activity_when_reply_in_thread_enabled(make_channe
assert kwargs["json"]["replyToId"] == "activity-1"
@pytest.mark.asyncio
async def test_send_success_refreshes_updated_at_and_persists_meta(make_channel, tmp_path, monkeypatch):
now = {"value": 1_800_000_000.0}
monkeypatch.setattr(msteams_module.time, "time", lambda: now["value"])
ch = make_channel(refTouchIntervalS=0)
fake_http = FakeHttpClient()
ch._http = fake_http
ch._token = "tok"
ch._token_expires_at = 9_999_999_999
ch._conversation_refs["conv-123"] = ConversationRef(
service_url="https://smba.trafficmanager.net/amer/",
conversation_id="conv-123",
activity_id="activity-1",
updated_at=now["value"] - 100,
)
now["value"] += 5
await ch.send(OutboundMessage(channel="msteams", chat_id="conv-123", content="Reply text"))
assert ch._conversation_refs["conv-123"].updated_at == now["value"]
saved_meta = json.loads(
(tmp_path / "state" / msteams_module.MSTEAMS_REF_META_FILENAME).read_text(encoding="utf-8"),
)
assert saved_meta["conv-123"]["updated_at"] == now["value"]
@pytest.mark.asyncio
async def test_send_posts_to_conversation_when_thread_reply_disabled(make_channel):
ch = make_channel(replyInThread=False)
@@ -756,6 +837,7 @@ def test_msteams_default_config_includes_restart_notify_fields():
assert cfg["refTtlDays"] == msteams_module.MSTEAMS_REF_TTL_DAYS
assert cfg["pruneWebChatRefs"] is True
assert cfg["pruneNonPersonalRefs"] is True
assert cfg["refTouchIntervalS"] == msteams_module.MSTEAMS_REF_TOUCH_INTERVAL_S
assert "restartNotifyEnabled" not in cfg
assert "restartNotifyPreMessage" not in cfg
assert "restartNotifyPostMessage" not in cfg