From 35f3084c031077a3c2ecf1e3b33477e2bd955396 Mon Sep 17 00:00:00 2001 From: chengyongru Date: Fri, 17 Apr 2026 10:00:28 +0800 Subject: [PATCH] feat(dream): per-line age annotations + dedup-aware prompt + max_iter=15 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three improvements to Dream's memory consolidation: 1. Per-line git-blame age annotations: MEMORY.md lines get `← Nd` suffixes (N>14) from dulwich annotate. SOUL.md/USER.md excluded as permanent. LLM uses content judgment, not just age, to decide what to prune. 2. Dedup-aware Phase 1 prompt: reframed as dual-task (extract facts + deduplicate existing files) with explicit redundancy patterns to scan for. Validated through 20 experiments (exp-002 prompt + max_iter=15 was best, averaging -1643 chars/5.4% compression per run). 3. Phase 1 analysis as commit body: dream git commits now include the full Phase 1 analysis for transparency via /dream-log. 4. max_iterations raised from 10 to 15: 30% improvement over 10 with no risk; 20 showed diminishing returns (exp-020: -701 vs exp-017: -1643). --- nanobot/agent/memory.py | 43 +++++++++++- nanobot/config/schema.py | 2 +- nanobot/templates/agent/dream_phase1.md | 24 +++++-- nanobot/utils/gitstore.py | 46 +++++++++++++ tests/agent/test_dream.py | 52 ++++++++++++++ tests/utils/test_gitstore.py | 91 +++++++++++++++++++++++++ 6 files changed, 247 insertions(+), 11 deletions(-) create mode 100644 tests/utils/test_gitstore.py diff --git a/nanobot/agent/memory.py b/nanobot/agent/memory.py index fbe27890..74a191aa 100644 --- a/nanobot/agent/memory.py +++ b/nanobot/agent/memory.py @@ -632,6 +632,40 @@ class Dream: # -- main entry ---------------------------------------------------------- + def _annotate_with_ages(self, content: str) -> str: + """Append per-line age suffixes to MEMORY.md content. + + Each non-blank line gets a suffix like ``← 30d`` indicating how + many days since it was last modified. Lines ≤14 days old get no + suffix. Returns the original content unchanged if git is unavailable. + SOUL.md and USER.md are never annotated. + """ + file_path = "memory/MEMORY.md" + try: + ages = self.store.git.line_ages(file_path) + except Exception: + logger.debug("line_ages failed for {}", file_path) + return content + if not ages: + return content + + had_trailing = content.endswith("\n") + lines = content.splitlines() + annotated: list[str] = [] + for i, line in enumerate(lines): + if not line.strip() or i >= len(ages): + annotated.append(line) + continue + d = ages[i].age_days + if d > 14: + annotated.append(f"{line} \u2190 {d}d") + else: + annotated.append(line) + result = "\n".join(annotated) + if had_trailing: + result += "\n" + return result + async def run(self) -> bool: """Process unprocessed history entries. Returns True if work was done.""" from nanobot.agent.skills import BUILTIN_SKILLS_DIR @@ -652,9 +686,10 @@ class Dream: f"[{e['timestamp']}] {e['content']}" for e in batch ) - # Current file contents + # Current file contents + per-line age annotations current_date = datetime.now().strftime("%Y-%m-%d") - current_memory = self.store.read_memory() or "(empty)" + raw_memory = self.store.read_memory() or "(empty)" + current_memory = self._annotate_with_ages(raw_memory) current_soul = self.store.read_soul() or "(empty)" current_user = self.store.read_user() or "(empty)" @@ -759,7 +794,9 @@ class Dream: # Git auto-commit (only when there are actual changes) if changelog and self.store.git.is_initialized(): ts = batch[-1]["timestamp"] - sha = self.store.git.auto_commit(f"dream: {ts}, {len(changelog)} change(s)") + summary = f"dream: {ts}, {len(changelog)} change(s)" + commit_msg = f"{summary}\n\n{analysis.strip()}" + sha = self.store.git.auto_commit(commit_msg) if sha: logger.info("Dream commit: {}", sha) diff --git a/nanobot/config/schema.py b/nanobot/config/schema.py index f6179c59..43fca612 100644 --- a/nanobot/config/schema.py +++ b/nanobot/config/schema.py @@ -43,7 +43,7 @@ class DreamConfig(Base): validation_alias=AliasChoices("modelOverride", "model", "model_override"), ) # Optional Dream-specific model override max_batch_size: int = Field(default=20, ge=1) # Max history entries per run - max_iterations: int = Field(default=10, ge=1) # Max tool calls per Phase 2 + max_iterations: int = Field(default=15, ge=1) # Max tool calls per Phase 2 def build_schedule(self, timezone: str) -> CronSchedule: """Build the runtime schedule, preferring the legacy cron override if present.""" diff --git a/nanobot/templates/agent/dream_phase1.md b/nanobot/templates/agent/dream_phase1.md index 3cc19b18..f42e7983 100644 --- a/nanobot/templates/agent/dream_phase1.md +++ b/nanobot/templates/agent/dream_phase1.md @@ -1,4 +1,6 @@ -Compare conversation history against current memory files. Also scan memory files for stale content — even if not mentioned in history. +You have TWO equally important tasks: +1. Extract new facts from conversation history +2. Deduplicate existing memory files — find and flag redundant, overlapping, or stale content even if NOT mentioned in history Output one line per finding: [FILE] atomic fact (not already in memory) @@ -12,12 +14,20 @@ Rules: - Corrections: [USER] location is Tokyo, not Osaka - Capture confirmed approaches the user validated -Staleness — flag for [FILE-REMOVE]: -- Time-sensitive data older than 14 days: weather, daily status, one-time meetings, passed events -- Completed one-time tasks: triage, one-time reviews, finished research, resolved incidents -- Resolved tracking: merged/closed PRs, fixed issues, completed migrations -- Detailed incident info after 14 days — reduce to one-line summary -- Superseded: approaches replaced by newer solutions, deprecated dependencies +Deduplication — scan ALL memory files for these redundancy patterns: +- Same fact stated in multiple places (e.g., "communicates in Chinese" in both USER.md and multiple MEMORY.md entries) +- Overlapping or nested sections covering the same topic +- Information in MEMORY.md that is already captured in USER.md or SOUL.md (MEMORY.md should not duplicate permanent-file content) +- Verbose entries that can be condensed without losing information +For each duplicate found, output [FILE-REMOVE] for the less authoritative copy (prefer keeping facts in their canonical location) + +Staleness — MEMORY.md lines may have a ``← Nd`` suffix showing days since last modification: +- SOUL.md and USER.md have no age annotations — they are permanent, only update with corrections +- Age only indicates when content was last touched, not whether it should be removed +- Use content judgment: user habits/preferences/personality traits are permanent regardless of age +- Only prune content that is objectively outdated: passed events, resolved tracking, superseded approaches +- Lines with ``← Nd`` (N>14) deserve closer review but are NOT automatically removable +- When removing: prefer deleting individual items over entire sections Skill discovery — flag [SKILL] when ALL of these are true: - A specific, repeatable workflow appeared 2+ times in the conversation history diff --git a/nanobot/utils/gitstore.py b/nanobot/utils/gitstore.py index c2f7d237..e51a63cc 100644 --- a/nanobot/utils/gitstore.py +++ b/nanobot/utils/gitstore.py @@ -5,6 +5,7 @@ from __future__ import annotations import io import time from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path from loguru import logger @@ -24,6 +25,23 @@ class CommitInfo: return f"{header}\n(no file changes)" +@dataclass +class LineAge: + """Age of a single line based on git blame.""" + + age_days: int # days since last modification + + +def _compute_line_ages(annotated) -> list[LineAge]: + """Convert annotate results to per-line ages.""" + now = datetime.now(tz=timezone.utc).date() + ages: list[LineAge] = [] + for (commit, _tree_entry), _line_bytes in annotated: + dt = datetime.fromtimestamp(commit.commit_time, tz=timezone.utc).date() + ages.append(LineAge(age_days=(now - dt).days)) + return ages + + class GitStore: """Git-backed version control for memory files.""" @@ -191,6 +209,34 @@ class GitStore: logger.warning("Git log failed") return [] + def line_ages(self, file_path: str) -> list[LineAge]: + """Compute the age of each line in a tracked file via git blame. + + Returns one LineAge per line, in order. + Returns an empty list if the repo is not initialized, the file is + empty, or annotation fails. + """ + + if not self.is_initialized(): + return [] + + target = self._workspace / file_path + if not target.exists() or target.stat().st_size == 0: + return [] + + try: + from dulwich import porcelain + + annotated = porcelain.annotate(str(self._workspace), file_path) + except Exception: + logger.warning("Git line_ages annotate failed for {}", file_path) + return [] + + if not annotated: + return [] + + return _compute_line_ages(annotated) + def diff_commits(self, sha1: str, sha2: str) -> str: """Show diff between two commits.""" if not self.is_initialized(): diff --git a/tests/agent/test_dream.py b/tests/agent/test_dream.py index eece79ed..2ca4286a 100644 --- a/tests/agent/test_dream.py +++ b/tests/agent/test_dream.py @@ -123,3 +123,55 @@ class TestDreamRun: assert "Successfully wrote" in result assert (store.workspace / "skills" / "test-skill" / "SKILL.md").exists() + async def test_phase1_prompt_includes_line_age_annotations(self, dream, mock_provider, mock_runner, store): + """Phase 1 prompt should have per-line age suffixes in MEMORY.md when git is available.""" + store.append_history("some event") + mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + + # Init git so line_ages works + store.git.init() + store.git.auto_commit("initial memory state") + + await dream.run() + + # The MEMORY.md section should not crash and should contain the memory content + call_args = mock_provider.chat_with_retry.call_args + user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"] + assert "## Current MEMORY.md" in user_msg + + async def test_phase1_annotates_only_memory_not_soul_or_user(self, dream, mock_provider, mock_runner, store): + """SOUL.md and USER.md should never have age annotations — they are permanent.""" + store.append_history("some event") + mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + + store.git.init() + store.git.auto_commit("initial state") + + await dream.run() + + call_args = mock_provider.chat_with_retry.call_args + user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"] + # The ← suffix should only appear in MEMORY.md section + memory_section = user_msg.split("## Current MEMORY.md")[1].split("## Current SOUL.md")[0] + soul_section = user_msg.split("## Current SOUL.md")[1].split("## Current USER.md")[0] + user_section = user_msg.split("## Current USER.md")[1] + # SOUL and USER should not contain age arrows + assert "\u2190" not in soul_section + assert "\u2190" not in user_section + + async def test_phase1_prompt_works_without_git(self, dream, mock_provider, mock_runner, store): + """Phase 1 should work fine even if git is not initialized (no age annotations).""" + store.append_history("some event") + mock_provider.chat_with_retry.return_value = MagicMock(content="[SKIP]") + mock_runner.run = AsyncMock(return_value=_make_run_result()) + + await dream.run() + + # Should still succeed — just without age annotations + mock_provider.chat_with_retry.assert_called_once() + call_args = mock_provider.chat_with_retry.call_args + user_msg = call_args.kwargs.get("messages", call_args[1].get("messages"))[1]["content"] + assert "## Current MEMORY.md" in user_msg + diff --git a/tests/utils/test_gitstore.py b/tests/utils/test_gitstore.py new file mode 100644 index 00000000..8c401e38 --- /dev/null +++ b/tests/utils/test_gitstore.py @@ -0,0 +1,91 @@ +"""Tests for GitStore — line_ages() and core git operations.""" + +import time +from datetime import datetime, timezone, timedelta +from unittest.mock import patch + +import pytest + +from nanobot.utils.gitstore import GitStore + + +@pytest.fixture +def git(tmp_path): + """Create an initialized GitStore with tracked MEMORY.md.""" + g = GitStore(tmp_path, tracked_files=["MEMORY.md", "SOUL.md"]) + g.init() + return g + + +class TestLineAges: + def test_returns_empty_when_not_initialized(self, tmp_path): + """line_ages should return [] if the git repo is not initialized.""" + git = GitStore(tmp_path, tracked_files=["MEMORY.md"]) + assert git.line_ages("MEMORY.md") == [] + + def test_returns_empty_for_missing_file(self, git): + """line_ages should return [] for a file that doesn't exist.""" + assert git.line_ages("SOUL.md") == [] + + def test_returns_empty_for_empty_file(self, git, tmp_path): + """line_ages should return [] for an empty tracked file.""" + (tmp_path / "SOUL.md").write_text("", encoding="utf-8") + git.auto_commit("empty soul") + assert git.line_ages("SOUL.md") == [] + + def test_one_age_per_line(self, git, tmp_path): + """line_ages should return one entry per line in the file.""" + content = "# Memory\n\n## Section A\n- item 1\n" + (tmp_path / "MEMORY.md").write_text(content, encoding="utf-8") + git.auto_commit("initial") + ages = git.line_ages("MEMORY.md") + assert len(ages) == len(content.splitlines()) + + def test_fresh_lines_have_age_zero(self, git, tmp_path): + """Lines committed today should have age_days=0.""" + (tmp_path / "MEMORY.md").write_text("## A\n- x\n", encoding="utf-8") + git.auto_commit("initial") + ages = git.line_ages("MEMORY.md") + assert all(a.age_days == 0 for a in ages) + + def test_age_differentiates_across_days(self, git, tmp_path): + """Lines committed today should show correct age when 'now' is mocked forward.""" + (tmp_path / "MEMORY.md").write_text("## A\n- x\n", encoding="utf-8") + git.auto_commit("initial") + + future_now = datetime.now(tz=timezone.utc) + timedelta(days=30) + with patch("nanobot.utils.gitstore.datetime") as mock_dt: + mock_dt.now.return_value = future_now + mock_dt.fromtimestamp = datetime.fromtimestamp + ages = git.line_ages("MEMORY.md") + + assert len(ages) == 2 + assert all(a.age_days == 30 for a in ages) + + def test_annotate_failure_returns_empty(self, tmp_path): + """If annotate fails, line_ages should return [] gracefully.""" + git = GitStore(tmp_path, tracked_files=["MEMORY.md"]) + # Don't init — annotate will fail + assert git.line_ages("MEMORY.md") == [] + + def test_partial_edit_only_updates_changed_lines(self, git, tmp_path): + """Only modified lines should reflect the new commit's timestamp.""" + (tmp_path / "MEMORY.md").write_text( + "# Memory\n\n## A\n- old\n\n## B\n- keep\n", encoding="utf-8" + ) + git.auto_commit("commit1") + time.sleep(1.1) + + # Only modify section A + (tmp_path / "MEMORY.md").write_text( + "# Memory\n\n## A\n- new\n\n## B\n- keep\n", encoding="utf-8" + ) + git.auto_commit("commit2") + + ages = git.line_ages("MEMORY.md") + lines = (tmp_path / "MEMORY.md").read_text(encoding="utf-8").splitlines() + # All lines are from today, but verify line-level tracking works + assert len(ages) == len(lines) + # "- new" line and "- keep" line both age=0 (same day), but + # the key point is we get per-line results + assert len(ages) == 7