fix(dream): ground memory audit records in the real git diff (#4673)

* fix(dream): ground commit messages and cursor advance in the real git diff

Dream consolidation could emit a /dream-log audit record that did not match
the actual file changes: build_dream_commit_message appended the LLM's
unverified resp.content, dream_run_completed only checked the stop reason, and
file contents were deliberately omitted from the prompt. The combination let a
single-turn self-report become the durable audit record.

- gitstore: add summarize_working_tree() — a structured, machine-derived
  summary (per-file +N/-M, totals, capped unified diff) of working-tree
  changes vs HEAD. Pure filesystem/git ground truth, never LLM narrative.
- memory: build_dream_commit_message now takes the diff body instead of resp;
  dream_content_diff() exposes the real delta over SOUL/USER/MEMORY.md only
  (excludes .dream_cursor so cursor writes aren't mistaken for edits);
  build_dream_prompt embeds current file contents so the model edits reality,
  not a stale mental model.
- builtin/cli: both Dream paths now compute the diff, gate cursor advance on
  a non-empty delta (no-op runs no longer swallow history), and commit with
  the diff-grounded message. Non-git workspaces fall back to the completion
  check.
- dream.md: document that contents are embedded, and add a chain-of-
  verification guardrail so the model's summary cannot claim unmade edits.

A regression test proves a lying resp.content never reaches the audit log
while the real diff does.

* fix(dream): mark non-UTF-8 memory files as binary in diff summary

Address review feedback (Q1 on PR #4673): summarize_working_tree read
working-tree files with errors="replace", which would emit U+FFFD
replacement chars into the audit record if a memory file ever held
invalid UTF-8 — misrepresenting the diff it is meant to make truthful.

Switch to errors="strict" and catch UnicodeDecodeError: a non-UTF-8
(or binary/corrupt) file is now recorded as "{path}: binary or
non-UTF-8 file changed" and omitted from the unified diff, so the
audit record stays honest. An empty diff block is also suppressed when
all changes are binary.

Adds a defensive regression test asserting no replacement char leaks.

nanobot/agent/memory.py

@@ -41,6 +41,14 @@ class MemoryStore:
     """Pure file I/O for memory files: MEMORY.md, history.jsonl, SOUL.md, USER.md."""
 
     _DEFAULT_MAX_HISTORY = 1000
+    # Durable files whose real working-tree delta grounds Dream commit messages
+    # and the cursor-advance gate. Deliberately excludes memory/.dream_cursor so
+    # that advancing the cursor itself is never mistaken for a productive edit.
+    _DREAM_CONTENT_PATHS = ("SOUL.md", "USER.md", "memory/MEMORY.md")
+    # Per-file cap when embedding current contents into the Dream prompt. The
+    # durable files are tiny in practice (~5 KB total), but a runaway file must
+    # not unbounded the prompt.
+    _DREAM_FILE_EMBED_CAP = 8000
     _INTERNAL_HISTORY_SESSION_PREFIXES = ("cron:", "dream:")
     _INTERNAL_HISTORY_SESSION_KEYS = {"heartbeat"}
     _LEGACY_ENTRY_START_RE = re.compile(r"^\[(\d{4}-\d{2}-\d{2}[^\]]*)\]\s*")
@@ -524,6 +532,11 @@ class MemoryStore:
         """Build the Dream prompt with unprocessed history context.
 
         Returns ``(prompt, last_cursor)`` or ``None`` if nothing to process.
+
+        The current contents of the durable memory files (SOUL.md, USER.md,
+        memory/MEMORY.md) are embedded so the model edits the real files rather
+        than a stale mental model — eliminating a class of failed/out-of-bounds
+        edits that previously produced hallucinated audit records.
         """
         last_cursor = self.get_last_dream_cursor()
         entries = self.read_unprocessed_history(since_cursor=last_cursor)
@@ -536,9 +549,46 @@ class MemoryStore:
             for e in batch
         )
         template = self._dream_template()
-        prompt = f"{template}\n\n## Conversation History\n{history_text}"
+        files_section = self._render_current_memory_files()
+        prompt = (
+            f"{template}\n\n{files_section}\n\n"
+            f"## Conversation History\n{history_text}"
+        )
         return (prompt, batch[-1]["cursor"])
 
+    def _render_current_memory_files(self) -> str:
+        """Render the durable memory files' current contents for the Dream prompt.
+
+        Missing files render as ``(empty)``; oversized files are capped. The
+        section is the ground truth the model must edit against.
+        """
+        files = [
+            ("SOUL.md", self.soul_file),
+            ("USER.md", self.user_file),
+            ("memory/MEMORY.md", self.memory_file),
+        ]
+        blocks = []
+        for label, path in files:
+            try:
+                content = path.read_text(encoding="utf-8") if path.exists() else ""
+            except OSError:
+                content = ""
+            if len(content) > self._DREAM_FILE_EMBED_CAP:
+                content = truncate_text(content, self._DREAM_FILE_EMBED_CAP) + "\n...[truncated]"
+            blocks.append(f"### {label}\n{content}" if content.strip() else f"### {label}\n(empty)")
+        return "## Current Memory Files\n" + "\n\n".join(blocks)
+
+    def dream_content_diff(self) -> str:
+        """Structured summary of uncommitted changes to the durable memory files.
+
+        Returns "" when git is unavailable or no content file changed. This is
+        the ground-truth input for diff-grounded Dream commit messages and for
+        gating cursor advance on real edits (never on LLM self-report).
+        """
+        if not self._git.is_initialized():
+            return ""
+        return self._git.summarize_working_tree(list(self._DREAM_CONTENT_PATHS))
+
     def build_dream_tools(self):
         """Build the restricted tool registry used by Dream runs."""
         from nanobot.agent.skills import BUILTIN_SKILLS_DIR
@@ -630,12 +680,22 @@ class MemoryStore:
         return f"dream:{datetime.now():%Y%m%d-%H%M%S}"
 
     @staticmethod
-    def build_dream_commit_message(prefix: str, resp: object | None) -> str:
-        """Build a Dream auto-commit message, appending the LLM summary if present."""
-        msg = prefix
-        if resp is not None and getattr(resp, "content", None):
-            msg = f"{msg}\n\n{resp.content.strip()}"
-        return msg
+    def build_dream_commit_message(prefix: str, diff_body: str) -> str:
+        """Build a Dream commit message grounded in the real working-tree diff.
+
+        *diff_body* is a structured, machine-derived summary of the actual file
+        changes (see :meth:`dream_content_diff` /
+        :meth:`GitStore.summarize_working_tree`). The LLM narrative is
+        deliberately excluded so the audit record (``/dream-log``) reflects the
+        filesystem's truth, not the model's self-report.
+
+        An empty *diff_body* yields the bare *prefix*, which ``auto_commit``
+        turns into a no-op when there is nothing to stage.
+        """
+        diff_body = (diff_body or "").strip()
+        if not diff_body:
+            return prefix
+        return f"{prefix}\n\n{diff_body}"
 
     @staticmethod
     def prune_dream_sessions(sessions_dir: Path, *, keep: int = 10) -> None:

nanobot/cli/commands.py

@@ -1397,6 +1397,7 @@ def _run_gateway(
 
             store = agent.context.memory
             resp = None
+            diff_body = ""
             try:
                 result = store.build_dream_prompt()
                 if result is None:
@@ -1411,9 +1412,20 @@ def _run_gateway(
                     tools=store.build_dream_tools(),
                     on_progress=_silent,
                 )
-                if MemoryStore.dream_run_completed(resp):
+                # Ground truth: the real file delta, not the LLM's self-report.
+                diff_body = store.dream_content_diff()
+                productive = bool(diff_body) or (
+                    not store.git.is_initialized()
+                    and MemoryStore.dream_run_completed(resp)
+                )
+                if productive:
                     store.set_last_dream_cursor(last_cursor)
                     logger.info("Dream cron job completed, cursor advanced to {}", last_cursor)
+                elif MemoryStore.dream_run_completed(resp):
+                    logger.info(
+                        "Dream cron job completed with no memory changes; "
+                        "cursor not advanced",
+                    )
                 else:
                     logger.warning(
                         "Dream cron job did not complete; cursor remains at {}",
@@ -1431,7 +1443,7 @@ def _run_gateway(
                 )
                 if store.git.is_initialized():
                     msg = build_dream_commit_message(
-                        "dream: periodic memory consolidation", resp,
+                        "dream: periodic memory consolidation", diff_body,
                     )
                     sha = store.git.auto_commit(msg)
                     if sha:

nanobot/command/builtin.py

@@ -359,6 +359,7 @@ async def cmd_dream(ctx: CommandContext) -> OutboundMessage:
         store = loop.context.memory
         content = ""
         resp = None
+        diff_body = ""
         t0 = time.monotonic()
         try:
             result = store.build_dream_prompt()
@@ -379,9 +380,17 @@ async def cmd_dream(ctx: CommandContext) -> OutboundMessage:
                 on_progress=_silent,
             )
             elapsed = time.monotonic() - t0
-            if MemoryStore.dream_run_completed(resp):
+            # Ground truth: the real file delta, not the LLM's self-report.
+            diff_body = store.dream_content_diff()
+            productive = bool(diff_body) or (
+                not store.git.is_initialized()
+                and MemoryStore.dream_run_completed(resp)
+            )
+            if productive:
                 store.set_last_dream_cursor(last_cursor)
                 content = f"Dream completed in {elapsed:.1f}s."
+            elif MemoryStore.dream_run_completed(resp):
+                content = f"Dream completed in {elapsed:.1f}s; no memory changes."
             else:
                 content = (
                     f"Dream did not complete after {elapsed:.1f}s; "
@@ -399,7 +408,7 @@ async def cmd_dream(ctx: CommandContext) -> OutboundMessage:
                 timezone_name=getattr(loop.context, "timezone", None),
             )
             if store.git.is_initialized():
-                commit_msg = build_dream_commit_message("dream: manual run", resp)
+                commit_msg = build_dream_commit_message("dream: manual run", diff_body)
                 sha = store.git.auto_commit(commit_msg)
                 if sha:
                     content += f" (commit {sha})"

nanobot/templates/agent/dream.md

@@ -99,7 +99,10 @@ For [SKILL] entries:
 - Skills are instruction sets with concrete values, commands, and examples. MEMORY.md keeps strategic context and high-level facts only.
 
 ## Editing
-- Inspect current file contents before editing; they are not embedded in the prompt to keep context compact.
+- Current contents of SOUL.md, USER.md, and memory/MEMORY.md are embedded in this prompt under "Current Memory Files". Edit those files directly; do not rely on a remembered version of a file.
 - Batch changes into as few calls as possible. Surgical edits only.
 
+## Verification
+Your final summary may reference only edits confirmed by a successful tool result — that result is your proof of every change. Do not narrate edits you did not make. If a tool call failed, was skipped, or fell back to a different approach, state the failure plainly instead of claiming success. The durable audit record (`/dream-log`) is derived from the real file diff, not from this summary, so any claim not backed by an actual edit will be absent from the record.
+
 Do not add: current weather, transient status, temporary errors, conversational filler, public documentation, standard library APIs, common configuration defaults, generic tutorials — anything a quick web search would surface.

nanobot/utils/gitstore.py

@@ -10,6 +10,11 @@ from pathlib import Path
 
 from loguru import logger
 
+# Cap on the unified-diff block embedded in Dream commit messages. Memory files
+# are tiny in practice, but a pathological rewrite must not blow up the audit
+# record. The structured per-file summary is always emitted in full regardless.
+_WORKING_TREE_DIFF_MAX_CHARS = 6000
+
 
 @dataclass
 class CommitInfo:
@@ -299,6 +304,119 @@ class GitStore:
             logger.exception("Git diff_commits failed")
             return ""
 
+    def summarize_working_tree(self, paths: list[str]) -> str:
+        """Structured summary of working-tree changes vs HEAD for *paths*.
+
+        Pure filesystem/git ground truth — never LLM narrative — suitable as a
+        truthful audit record. Returns "" when the repo is not initialized or
+        none of *paths* differ from HEAD.
+
+        Format::
+
+            SOUL.md: +3 -1
+            memory/MEMORY.md: +12 -8
+
+            2 files changed, 15 insertions(+), 9 deletions(-)
+
+            ```diff
+            --- SOUL.md
+            +++ SOUL.md
+            @@ ...
+            - old
+            + new
+            ```
+        """
+        if not self.is_initialized():
+            return ""
+
+        try:
+            import difflib
+
+            from dulwich.repo import Repo
+        except ImportError:
+            return ""
+
+        summary_lines: list[str] = []
+        diff_lines: list[str] = []
+        total_added = 0
+        total_removed = 0
+        changed = 0
+
+        try:
+            with Repo(str(self._workspace)) as repo:
+                head_tree = self._head_tree(repo)
+                for path in paths:
+                    head_text = (
+                        self._read_blob_from_tree(repo, head_tree, path)
+                        if head_tree is not None
+                        else None
+                    )
+                    if head_text is None:
+                        head_text = ""
+                    wt_path = self._workspace / path
+                    try:
+                        wt_text = (
+                            wt_path.read_text(encoding="utf-8")
+                            if wt_path.exists()
+                            else ""
+                        )
+                    except UnicodeDecodeError:
+                        # Non-UTF-8 (binary/corrupt) working-tree file: record
+                        # the change without a unified diff, which would
+                        # otherwise be polluted with replacement characters and
+                        # misrepresent the audit record.
+                        changed += 1
+                        summary_lines.append(f"{path}: binary or non-UTF-8 file changed")
+                        continue
+                    if head_text == wt_text:
+                        continue
+                    changed += 1
+                    hunks = list(difflib.unified_diff(
+                        head_text.splitlines(),
+                        wt_text.splitlines(),
+                        fromfile=path,
+                        tofile=path,
+                        lineterm="",
+                    ))
+                    added = sum(1 for line in hunks if line.startswith("+") and not line.startswith("+++"))
+                    removed = sum(1 for line in hunks if line.startswith("-") and not line.startswith("---"))
+                    total_added += added
+                    total_removed += removed
+                    summary_lines.append(f"{path}: +{added} -{removed}")
+                    diff_lines.extend(hunks)
+        except Exception:
+            logger.exception("Git summarize_working_tree failed")
+            return ""
+
+        if changed == 0:
+            return ""
+
+        diff_text = "\n".join(diff_lines)
+        if len(diff_text) > _WORKING_TREE_DIFF_MAX_CHARS:
+            diff_text = diff_text[:_WORKING_TREE_DIFF_MAX_CHARS] + "\n...[diff truncated]"
+
+        body = "\n".join(summary_lines)
+        body += (
+            f"\n{changed} file{'s' if changed != 1 else ''} changed, "
+            f"{total_added} insertion{'s' if total_added != 1 else ''}(+), "
+            f"{total_removed} deletion{'s' if total_removed != 1 else ''}(-)"
+        )
+        if diff_lines:
+            body += f"\n\n```diff\n{diff_text}\n```"
+        return body
+
+    @staticmethod
+    def _head_tree(repo) -> object | None:
+        """Return the tree object at HEAD, or None if there are no commits."""
+        try:
+            head = repo.refs[b"HEAD"]
+        except KeyError:
+            return None
+        commit = repo[head]
+        if commit.type_name != b"commit":
+            return None
+        return repo[commit.tree]
+
     def find_commit(self, short_sha: str, max_entries: int = 20) -> CommitInfo | None:
         """Find a commit by short SHA prefix match."""
         for c in self.log(max_entries=max_entries):