Grayscale/nanobot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve tool call validation strictness (#4190)

Browse Source 
* Improve tool call validation strictness

Reject near-miss tool names without executing suggested tools. Require object-shaped tool parameters while preserving only lossless JSON wire-shape normalization.

* Tighten tool call argument validation

* Simplify tool argument validation tests

* Improve tool name suggestions

* Simplify tool suggestion helpers

* Limit tool suggestions to canonical matches

* Allow repair only for tool history replay

* Clarify non-object tool argument errors

* Inline replay tool argument normalization

* Track only successful tool executions

* Reject JSON null tool arguments
This commit is contained in:
chengyongru
2026-06-09 14:50:40 +08:00
committed by GitHub
parent f3eb2aa08b
commit 0a396aa6e2
17 changed files with 769 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/providers/test_openai_responses.py
+90 -4
View File
@@ -155,6 +155,19 @@ class TestConvertMessages:
assert items[0]["call_id"] == "call_abc"
assert items[0]["id"] == "fc_1"
assert items[0]["name"] == "get_weather"
assert items[0]["arguments"] == '{"city": "SF"}'
def test_assistant_tool_call_history_repairs_malformed_arguments(self):
_, items = convert_messages([{
"role": "assistant",
"content": None,
"tool_calls": [{
"id": "call_abc|fc_1",
"function": {"name": "read_file", "arguments": '{path:"foo.txt"}'},
}],
}])
assert json.loads(items[0]["arguments"]) == {"path": "foo.txt"}
def test_duplicate_response_item_ids_are_made_unique(self):
"""Codex rejects replayed Responses input items with duplicate ids."""
@@ -367,7 +380,7 @@ class TestParseResponseOutput:
assert result.tool_calls[0].id == "call_1|fc_1"
def test_malformed_tool_arguments_logged(self):
"""Malformed JSON arguments should log a warning and fallback."""
"""Malformed JSON arguments should log a warning and remain non-object."""
resp = {
"output": [{
"type": "function_call",
@@ -378,10 +391,29 @@ class TestParseResponseOutput:
}
with patch("nanobot.providers.openai_responses.parsing.logger") as mock_logger:
result = parse_response_output(resp)
assert result.tool_calls[0].arguments == {"raw": "{bad json"}
assert result.tool_calls[0].arguments == "{bad json"
mock_logger.warning.assert_called_once()
assert "Failed to parse tool call arguments" in str(mock_logger.warning.call_args)
@pytest.mark.parametrize("arguments", [[], False, 0])
def test_falsy_non_object_tool_arguments_preserved(self, arguments):
resp = {
"output": [{
"type": "function_call",
"call_id": "c1",
"id": "fc1",
"name": "f",
"arguments": arguments,
}],
"status": "completed",
"usage": {},
}
result = parse_response_output(resp)
assert result.tool_calls[0].arguments == arguments
assert type(result.tool_calls[0].arguments) is type(arguments)
def test_reasoning_content_extracted(self):
resp = {
"output": [
@@ -611,6 +643,38 @@ class TestConsumeSse:
},
]
@pytest.mark.asyncio
@pytest.mark.parametrize("arguments", [[], False, 0])
async def test_falsy_non_object_tool_arguments_preserved(self, arguments):
response = _SseResponse([
{
"type": "response.output_item.added",
"item": {
"type": "function_call",
"call_id": "c1",
"id": "fc1",
"name": "f",
"arguments": "",
},
},
{
"type": "response.output_item.done",
"item": {
"type": "function_call",
"call_id": "c1",
"id": "fc1",
"name": "f",
"arguments": arguments,
},
},
{"type": "response.completed", "response": {"status": "completed"}},
])
_, tool_calls, _, _, _ = await consume_sse_with_reasoning(response)
assert tool_calls[0].arguments == arguments
assert type(tool_calls[0].arguments) is type(arguments)
# ======================================================================
# parsing - consume_sdk_stream
@@ -764,6 +828,28 @@ class TestConsumeSdkStream:
},
]
@pytest.mark.asyncio
@pytest.mark.parametrize("arguments", [[], False, 0])
async def test_falsy_non_object_tool_arguments_preserved(self, arguments):
item_added = MagicMock(type="function_call", call_id="c1", id="fc1", arguments="")
item_added.name = "f"
ev1 = MagicMock(type="response.output_item.added", item=item_added)
item_done = MagicMock(type="function_call", call_id="c1", id="fc1")
item_done.name = "f"
item_done.arguments = arguments
ev2 = MagicMock(type="response.output_item.done", item=item_done)
resp_obj = MagicMock(status="completed", usage=None, output=[])
ev3 = MagicMock(type="response.completed", response=resp_obj)
async def stream():
for e in [ev1, ev2, ev3]:
yield e
_, tool_calls, _, _, _ = await consume_sdk_stream(stream())
assert tool_calls[0].arguments == arguments
assert type(tool_calls[0].arguments) is type(arguments)
@pytest.mark.asyncio
async def test_usage_extracted(self):
usage_obj = MagicMock(input_tokens=10, output_tokens=5, total_tokens=15)
@@ -811,7 +897,7 @@ class TestConsumeSdkStream:
@pytest.mark.asyncio
async def test_malformed_tool_args_logged(self):
"""Malformed JSON in streaming tool args should log a warning."""
"""Malformed JSON in streaming tool args should log a warning and remain non-object."""
item_added = MagicMock(type="function_call", call_id="c1", id="fc1", arguments="")
item_added.name = "f"
ev1 = MagicMock(type="response.output_item.added", item=item_added)
@@ -828,6 +914,6 @@ class TestConsumeSdkStream:
with patch("nanobot.providers.openai_responses.parsing.logger") as mock_logger:
_, tool_calls, _, _, _ = await consume_sdk_stream(stream())
assert tool_calls[0].arguments == {"raw": "{bad"}
assert tool_calls[0].arguments == "{bad"
mock_logger.warning.assert_called_once()
assert "Failed to parse tool call arguments" in str(mock_logger.warning.call_args)