b60439152b
- harness/es.py: append-only event store + проекции (образец — кооп-портал) - harness/levels.py: уровни 1с/1м/1ч/1д, контекст ∝ масштабу - harness/scheduler.py: уровневые потоки, идемпотентные тики, absorbed-стабы - harness/isolated_runner.py + _llm_worker.py: LLM-тик в сабпроцессе (глобальный asyncio-рантайм Strands не переживает 2+ create_harness) - cli.py: init/task/tasks/status/events + requeue (написан самим минутным агентом) - 9router: отдельный ключ sk-simbiot, модель auto (glm-5.3 первым)
134 lines
4.8 KiB
Python
134 lines
4.8 KiB
Python
"""Built-in LLM risk classifier for the HumanInTheLoop handler.
|
|
|
|
Uses an inner agent with structured output to evaluate whether a tool call
|
|
requires human approval based on risk criteria.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import Awaitable
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Protocol, runtime_checkable
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
from ...hooks.events import BeforeToolCallEvent
|
|
from ...models import Model
|
|
|
|
|
|
@dataclass
|
|
class ClassifierResult:
|
|
"""Result from a classifier evaluation."""
|
|
|
|
requires_human_in_the_loop: bool
|
|
reason: str | None = field(default=None)
|
|
|
|
|
|
@runtime_checkable
|
|
class HumanInTheLoopClassifier(Protocol):
|
|
"""Callable (sync or async) that decides whether a tool call requires human approval."""
|
|
|
|
def __call__(self, event: BeforeToolCallEvent, **kwargs: Any) -> ClassifierResult | Awaitable[ClassifierResult]:
|
|
"""Evaluate whether a tool call requires human approval.
|
|
|
|
Args:
|
|
event: The tool call event under evaluation.
|
|
**kwargs: Additional keyword arguments for future extensibility.
|
|
|
|
Returns:
|
|
ClassifierResult indicating whether approval is required.
|
|
"""
|
|
...
|
|
|
|
|
|
@dataclass
|
|
class LLMClassifierConfig:
|
|
"""Configuration for the built-in LLM risk classifier.
|
|
|
|
Args:
|
|
system_prompt: Risk criteria prompt. Defaults to a general-purpose risk prompt.
|
|
model: Model for risk evaluation. Defaults to the parent agent's model.
|
|
"""
|
|
|
|
system_prompt: str | None = field(default=None)
|
|
model: Model | None = field(default=None)
|
|
|
|
|
|
_DEFAULT_SYSTEM_PROMPT = """You are a risk evaluator for an AI agent's tool calls. Your job is to decide whether \
|
|
each tool call requires human approval before executing.
|
|
|
|
## When to require approval
|
|
|
|
Require approval when the tool call:
|
|
- Is destructive or irreversible (deleting data, dropping tables, revoking access)
|
|
- Modifies important state in production or shared environments
|
|
- Accesses or transmits sensitive data (credentials, PII, financial records)
|
|
- Communicates externally (sending emails, posting messages, making payments)
|
|
- Has a large blast radius (affecting many records, users, or systems)
|
|
|
|
## When approval is NOT needed
|
|
|
|
Do not require approval when the tool call:
|
|
- Is read-only AND does not access sensitive data (listing non-sensitive files, querying public data, searching)
|
|
- Operates on local or temporary resources
|
|
- Has easily reversible effects
|
|
- Is scoped to a single non-critical resource
|
|
|
|
Note: even read-only operations that access credentials, secrets, PII, or financial data still require approval.
|
|
|
|
## Instructions
|
|
|
|
Evaluate the tool name and its input arguments. Consider what could go wrong if this specific call executes \
|
|
with these specific arguments. When uncertain, require approval.
|
|
|
|
Keep your reason under 10 words — it is shown to a human in a CLI prompt."""
|
|
|
|
|
|
class _RiskDecision(BaseModel):
|
|
"""Structured output schema for the inner classification agent."""
|
|
|
|
requires_approval: bool = Field(description="Whether this tool call requires human approval before executing")
|
|
reason: str = Field(description="Brief reason (under 10 words) why approval is or is not required")
|
|
|
|
|
|
def _create_llm_risk_classifier(config: LLMClassifierConfig | None = None) -> HumanInTheLoopClassifier:
|
|
"""Create the built-in LLM risk classifier.
|
|
|
|
Args:
|
|
config: Optional configuration for the classifier.
|
|
|
|
Returns:
|
|
A classifier function that uses an inner LLM agent to evaluate risk.
|
|
"""
|
|
system_prompt = (config.system_prompt if config else None) or _DEFAULT_SYSTEM_PROMPT
|
|
configured_model = config.model if config else None
|
|
|
|
async def classifier(event: BeforeToolCallEvent, **kwargs: Any) -> ClassifierResult:
|
|
from ...agent import Agent
|
|
|
|
model = configured_model or event.agent.model
|
|
if not model:
|
|
raise ValueError(
|
|
"LLM risk classifier has no model — pass `model` in "
|
|
"`LLMClassifierConfig(model=...)`, or ensure the parent agent has a model."
|
|
)
|
|
|
|
inner = Agent(model=model, system_prompt=system_prompt, callback_handler=None)
|
|
tool_use = event.tool_use
|
|
prompt = (
|
|
f"Should this tool call require human approval?\n\n"
|
|
f"Tool: {tool_use['name']}\n"
|
|
f"Input: {json.dumps(tool_use['input'], indent=2)}"
|
|
)
|
|
result = await inner.invoke_async(prompt, structured_output_model=_RiskDecision)
|
|
decision = result.structured_output
|
|
if not isinstance(decision, _RiskDecision):
|
|
raise ValueError(f"LLM risk classifier produced no structured output (stop_reason={result.stop_reason!r})")
|
|
|
|
return ClassifierResult(
|
|
requires_human_in_the_loop=decision.requires_approval,
|
|
reason=decision.reason,
|
|
)
|
|
|
|
return classifier
|