migrate_hermes: Гермес → Simbiot (t-261001124130) — память/персона/история/config/cron → пакет + PolicyChanged в стор целевого инстанса + RAG-индексация; dry-run по умолчанию
This commit is contained in:
Simbiot Harness
parent
b9d7fd9d23
commit
61858a408b
1 changed files with 409 additions and 0 deletions
409
scripts/migrate_hermes.py
Normal file
409
scripts/migrate_hermes.py
Normal file
|
|
@ -0,0 +1,409 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""Simbiot Harness — миграция данных агента Гермес → Simbiot (t-261001124130).
|
||||
|
||||
Читает каталог Гермеса (~/.hermes): память (memories/MEMORY.md, USER.md),
|
||||
персону (SOUL.md), историю диалогов (state.db: sessions/messages), настройки
|
||||
(config.yaml), кроны (cron/jobs.json) — и собирает пакет для нового агента
|
||||
Simbiot:
|
||||
|
||||
OUT/identity.md — склейка персоны+памяти → policy.identity
|
||||
OUT/memory/MEMORY.md — факты памяти (как были, в формате Гермеса)
|
||||
OUT/memory/USER.md — профиль пользователя
|
||||
OUT/history/*.md — история диалогов, активные сессии (по файлу на сессию)
|
||||
OUT/history-index.md — индекс сессий
|
||||
OUT/config-hermes.yaml — настройка (секреты замаскированы)
|
||||
OUT/cron-map.json — джобы крона → эквиваленты Simbiot (задачи/таймеры)
|
||||
OUT/report.json — отчёт миграции (counts, notes, events)
|
||||
|
||||
Режимы:
|
||||
--apply — применить (по умолчанию dry-run: только отчёт, ничего
|
||||
не пишет в RAG/стор)
|
||||
--harness-dir DIR — при --apply: записать PolicyChanged/MemoryIngested в
|
||||
event store ЭТОГО инстанса Simbiot (агент получает
|
||||
память сразу в контекст тиков)
|
||||
--rag-db PATH --rag-py P --rag-script P — при --apply проиндексировать
|
||||
память/историю в RAG-базу TARGET-хоста
|
||||
|
||||
Идемпотентность: события с фиксированными aggregate_id (harness/policy-h-<hash>),
|
||||
RAG-индексация дедуплицируется по sha (вендор), файлы_OUT перезаписываются —
|
||||
повторный прогон безопасен.
|
||||
|
||||
Запуск: python3 scripts/migrate_hermes.py [--hermes /root/.hermes] [--apply ...]
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
MAX_HISTORY_SESSIONS = 40 # сколько последних сессий с диалогом переносим
|
||||
MAX_HISTORY_MSGS_PER_SESSION = 40
|
||||
MAX_TOTAL_HISTORY_MSGS = 150
|
||||
SECRET_RE = re.compile(
|
||||
r"(sk-[A-Za-z0-9_\-]{16,}|sk-or-v1-[A-Za-z0-9_\-]{16,}|agt_[A-Za-z0-9_\-]{10,}|"
|
||||
r"[0-9]{8,10}:[A-Za-z0-9_\-]{30,}|xox[bpars]-[A-Za-z0-9_\-]{10,}|"
|
||||
r"gh[pousr]_[A-Za-z0-9_\-]{20,})")
|
||||
|
||||
|
||||
def redact(text: str) -> str:
|
||||
return SECRET_RE.sub("<REDACTED>", text or "")
|
||||
|
||||
|
||||
def _jlog(msg: str):
|
||||
print(f"[migrate] {msg}", flush=True)
|
||||
|
||||
|
||||
# ─── источники ───────────────────────────────────────────────────────────────
|
||||
|
||||
def probe_sources(hermes_dir: str) -> dict:
|
||||
src = {"hermes_dir": hermes_dir, "files": {}, "state_db": {}, "cron": {}}
|
||||
cfg = os.path.join(hermes_dir, "config.yaml")
|
||||
src["files"]["config.yaml"] = os.path.getsize(cfg) if os.path.exists(cfg) else None
|
||||
for name, ext in (("MEMORY", "memories/MEMORY.md"), ("USER", "memories/USER.md"),
|
||||
("SOUL", "SOUL.md")):
|
||||
p = os.path.join(hermes_dir, ext)
|
||||
src["files"][ext] = os.path.getsize(p) if os.path.exists(p) else None
|
||||
state = os.path.join(hermes_dir, "state.db")
|
||||
src["files"]["state.db"] = os.path.getsize(state) if os.path.exists(state) else None
|
||||
if os.path.exists(state):
|
||||
db = sqlite3.connect(f"file:{state}?mode=ro", uri=True)
|
||||
db.row_factory = sqlite3.Row
|
||||
for t in ("messages", "sessions", "system_prompts"):
|
||||
try:
|
||||
src["state_db"][t] = db.execute(f"SELECT COUNT(*) c FROM {t}").fetchone()["c"]
|
||||
except sqlite3.Error:
|
||||
src["state_db"][t] = None
|
||||
try:
|
||||
r = db.execute("SELECT COUNT(*) c FROM messages WHERE active=1 "
|
||||
"AND role IN ('user','assistant')").fetchone()
|
||||
src["state_db"]["messages_active_user_assistant"] = r["c"]
|
||||
except sqlite3.Error:
|
||||
pass
|
||||
db.close()
|
||||
jobs = os.path.join(hermes_dir, "cron", "jobs.json")
|
||||
if os.path.exists(jobs):
|
||||
try:
|
||||
with open(jobs) as f:
|
||||
data = json.load(f)
|
||||
jlist = data.get("jobs") if isinstance(data, dict) else data
|
||||
src["cron"]["jobs_total"] = len(jlist or [])
|
||||
src["cron"]["jobs_enabled"] = sum(1 for j in (jlist or []) if j.get("enabled"))
|
||||
except Exception as e:
|
||||
src["cron"]["error"] = str(e)[:200]
|
||||
return src
|
||||
|
||||
|
||||
def extract_memory(hermes_dir: str) -> dict:
|
||||
"""Персона + память целиком (§-факты Гермеса и профиль юзера)."""
|
||||
out = {}
|
||||
parts_id = []
|
||||
soul = os.path.join(hermes_dir, "SOUL.md")
|
||||
if os.path.exists(soul):
|
||||
with open(soul, encoding="utf-8", errors="replace") as f:
|
||||
out["SOUL_raw_md"] = f.read()
|
||||
parts_id.append("## Персона (SOUL, из Гермеса)\n" + out["SOUL_raw_md"])
|
||||
mem = os.path.join(hermes_dir, "memories", "MEMORY.md")
|
||||
if os.path.exists(mem):
|
||||
with open(mem, encoding="utf-8", errors="replace") as f:
|
||||
out["MEMORY_raw_md"] = f.read()
|
||||
parts_id.append("## Память (MEMORY, из Гермеса, § = разделитель фактов)\n" + out["MEMORY_raw_md"])
|
||||
user = os.path.join(hermes_dir, "memories", "USER.md")
|
||||
if os.path.exists(user):
|
||||
with open(user, encoding="utf-8", errors="replace") as f:
|
||||
out["USER_raw_md"] = f.read()
|
||||
parts_id.append("## Профиль пользователя (USER, из Гермеса)\n" + out["USER_raw_md"])
|
||||
out["identity_md"] = "\n\n".join(parts_id)
|
||||
return out
|
||||
|
||||
|
||||
def extract_history(hermes_dir: str, out_dir: str) -> dict:
|
||||
"""Активные сессии с реальным диалогом (user+assistant) → markdown по файлам.
|
||||
|
||||
Крон-сессии (по 1 сообщению-запуску) не тянут — это шум; живые чаты
|
||||
(telegram/matrix/cli/subagent) переносим с капами (капы в шапке).
|
||||
"""
|
||||
state = os.path.join(hermes_dir, "state.db")
|
||||
res = {"sessions_total": 0, "sessions_written": 0, "messages_written": 0,
|
||||
"skipped_single_ping": 0, "files": []}
|
||||
if not os.path.exists(state):
|
||||
res["error"] = "state.db не найден"
|
||||
return res
|
||||
db = sqlite3.connect(f"file:{state}?mode=ro", uri=True)
|
||||
db.row_factory = sqlite3.Row
|
||||
sessions = db.execute("""
|
||||
SELECT s.id, s.source, s.chat_type, s.display_name, s.title,
|
||||
s.started_at, s.last_activity_at, COUNT(m.id) n
|
||||
FROM sessions s JOIN messages m ON m.session_id = s.id
|
||||
WHERE m.active = 1 AND m.role IN ('user', 'assistant')
|
||||
GROUP BY s.id
|
||||
HAVING n >= 2
|
||||
ORDER BY COALESCE(s.last_activity_at, s.started_at) DESC
|
||||
LIMIT ?""", (MAX_HISTORY_SESSIONS,)).fetchall()
|
||||
res["sessions_total"] = len(sessions)
|
||||
hdir = os.path.join(out_dir, "history")
|
||||
os.makedirs(hdir, exist_ok=True)
|
||||
idx_lines = ["# История диалогов Гермеса → Simbiot",
|
||||
"", "сессии с реальным диалогом (active=1, ≥2 сообщения),",
|
||||
f"капы: {MAX_HISTORY_MSGS_PER_SESSION}/сессия, {MAX_TOTAL_HISTORY_MSGS} всего", ""]
|
||||
total = 0
|
||||
for s in sessions:
|
||||
if total >= MAX_TOTAL_HISTORY_MSGS:
|
||||
idx_lines.append(f"- (усечено: достигнут глобальный кап {MAX_TOTAL_HISTORY_MSGS} сообщений)")
|
||||
break
|
||||
msgs = db.execute("""SELECT role, content, timestamp FROM messages
|
||||
WHERE session_id=? AND active=1
|
||||
AND role IN ('user','assistant')
|
||||
ORDER BY timestamp ASC""",
|
||||
(s["id"],)).fetchall()
|
||||
if len(msgs) < 2:
|
||||
res["skipped_single_ping"] += 1
|
||||
continue
|
||||
msgs = msgs[-MAX_HISTORY_MSGS_PER_SESSION:]
|
||||
title = (s["title"] or s["display_name"] or s["chat_type"] or s["id"])[:90]
|
||||
when = datetime.fromtimestamp(s["started_at"] or 0, tz=timezone.utc).strftime("%Y-%m-%d")
|
||||
body = [f"# {title}", "",
|
||||
f"session: {s['id']} · source: {s['source'] or '?'} · "
|
||||
f"type: {s['chat_type'] or '-'} · начать: {when} UTC", ""]
|
||||
for m in msgs:
|
||||
ts = datetime.fromtimestamp(m["timestamp"] or 0, tz=timezone.utc).strftime("%Y-%m-%d %H:%M")
|
||||
body.append(f"**[{ts} UTC] {m['role']}:** {redact(m['content'] or '')}")
|
||||
body.append("")
|
||||
fname = f"{s['id']}.md"
|
||||
path = os.path.join(hdir, fname)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(body))
|
||||
res["files"].append(path)
|
||||
res["sessions_written"] += 1
|
||||
res["messages_written"] += len(msgs)
|
||||
total += len(msgs)
|
||||
idx_lines.append(f"- [{title}]({os.path.basename(path)}) — "
|
||||
f"{s['source'] or '?'}, {len(msgs)} сообщ., "
|
||||
f"последнее {datetime.fromtimestamp(s['last_activity_at'] or s['started_at'] or 0, tz=timezone.utc).strftime('%Y-%m-%d')}")
|
||||
db.close()
|
||||
with open(os.path.join(out_dir, "history-index.md"), "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(idx_lines) + "\n")
|
||||
return res
|
||||
|
||||
|
||||
def extract_config(hermes_dir: str, out_dir: str) -> dict:
|
||||
cfg = os.path.join(hermes_dir, "config.yaml")
|
||||
res = {"file": None, "keys": []}
|
||||
if not os.path.exists(cfg):
|
||||
res["error"] = "config.yaml не найден"
|
||||
return res
|
||||
with open(cfg, encoding="utf-8", errors="replace") as f:
|
||||
raw = f.read()
|
||||
safe = redact(raw)
|
||||
dest = os.path.join(out_dir, "config-hermes.yaml")
|
||||
with open(dest, "w", encoding="utf-8") as f:
|
||||
f.write("# Настройка Гермеса (секреты замаскированы <REDACTED>)\n" + safe)
|
||||
res["file"] = dest
|
||||
res["keys"] = [ln.split(":", 1)[0] for ln in raw.splitlines()
|
||||
if ln and not ln[0].isspace() and not ln.startswith("#")
|
||||
and ln.endswith(":")]
|
||||
# маппинг в Simbiot (.env) — подсказки в отчёт
|
||||
mapping = []
|
||||
try:
|
||||
import yaml
|
||||
cfgd = yaml.safe_load(raw) or {}
|
||||
model = ((cfgd.get("model") or {}).get("default")
|
||||
if isinstance(cfgd.get("model"), dict) else None) or cfgd.get("model")
|
||||
if isinstance(model, str) and model:
|
||||
mapping.append({"hermes": "model", "simbiot": "SIMBIOT_MODEL", "value": model})
|
||||
tz = cfgd.get("timezone")
|
||||
if tz:
|
||||
mapping.append({"hermes": "timezone", "simbiot": "(env TZ хоста)", "value": tz})
|
||||
except Exception as e:
|
||||
mapping.append({"warning": f"yaml не разобран: {str(e)[:150]}"})
|
||||
res["simbiot_env_mapping"] = mapping
|
||||
return res
|
||||
|
||||
|
||||
def extract_cron(hermes_dir: str, out_dir: str) -> dict:
|
||||
jobs_path = os.path.join(hermes_dir, "cron", "jobs.json")
|
||||
res = {"jobs_total": 0, "jobs_enabled": 0, "mapped": []}
|
||||
if not os.path.exists(jobs_path):
|
||||
res["error"] = "cron/jobs.json не найден"
|
||||
return res
|
||||
with open(jobs_path) as f:
|
||||
data = json.load(f)
|
||||
jlist = data.get("jobs") if isinstance(data, dict) else data
|
||||
res["jobs_total"] = len(jlist or [])
|
||||
rows = []
|
||||
for j in (jlist or []):
|
||||
if not j.get("enabled"):
|
||||
continue
|
||||
res["jobs_enabled"] += 1
|
||||
sched = j.get("schedule") or {}
|
||||
expr = sched.get("expr") or sched.get("display") or ""
|
||||
rows.append({
|
||||
"id": (j.get("id") or "")[:16], "name": j.get("name"),
|
||||
"cadence": "cron " + expr if expr else sched.get("kind"),
|
||||
"prompt_sha": hashlib.sha256((j.get("prompt") or "").encode()).hexdigest()[:12],
|
||||
"prompt_chars": len(j.get("prompt") or ""),
|
||||
"simbiot_equivalent": "задача на доске по такту / systemd timer — см. docs",
|
||||
})
|
||||
res["mapped"] = rows
|
||||
with open(os.path.join(out_dir, "cron-map.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(res, f, ensure_ascii=False, indent=2)
|
||||
return res
|
||||
|
||||
|
||||
def write_identity_files(mem: dict, out_dir: str) -> list:
|
||||
os.makedirs(os.path.join(out_dir, "memory"), exist_ok=True)
|
||||
written = []
|
||||
id_path = os.path.join(out_dir, "identity.md")
|
||||
with open(id_path, "w", encoding="utf-8") as f:
|
||||
f.write((mem.get("identity_md") or "").strip() + "\n")
|
||||
written.append(id_path)
|
||||
if mem.get("MEMORY_raw_md"):
|
||||
p = os.path.join(out_dir, "memory", "MEMORY.md")
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
f.write(mem["MEMORY_raw_md"])
|
||||
written.append(p)
|
||||
if mem.get("USER_raw_md"):
|
||||
p = os.path.join(out_dir, "memory", "USER.md")
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
f.write(mem["USER_raw_md"])
|
||||
written.append(p)
|
||||
return written
|
||||
|
||||
|
||||
def rag_index(files: list, rag_db: str, rag_py: str, rag_script: str) -> dict:
|
||||
"""Память/история/личность → целевая RAG-база (вендор дедупит по sha)."""
|
||||
res = {"indexed": [], "failed": []}
|
||||
if not (os.path.exists(rag_py) and os.path.exists(rag_script)):
|
||||
res["error"] = "RAG-вендор не найден (rag-venv/rag.py) — индексация пропущена"
|
||||
return res
|
||||
for path in files:
|
||||
env = dict(os.environ, RAG_DB=rag_db)
|
||||
import subprocess
|
||||
try:
|
||||
proc = subprocess.run([rag_py, rag_script, "add", path],
|
||||
capture_output=True, text=True, timeout=120, env=env)
|
||||
if proc.returncode == 0:
|
||||
res["indexed"].append(os.path.basename(path))
|
||||
else:
|
||||
res["failed"].append({"file": os.path.basename(path),
|
||||
"err": (proc.stderr or proc.stdout)[-200:]})
|
||||
except Exception as e:
|
||||
res["failed"].append({"file": os.path.basename(path), "err": str(e)[:200]})
|
||||
return res
|
||||
|
||||
|
||||
def apply_to_store(harness_dir: str, identity_md: str, counts: dict) -> dict:
|
||||
"""PolicyChanged с памятью Гермеса в identity → тики нового агента видят её."""
|
||||
res = {"events": [], "error": None}
|
||||
try:
|
||||
sys.path.insert(0, harness_dir)
|
||||
from harness import es as target_es
|
||||
pol_marker = "policy-hermes-" + hashlib.sha256(identity_md.encode()).hexdigest()[:10]
|
||||
existing = {}
|
||||
try:
|
||||
model = target_es.run_projections()
|
||||
existing = model.get("policy") or {}
|
||||
except Exception:
|
||||
pass
|
||||
if existing.get("policy_hermes_marker") == pol_marker:
|
||||
res["events"].append({"event": "PolicyChanged", "status": "skip-duplicate"})
|
||||
return res
|
||||
identity = (
|
||||
"Миграция Гермеса → Simbiot: у агента есть перенесённая память и история "
|
||||
f"(сессий: {counts.get('sessions_written', 0)}). Ниже — персона и память. "
|
||||
"Факты выше считаются проверенными данными организации.\n\n" + identity_md)
|
||||
target_es.append_event("harness", "policy", "PolicyChanged", {
|
||||
**(existing or {}),
|
||||
"identity": (identity or "")[:12000],
|
||||
"policy_hermes_marker": pol_marker,
|
||||
"hermes_migration": {"applied_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"history_sessions": counts.get("sessions_written", 0)},
|
||||
})
|
||||
res["events"].append({"event": "PolicyChanged", "status": "applied",
|
||||
"marker": pol_marker})
|
||||
except Exception as e:
|
||||
res["error"] = str(e)[:300]
|
||||
return res
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Гермес → Simbiot")
|
||||
ap.add_argument("--hermes", default=os.path.expanduser("~/.hermes"))
|
||||
ap.add_argument("--out", default=None, help="каталог пакета (default: OUT/hermes-migration)")
|
||||
ap.add_argument("--apply", action="store_true", help="писать (default: dry-run)")
|
||||
ap.add_argument("--harness-dir", default=None, help="инстанс Simbiot для события PolicyChanged")
|
||||
ap.add_argument("--rag-db", default=None)
|
||||
ap.add_argument("--rag-py", default="/root/rag-venv/bin/python")
|
||||
ap.add_argument("--rag-script", default="/root/rag/rag.py")
|
||||
args = ap.parse_args()
|
||||
|
||||
simbiot_repo = os.path.dirname(os.path.abspath(__file__))
|
||||
out_dir = args.out or os.path.join(simbiot_repo, "hermes-migration")
|
||||
if args.apply:
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
|
||||
report = {"hermes_dir": args.hermes, "mode": "apply" if args.apply else "dry-run",
|
||||
"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
|
||||
|
||||
# 0) dry-run: ничего не создаём, только отчёт-превью в stdout
|
||||
if not args.apply:
|
||||
report["sources"] = probe_sources(args.hermes)
|
||||
mem = extract_memory(args.hermes)
|
||||
report["memory_preview"] = {k: len(v) for k, v in mem.items()
|
||||
if isinstance(v, str)}
|
||||
report["note"] = "dry-run: запусти с --apply (и --harness-dir <симбиот-инсталл>)"
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
# 1) карта источников (фиксируем ДО работы)
|
||||
report["sources"] = probe_sources(args.hermes)
|
||||
_jlog("sources: " + json.dumps(report["sources"], ensure_ascii=False)[:400])
|
||||
|
||||
# 2) память/личность
|
||||
mem = extract_memory(args.hermes)
|
||||
files = write_identity_files(mem, out_dir)
|
||||
_jlog(f"identity files: {len(files)}")
|
||||
|
||||
# 3) история
|
||||
hist = extract_history(args.hermes, out_dir)
|
||||
report["history"] = {k: v for k, v in hist.items() if k != "files"}
|
||||
_jlog(f"history: sessions={hist.get('sessions_written')}, msgs={hist.get('messages_written')}")
|
||||
files = files + hist.get("files", []) + [os.path.join(out_dir, "history-index.md")]
|
||||
|
||||
# 4) config.yaml
|
||||
report["config"] = extract_config(args.hermes, out_dir)
|
||||
if report["config"].get("file"):
|
||||
files.append(report["config"]["file"])
|
||||
|
||||
# 5) cron
|
||||
report["cron"] = extract_cron(args.hermes, out_dir)
|
||||
files.append(os.path.join(out_dir, "cron-map.json"))
|
||||
|
||||
# 6) RAG (память+история)
|
||||
if args.rag_db:
|
||||
report["rag"] = rag_index([f for f in files if f.endswith(".md")],
|
||||
args.rag_db, args.rag_py, args.rag_script)
|
||||
_jlog("rag: indexed=" + str(len(report["rag"].get("indexed", [])))
|
||||
+ " failed=" + str(len(report["rag"].get("failed", []))))
|
||||
|
||||
# 7) события нового инстанса
|
||||
counts = {"sessions_written": hist.get("sessions_written", 0)}
|
||||
report["store"] = apply_to_store(args.harness_dir if args.harness_dir
|
||||
else simbiot_repo, mem.get("identity_md", ""), counts)
|
||||
|
||||
# 8) отчёт
|
||||
report["written"] = {"files": len(files), "out_dir": out_dir,
|
||||
"identity_chars": len(mem.get("identity_md") or "")}
|
||||
with open(os.path.join(out_dir, "report.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||||
_jlog("report.json записан")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Reference in a new issue