Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions apps/ai/ai-analyzer/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,21 @@ Alertmanager (continue:true)┤ 메시지 본문 끝에 `fingerprint=<value>`
| `BEDROCK_MODEL_ID` | `apac.anthropic.claude-sonnet-4-20250514-v1:0` | APAC inference profile (GLOBAL Haiku 4.5는 Marketplace 구독 이슈로 임시 우회) |
| `BEDROCK_MAX_TOKENS` | `1024` | |
| `BEDROCK_TEMPERATURE` | `0.2` | 낮게 — 정확성 우선 |
| `AGENT_ENABLED` | `true` | tool-using agentic loop 활성화. `false` 시 single-shot 경로 fallback |
| `AGENT_MAX_ITERATIONS` | `5` | 한 알람 분석당 최대 model 호출 횟수 |
| `AM_URL` | `http://alertmanager.alertmanager:9093` | `list_recent_alerts` 도구의 Alertmanager v2 endpoint |

## Agentic 도구 (Tier 1)

`AGENT_ENABLED=true` 일 때 모델이 호출 가능한 도구:

| Tool | 인자 | 동작 |
|---|---|---|
| `query_metrics` | `promql`, `lookback_minutes` (≤360) | VictoriaMetrics 추가 PromQL 조회. 라벨은 dot 표기 |
| `query_logs` | `logql`, `lookback_minutes` (≤360), `limit` (≤500) | Loki 추가 LogQL 조회. 라벨은 underscore |
| `list_recent_alerts` | `window_minutes` (≤360) | Alertmanager 활성 알람 (트리거된 본인 제외). cascade 판별용 |

모든 도구는 read-only. 같은 (도구, 인자) 2회 중복 호출 감지 시 즉시 종료. 결과는 Slack footer에 `agent_iters=N` 으로 표시됨.

## 런북 추가 방법

Expand Down
244 changes: 244 additions & 0 deletions apps/ai/ai-analyzer/app/bedrock.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from __future__ import annotations

import asyncio
import json
import logging
import os
Expand All @@ -27,6 +28,9 @@
MAX_TOKENS = int(os.getenv("BEDROCK_MAX_TOKENS", "1024"))
TEMPERATURE = float(os.getenv("BEDROCK_TEMPERATURE", "0.2"))

AGENT_ENABLED = os.getenv("AGENT_ENABLED", "true").lower() in ("1", "true", "yes")
AGENT_MAX_ITERATIONS = int(os.getenv("AGENT_MAX_ITERATIONS", "5"))

SYSTEM_PROMPT = textwrap.dedent(
"""\
당신은 HighCloud 옵저버빌리티 스택(k3s + VictoriaMetrics + Loki + OTel)에서
Expand All @@ -48,13 +52,147 @@
`k8s.cluster.name`), Loki는 언더스코어로 정규화합니다 (예:
`k8s_cluster_name`). 제안하는 도구에 맞는 형식을 사용하세요.

제공된 도구 사용 지침:
- 1차 컨텍스트만으로 원인이 명확하면 **도구를 호출하지 말고 즉시 답변**.
- 가설을 좁히기 위해 추가 정보가 필요할 때만 도구를 호출. 한 번에 한 가지
가설씩, 가장 정보 가치가 큰 도구 하나를 골라 호출.
- 같은 `(도구, 인자)` 조합을 두 번 이상 호출하지 마세요 (중복 감지 시 강제 종료).
- PromQL/LogQL 은 가능한 한 좁은 selector 사용. 와일드카드 남발 금지.

출력 형식 (정확히):
1번째 줄: 가장 유력한 원인을 한 줄로 요약 (한국어).
2번째 줄 이후: 운영자가 실행할 읽기 전용 조사 명령어를 2~4개의 bullet로.
각 bullet은 한국어 설명 + 영문 코드 블록 조합으로 자기 완결적이어야 합니다.
"""
)


# Tool schemas exposed to the model via Bedrock Converse toolConfig.
TOOL_SCHEMAS: list[dict] = [
{
"toolSpec": {
"name": "query_metrics",
"description": (
"Run a PromQL/MetricsQL range query against VictoriaMetrics and "
"return the resulting samples. Use when you need to inspect a "
"metric that is not already in the 1차 컨텍스트 (e.g. correlating "
"another resource on the same node). Labels use DOT form on VM "
"(k8s.cluster.name)."
),
"inputSchema": {
"json": {
"type": "object",
"properties": {
"promql": {"type": "string", "description": "PromQL/MetricsQL expression"},
"lookback_minutes": {"type": "integer", "default": 15, "minimum": 1, "maximum": 360},
},
"required": ["promql"],
}
},
}
},
{
"toolSpec": {
"name": "query_logs",
"description": (
"Run a LogQL range query against Loki and return the matching log "
"lines (newest first). Use when you need to inspect logs beyond "
"what is in the 1차 컨텍스트 (different namespace, different time "
"window, different filter). Labels use UNDERSCORE form on Loki "
"(k8s_cluster_name)."
),
"inputSchema": {
"json": {
"type": "object",
"properties": {
"logql": {"type": "string", "description": "LogQL expression"},
"lookback_minutes": {"type": "integer", "default": 15, "minimum": 1, "maximum": 360},
"limit": {"type": "integer", "default": 100, "minimum": 1, "maximum": 500},
},
"required": ["logql"],
}
},
}
},
{
"toolSpec": {
"name": "list_recent_alerts",
"description": (
"List currently-active Alertmanager alerts that started within "
"the past `window_minutes`. Use to distinguish a single-shot "
"alert from a cascade. The triggering alert itself is excluded."
),
"inputSchema": {
"json": {
"type": "object",
"properties": {
"window_minutes": {"type": "integer", "default": 30, "minimum": 1, "maximum": 360},
},
"required": [],
}
},
}
},
]


def _summarize_tool_metrics(samples: list[dict], cap: int = 30) -> str:
if not samples:
return "(no samples)"
head = samples[:cap]
out = []
for s in head:
metric = s.get("metric") or {}
label_str = ",".join(f"{k}={v}" for k, v in list(metric.items())[:4])
out.append(f"{s.get('ts')}: {s.get('value')} {{{label_str}}}")
if len(samples) > cap:
out.append(f"… [{len(samples) - cap} more samples truncated]")
return "\n".join(out)


def _summarize_tool_logs(lines: list[str], cap: int = 40) -> str:
if not lines:
return "(no lines)"
head = lines[:cap]
rendered = "\n".join(head)
if len(lines) > cap:
rendered += f"\n… [{len(lines) - cap} more lines truncated]"
return rendered


def _summarize_alerts(alerts: list[dict]) -> str:
if not alerts:
return "(no other active alerts)"
return json.dumps(alerts[:25], indent=2, ensure_ascii=False)


async def _dispatch_tool(name: str, args: dict, fingerprint: str | None) -> str:
"""Run one tool call and return a model-facing text result."""
# Local import keeps bedrock module decoupled at import time and avoids a
# cycle if the context modules grow further.
from app.context import alertmanager, loki, vm

if name == "query_metrics":
samples = await vm.fetch_promql(
promql=args.get("promql", ""),
lookback_minutes=int(args.get("lookback_minutes", 15)),
)
return _summarize_tool_metrics(samples)
if name == "query_logs":
lines = await loki.fetch_logql(
logql=args.get("logql", ""),
lookback_minutes=int(args.get("lookback_minutes", 15)),
limit=int(args.get("limit", 100)),
)
return _summarize_tool_logs(lines)
if name == "list_recent_alerts":
alerts = await alertmanager.list_active(
window_minutes=int(args.get("window_minutes", 30)),
exclude_fingerprint=fingerprint,
)
return _summarize_alerts(alerts)
return f"(unknown tool: {name})"

_client = None


Expand Down Expand Up @@ -132,3 +270,109 @@ def analyze(
except Exception as e:
log.warning("bedrock failed: %s", e)
return None


async def analyze_agentic(
alert: dict,
loki_lines: list[str],
vm_samples: list[dict],
runbook: str,
fingerprint: str | None = None,
) -> tuple[str | None, int]:
"""Tool-using analyze loop. Returns (text, iterations).

Each iteration the model can either emit a final answer (end_turn) or
request a tool call (tool_use); we run the tool, append the result, and
let the model continue. Guard rails:
- max iterations = AGENT_MAX_ITERATIONS
- duplicate (tool, args) detection breaks the loop early
Falls back to None on any Bedrock failure so the caller can degrade.
"""
client = _client_lazy()
user_prompt = _build_user(alert, loki_lines, vm_samples, runbook)
messages: list[dict] = [
{"role": "user", "content": [{"text": user_prompt}]},
]
seen_calls: set[str] = set()
final_text: str | None = None
iterations = 0

for it in range(1, AGENT_MAX_ITERATIONS + 1):
iterations = it
try:
resp = await asyncio.get_event_loop().run_in_executor(
None,
lambda: client.converse(
modelId=MODEL_ID,
system=[{"text": SYSTEM_PROMPT}],
messages=messages,
toolConfig={"tools": TOOL_SCHEMAS},
inferenceConfig={"maxTokens": MAX_TOKENS, "temperature": TEMPERATURE},
),
)
except Exception as e:
log.warning("bedrock agentic failed (iter=%d): %s", it, e)
return None, iterations

usage = resp.get("usage", {})
stop_reason = resp.get("stopReason")
log.info(
"agent iter=%d stop=%s input=%s output=%s",
it, stop_reason, usage.get("inputTokens"), usage.get("outputTokens"),
)

message = resp["output"]["message"]
content = message.get("content", []) or []
messages.append({"role": "assistant", "content": content})

if stop_reason == "end_turn" or stop_reason == "max_tokens":
for block in content:
if "text" in block:
final_text = (final_text or "") + block["text"]
break

if stop_reason != "tool_use":
for block in content:
if "text" in block:
final_text = (final_text or "") + block["text"]
break

tool_uses = [b["toolUse"] for b in content if "toolUse" in b]
if not tool_uses:
for block in content:
if "text" in block:
final_text = (final_text or "") + block["text"]
break

tool_results: list[dict] = []
for tu in tool_uses:
name = tu.get("name", "")
args = tu.get("input", {}) or {}
use_id = tu.get("toolUseId", "")
sig = f"{name}|{json.dumps(args, sort_keys=True, ensure_ascii=False)}"
if sig in seen_calls:
log.info("agent duplicate tool call detected: %s", sig[:160])
tool_results.append({
"toolResult": {
"toolUseId": use_id,
"content": [{"text": "(duplicate tool call ignored — please conclude with current evidence.)"}],
"status": "error",
}
})
continue
seen_calls.add(sig)
try:
text = await _dispatch_tool(name, args, fingerprint)
except Exception as e:
log.warning("agent tool '%s' failed: %s", name, e)
text = f"(tool error: {e})"
tool_results.append({
"toolResult": {
"toolUseId": use_id,
"content": [{"text": text}],
}
})

messages.append({"role": "user", "content": tool_results})

return final_text, iterations
64 changes: 64 additions & 0 deletions apps/ai/ai-analyzer/app/context/alertmanager.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
"""Alertmanager active alert fetcher for agent context.

Used by the `list_recent_alerts` tool so the model can correlate the firing
alert with other concurrent alerts (cascade vs single-shot).
"""

from __future__ import annotations

import logging
import os
from datetime import datetime, timedelta, timezone

import httpx

log = logging.getLogger("ai-analyzer.am")

AM_URL = os.getenv("AM_URL", "")
HTTP_TIMEOUT = float(os.getenv("AM_HTTP_TIMEOUT", "8"))


async def list_active(window_minutes: int = 30, exclude_fingerprint: str | None = None) -> list[dict]:
"""Return a compact summary of active alerts within `window_minutes`.

Soft-fails: returns [] on any error so the agent can keep going.
Each entry: {alertname, severity, cluster, startsAt, status, fingerprint}.
"""
if not AM_URL:
log.warning("AM_URL not set; returning empty active-alert list")
return []

try:
async with httpx.AsyncClient(timeout=HTTP_TIMEOUT) as client:
r = await client.get(f"{AM_URL}/api/v2/alerts", params={"active": "true"})
r.raise_for_status()
data = r.json() or []
except Exception as e:
log.warning("alertmanager fetch failed: %s", e)
return []

cutoff = datetime.now(timezone.utc) - timedelta(minutes=window_minutes)
out: list[dict] = []
for a in data:
fp = a.get("fingerprint")
if exclude_fingerprint and fp == exclude_fingerprint:
continue
starts = a.get("startsAt") or ""
try:
started = datetime.fromisoformat(starts.replace("Z", "+00:00"))
except ValueError:
continue
if started < cutoff:
continue
labels = a.get("labels", {}) or {}
status = (a.get("status") or {}).get("state", "")
out.append({
"alertname": labels.get("alertname", ""),
"severity": labels.get("severity", ""),
"cluster": labels.get("cluster") or labels.get("k8s_cluster_name") or "",
"startsAt": starts,
"state": status,
"fingerprint": fp,
})
out.sort(key=lambda x: x.get("startsAt", ""), reverse=True)
return out
Loading
Loading