diff --git a/apps/ai/ai-analyzer/README.md b/apps/ai/ai-analyzer/README.md index afca26f..11bd919 100644 --- a/apps/ai/ai-analyzer/README.md +++ b/apps/ai/ai-analyzer/README.md @@ -92,6 +92,21 @@ Alertmanager (continue:true)┤ 메시지 본문 끝에 `fingerprint=` | `BEDROCK_MODEL_ID` | `apac.anthropic.claude-sonnet-4-20250514-v1:0` | APAC inference profile (GLOBAL Haiku 4.5는 Marketplace 구독 이슈로 임시 우회) | | `BEDROCK_MAX_TOKENS` | `1024` | | | `BEDROCK_TEMPERATURE` | `0.2` | 낮게 — 정확성 우선 | +| `AGENT_ENABLED` | `true` | tool-using agentic loop 활성화. `false` 시 single-shot 경로 fallback | +| `AGENT_MAX_ITERATIONS` | `5` | 한 알람 분석당 최대 model 호출 횟수 | +| `AM_URL` | `http://alertmanager.alertmanager:9093` | `list_recent_alerts` 도구의 Alertmanager v2 endpoint | + +## Agentic 도구 (Tier 1) + +`AGENT_ENABLED=true` 일 때 모델이 호출 가능한 도구: + +| Tool | 인자 | 동작 | +|---|---|---| +| `query_metrics` | `promql`, `lookback_minutes` (≤360) | VictoriaMetrics 추가 PromQL 조회. 라벨은 dot 표기 | +| `query_logs` | `logql`, `lookback_minutes` (≤360), `limit` (≤500) | Loki 추가 LogQL 조회. 라벨은 underscore | +| `list_recent_alerts` | `window_minutes` (≤360) | Alertmanager 활성 알람 (트리거된 본인 제외). cascade 판별용 | + +모든 도구는 read-only. 같은 (도구, 인자) 2회 중복 호출 감지 시 즉시 종료. 결과는 Slack footer에 `agent_iters=N` 으로 표시됨. ## 런북 추가 방법 diff --git a/apps/ai/ai-analyzer/app/bedrock.py b/apps/ai/ai-analyzer/app/bedrock.py index a38a50e..e2f70ad 100644 --- a/apps/ai/ai-analyzer/app/bedrock.py +++ b/apps/ai/ai-analyzer/app/bedrock.py @@ -10,6 +10,7 @@ from __future__ import annotations +import asyncio import json import logging import os @@ -27,6 +28,9 @@ MAX_TOKENS = int(os.getenv("BEDROCK_MAX_TOKENS", "1024")) TEMPERATURE = float(os.getenv("BEDROCK_TEMPERATURE", "0.2")) +AGENT_ENABLED = os.getenv("AGENT_ENABLED", "true").lower() in ("1", "true", "yes") +AGENT_MAX_ITERATIONS = int(os.getenv("AGENT_MAX_ITERATIONS", "5")) + SYSTEM_PROMPT = textwrap.dedent( """\ 당신은 HighCloud 옵저버빌리티 스택(k3s + VictoriaMetrics + Loki + OTel)에서 @@ -48,6 +52,13 @@ `k8s.cluster.name`), Loki는 언더스코어로 정규화합니다 (예: `k8s_cluster_name`). 제안하는 도구에 맞는 형식을 사용하세요. + 제공된 도구 사용 지침: + - 1차 컨텍스트만으로 원인이 명확하면 **도구를 호출하지 말고 즉시 답변**. + - 가설을 좁히기 위해 추가 정보가 필요할 때만 도구를 호출. 한 번에 한 가지 + 가설씩, 가장 정보 가치가 큰 도구 하나를 골라 호출. + - 같은 `(도구, 인자)` 조합을 두 번 이상 호출하지 마세요 (중복 감지 시 강제 종료). + - PromQL/LogQL 은 가능한 한 좁은 selector 사용. 와일드카드 남발 금지. + 출력 형식 (정확히): 1번째 줄: 가장 유력한 원인을 한 줄로 요약 (한국어). 2번째 줄 이후: 운영자가 실행할 읽기 전용 조사 명령어를 2~4개의 bullet로. @@ -55,6 +66,133 @@ """ ) + +# Tool schemas exposed to the model via Bedrock Converse toolConfig. +TOOL_SCHEMAS: list[dict] = [ + { + "toolSpec": { + "name": "query_metrics", + "description": ( + "Run a PromQL/MetricsQL range query against VictoriaMetrics and " + "return the resulting samples. Use when you need to inspect a " + "metric that is not already in the 1차 컨텍스트 (e.g. correlating " + "another resource on the same node). Labels use DOT form on VM " + "(k8s.cluster.name)." + ), + "inputSchema": { + "json": { + "type": "object", + "properties": { + "promql": {"type": "string", "description": "PromQL/MetricsQL expression"}, + "lookback_minutes": {"type": "integer", "default": 15, "minimum": 1, "maximum": 360}, + }, + "required": ["promql"], + } + }, + } + }, + { + "toolSpec": { + "name": "query_logs", + "description": ( + "Run a LogQL range query against Loki and return the matching log " + "lines (newest first). Use when you need to inspect logs beyond " + "what is in the 1차 컨텍스트 (different namespace, different time " + "window, different filter). Labels use UNDERSCORE form on Loki " + "(k8s_cluster_name)." + ), + "inputSchema": { + "json": { + "type": "object", + "properties": { + "logql": {"type": "string", "description": "LogQL expression"}, + "lookback_minutes": {"type": "integer", "default": 15, "minimum": 1, "maximum": 360}, + "limit": {"type": "integer", "default": 100, "minimum": 1, "maximum": 500}, + }, + "required": ["logql"], + } + }, + } + }, + { + "toolSpec": { + "name": "list_recent_alerts", + "description": ( + "List currently-active Alertmanager alerts that started within " + "the past `window_minutes`. Use to distinguish a single-shot " + "alert from a cascade. The triggering alert itself is excluded." + ), + "inputSchema": { + "json": { + "type": "object", + "properties": { + "window_minutes": {"type": "integer", "default": 30, "minimum": 1, "maximum": 360}, + }, + "required": [], + } + }, + } + }, +] + + +def _summarize_tool_metrics(samples: list[dict], cap: int = 30) -> str: + if not samples: + return "(no samples)" + head = samples[:cap] + out = [] + for s in head: + metric = s.get("metric") or {} + label_str = ",".join(f"{k}={v}" for k, v in list(metric.items())[:4]) + out.append(f"{s.get('ts')}: {s.get('value')} {{{label_str}}}") + if len(samples) > cap: + out.append(f"… [{len(samples) - cap} more samples truncated]") + return "\n".join(out) + + +def _summarize_tool_logs(lines: list[str], cap: int = 40) -> str: + if not lines: + return "(no lines)" + head = lines[:cap] + rendered = "\n".join(head) + if len(lines) > cap: + rendered += f"\n… [{len(lines) - cap} more lines truncated]" + return rendered + + +def _summarize_alerts(alerts: list[dict]) -> str: + if not alerts: + return "(no other active alerts)" + return json.dumps(alerts[:25], indent=2, ensure_ascii=False) + + +async def _dispatch_tool(name: str, args: dict, fingerprint: str | None) -> str: + """Run one tool call and return a model-facing text result.""" + # Local import keeps bedrock module decoupled at import time and avoids a + # cycle if the context modules grow further. + from app.context import alertmanager, loki, vm + + if name == "query_metrics": + samples = await vm.fetch_promql( + promql=args.get("promql", ""), + lookback_minutes=int(args.get("lookback_minutes", 15)), + ) + return _summarize_tool_metrics(samples) + if name == "query_logs": + lines = await loki.fetch_logql( + logql=args.get("logql", ""), + lookback_minutes=int(args.get("lookback_minutes", 15)), + limit=int(args.get("limit", 100)), + ) + return _summarize_tool_logs(lines) + if name == "list_recent_alerts": + alerts = await alertmanager.list_active( + window_minutes=int(args.get("window_minutes", 30)), + exclude_fingerprint=fingerprint, + ) + return _summarize_alerts(alerts) + return f"(unknown tool: {name})" + _client = None @@ -132,3 +270,109 @@ def analyze( except Exception as e: log.warning("bedrock failed: %s", e) return None + + +async def analyze_agentic( + alert: dict, + loki_lines: list[str], + vm_samples: list[dict], + runbook: str, + fingerprint: str | None = None, +) -> tuple[str | None, int]: + """Tool-using analyze loop. Returns (text, iterations). + + Each iteration the model can either emit a final answer (end_turn) or + request a tool call (tool_use); we run the tool, append the result, and + let the model continue. Guard rails: + - max iterations = AGENT_MAX_ITERATIONS + - duplicate (tool, args) detection breaks the loop early + Falls back to None on any Bedrock failure so the caller can degrade. + """ + client = _client_lazy() + user_prompt = _build_user(alert, loki_lines, vm_samples, runbook) + messages: list[dict] = [ + {"role": "user", "content": [{"text": user_prompt}]}, + ] + seen_calls: set[str] = set() + final_text: str | None = None + iterations = 0 + + for it in range(1, AGENT_MAX_ITERATIONS + 1): + iterations = it + try: + resp = await asyncio.get_event_loop().run_in_executor( + None, + lambda: client.converse( + modelId=MODEL_ID, + system=[{"text": SYSTEM_PROMPT}], + messages=messages, + toolConfig={"tools": TOOL_SCHEMAS}, + inferenceConfig={"maxTokens": MAX_TOKENS, "temperature": TEMPERATURE}, + ), + ) + except Exception as e: + log.warning("bedrock agentic failed (iter=%d): %s", it, e) + return None, iterations + + usage = resp.get("usage", {}) + stop_reason = resp.get("stopReason") + log.info( + "agent iter=%d stop=%s input=%s output=%s", + it, stop_reason, usage.get("inputTokens"), usage.get("outputTokens"), + ) + + message = resp["output"]["message"] + content = message.get("content", []) or [] + messages.append({"role": "assistant", "content": content}) + + if stop_reason == "end_turn" or stop_reason == "max_tokens": + for block in content: + if "text" in block: + final_text = (final_text or "") + block["text"] + break + + if stop_reason != "tool_use": + for block in content: + if "text" in block: + final_text = (final_text or "") + block["text"] + break + + tool_uses = [b["toolUse"] for b in content if "toolUse" in b] + if not tool_uses: + for block in content: + if "text" in block: + final_text = (final_text or "") + block["text"] + break + + tool_results: list[dict] = [] + for tu in tool_uses: + name = tu.get("name", "") + args = tu.get("input", {}) or {} + use_id = tu.get("toolUseId", "") + sig = f"{name}|{json.dumps(args, sort_keys=True, ensure_ascii=False)}" + if sig in seen_calls: + log.info("agent duplicate tool call detected: %s", sig[:160]) + tool_results.append({ + "toolResult": { + "toolUseId": use_id, + "content": [{"text": "(duplicate tool call ignored — please conclude with current evidence.)"}], + "status": "error", + } + }) + continue + seen_calls.add(sig) + try: + text = await _dispatch_tool(name, args, fingerprint) + except Exception as e: + log.warning("agent tool '%s' failed: %s", name, e) + text = f"(tool error: {e})" + tool_results.append({ + "toolResult": { + "toolUseId": use_id, + "content": [{"text": text}], + } + }) + + messages.append({"role": "user", "content": tool_results}) + + return final_text, iterations diff --git a/apps/ai/ai-analyzer/app/context/alertmanager.py b/apps/ai/ai-analyzer/app/context/alertmanager.py new file mode 100644 index 0000000..41f37ae --- /dev/null +++ b/apps/ai/ai-analyzer/app/context/alertmanager.py @@ -0,0 +1,64 @@ +"""Alertmanager active alert fetcher for agent context. + +Used by the `list_recent_alerts` tool so the model can correlate the firing +alert with other concurrent alerts (cascade vs single-shot). +""" + +from __future__ import annotations + +import logging +import os +from datetime import datetime, timedelta, timezone + +import httpx + +log = logging.getLogger("ai-analyzer.am") + +AM_URL = os.getenv("AM_URL", "") +HTTP_TIMEOUT = float(os.getenv("AM_HTTP_TIMEOUT", "8")) + + +async def list_active(window_minutes: int = 30, exclude_fingerprint: str | None = None) -> list[dict]: + """Return a compact summary of active alerts within `window_minutes`. + + Soft-fails: returns [] on any error so the agent can keep going. + Each entry: {alertname, severity, cluster, startsAt, status, fingerprint}. + """ + if not AM_URL: + log.warning("AM_URL not set; returning empty active-alert list") + return [] + + try: + async with httpx.AsyncClient(timeout=HTTP_TIMEOUT) as client: + r = await client.get(f"{AM_URL}/api/v2/alerts", params={"active": "true"}) + r.raise_for_status() + data = r.json() or [] + except Exception as e: + log.warning("alertmanager fetch failed: %s", e) + return [] + + cutoff = datetime.now(timezone.utc) - timedelta(minutes=window_minutes) + out: list[dict] = [] + for a in data: + fp = a.get("fingerprint") + if exclude_fingerprint and fp == exclude_fingerprint: + continue + starts = a.get("startsAt") or "" + try: + started = datetime.fromisoformat(starts.replace("Z", "+00:00")) + except ValueError: + continue + if started < cutoff: + continue + labels = a.get("labels", {}) or {} + status = (a.get("status") or {}).get("state", "") + out.append({ + "alertname": labels.get("alertname", ""), + "severity": labels.get("severity", ""), + "cluster": labels.get("cluster") or labels.get("k8s_cluster_name") or "", + "startsAt": starts, + "state": status, + "fingerprint": fp, + }) + out.sort(key=lambda x: x.get("startsAt", ""), reverse=True) + return out diff --git a/apps/ai/ai-analyzer/app/context/loki.py b/apps/ai/ai-analyzer/app/context/loki.py index a5ec5d9..74807f9 100644 --- a/apps/ai/ai-analyzer/app/context/loki.py +++ b/apps/ai/ai-analyzer/app/context/loki.py @@ -108,3 +108,44 @@ async def fetch(alert: dict, when: datetime | None = None) -> list[str]: for stream in streams: lines.extend(_format_lines(stream.get("values", []))) return lines[:MAX_LINES] + + +async def fetch_logql( + logql: str, + lookback_minutes: int = 15, + limit: int = 100, + when: datetime | None = None, +) -> list[str]: + """Run an arbitrary LogQL range query for the agent's tool use. + + Caller-supplied limit is clamped to MAX_LINES so a runaway tool call + can't drag in megabytes of logs. Soft-fails: returns [] on error. + """ + if not LOKI_URL: + return [] + if not logql or not logql.strip(): + return [] + safe_limit = max(1, min(limit, MAX_LINES)) + + when = when or datetime.now(timezone.utc) + start = when - timedelta(minutes=lookback_minutes) + params = { + "query": logql, + "start": int(start.timestamp() * 1e9), + "end": int(when.timestamp() * 1e9), + "limit": safe_limit, + "direction": "backward", + } + try: + async with httpx.AsyncClient(timeout=HTTP_TIMEOUT) as client: + r = await client.get(f"{LOKI_URL}/loki/api/v1/query_range", params=params) + r.raise_for_status() + streams = r.json().get("data", {}).get("result", []) or [] + except Exception as e: + log.warning("loki tool query failed: %s", e) + return [] + + lines: list[str] = [] + for stream in streams: + lines.extend(_format_lines(stream.get("values", []))) + return lines[:safe_limit] diff --git a/apps/ai/ai-analyzer/app/context/vm.py b/apps/ai/ai-analyzer/app/context/vm.py index b662f72..55a2769 100644 --- a/apps/ai/ai-analyzer/app/context/vm.py +++ b/apps/ai/ai-analyzer/app/context/vm.py @@ -83,3 +83,42 @@ async def fetch(alert: dict, when: datetime | None = None) -> list[dict]: for ts, val in series.get("values", []): samples.append({"ts": ts, "value": val}) return samples + + +async def fetch_promql( + promql: str, + lookback_minutes: int = 15, + when: datetime | None = None, +) -> list[dict]: + """Run an arbitrary PromQL/MetricsQL range query for the agent's tool use. + + Soft-fails: returns [] on any error. + """ + if not VM_URL: + return [] + if not promql or not promql.strip(): + return [] + + when = when or datetime.now(timezone.utc) + start = when - timedelta(minutes=lookback_minutes) + params = { + "query": promql, + "start": int(start.timestamp()), + "end": int(when.timestamp()), + "step": STEP_SECONDS, + } + try: + async with httpx.AsyncClient(timeout=HTTP_TIMEOUT) as client: + r = await client.get(f"{VM_URL}/api/v1/query_range", params=params) + r.raise_for_status() + result = r.json().get("data", {}).get("result", []) or [] + except Exception as e: + log.warning("vm tool query failed: %s", e) + return [] + + samples: list[dict] = [] + for series in result: + metric = series.get("metric", {}) + for ts, val in series.get("values", []): + samples.append({"ts": ts, "value": val, "metric": metric}) + return samples diff --git a/apps/ai/ai-analyzer/app/main.py b/apps/ai/ai-analyzer/app/main.py index 065d3c8..b52100f 100644 --- a/apps/ai/ai-analyzer/app/main.py +++ b/apps/ai/ai-analyzer/app/main.py @@ -45,10 +45,16 @@ async def _analyze_alert(alert: dict, group_key: str | None) -> None: ) runbook = runbooks.find(alertname) - # boto3 is sync — offload to a thread so we don't block the loop. - summary = await asyncio.get_event_loop().run_in_executor( - None, bedrock.analyze, alert, loki_lines, vm_samples, runbook - ) + iterations = 0 + if bedrock.AGENT_ENABLED: + summary, iterations = await bedrock.analyze_agentic( + alert, loki_lines, vm_samples, runbook, fingerprint=alert.get("fingerprint"), + ) + else: + # boto3 is sync — offload to a thread so we don't block the loop. + summary = await asyncio.get_event_loop().run_in_executor( + None, bedrock.analyze, alert, loki_lines, vm_samples, runbook + ) result = { "alertname": alertname, @@ -58,6 +64,7 @@ async def _analyze_alert(alert: dict, group_key: str | None) -> None: "loki_lines": len(loki_lines), "vm_samples": len(vm_samples), "runbook_matched": bool(runbook) and not runbook.startswith("# Default runbook"), + "agent_iterations": iterations, "summary": summary, "fallback": summary is None, } diff --git a/apps/ai/ai-analyzer/app/slack.py b/apps/ai/ai-analyzer/app/slack.py index d78a2c2..065a61c 100644 --- a/apps/ai/ai-analyzer/app/slack.py +++ b/apps/ai/ai-analyzer/app/slack.py @@ -107,6 +107,7 @@ def _format_blocks(result: dict) -> list[dict]: loki_n = result.get("loki_lines", 0) vm_n = result.get("vm_samples", 0) runbook_matched = result.get("runbook_matched", False) + iters = result.get("agent_iterations", 0) header_bits = [f"🤖 AI 분석: {alertname}"] if severity: @@ -119,6 +120,8 @@ def _format_blocks(result: dict) -> list[dict]: f"loki_lines={loki_n} · vm_samples={vm_n} · " f"runbook={'✓' if runbook_matched else '✗'}" ) + if iters: + footer += f" · agent_iters={iters}" # Slack section text caps at 3000 chars; truncate with a marker. if len(summary) > 2900: diff --git a/apps/ai/ai-analyzer/chart/values.yaml b/apps/ai/ai-analyzer/chart/values.yaml index 9e03904..db75a93 100644 --- a/apps/ai/ai-analyzer/chart/values.yaml +++ b/apps/ai/ai-analyzer/chart/values.yaml @@ -37,6 +37,14 @@ env: BEDROCK_MODEL_ID: "apac.amazon.nova-pro-v1:0" BEDROCK_MAX_TOKENS: "1024" BEDROCK_TEMPERATURE: "0.2" + # Agentic loop — model can call query_metrics / query_logs / + # list_recent_alerts as tools to dig deeper before concluding. + # Set to "false" to fall back to the single-shot analyze() path. + AGENT_ENABLED: "true" + AGENT_MAX_ITERATIONS: "5" + # Alertmanager endpoint for list_recent_alerts. In-cluster service — + # uses cluster DNS, not Tailscale, so no IP pinning needed. + AM_URL: "http://alertmanager.alertmanager:9093" # Slack channel ID (e.g. C0123456789) — find via the Slack UI or # conversations.list. Empty means "do not post to Slack". SLACK_CHANNEL: "C0B7SHREBAB"