Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions apps/ai/ai-analyzer/app/context/loki.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,6 @@
"k8s_namespace_name",
"k8s_pod_name",
"k8s_node_name",
"cluster",
"namespace",
)


Expand All @@ -44,12 +42,22 @@ def _build_selector(labels: dict[str, str]) -> str:
norm = _dot_to_underscore(k)
if norm in _PASSTHROUGH_LABELS and v:
parts.append(f'{norm}="{v}"')
elif norm == "cluster" and v:
parts.append(f'k8s_cluster_name="{v}"')
if not parts:
# Fall back to the broadest filter so the LLM still gets something.
parts.append('k8s_cluster_name=~".+"')
return "{" + ",".join(parts) + "}"


def _query_from_alert(alert: dict) -> str:
annotations = alert.get("annotations", {}) or {}
query = annotations.get("logql_context_query")
if query:
return query
return _build_selector(alert.get("labels", {}) or {})


def _format_lines(values: Iterable[list[str]]) -> list[str]:
"""Loki query_range returns [[ts_ns, line], ...] under each stream."""
out: list[str] = []
Expand All @@ -76,9 +84,7 @@ async def fetch(alert: dict, when: datetime | None = None) -> list[str]:
start = when - timedelta(minutes=WINDOW_MINUTES)
end = when + timedelta(minutes=WINDOW_MINUTES)

labels = alert.get("labels", {}) or {}
selector = _build_selector(labels)
query = f"{selector}"
query = _query_from_alert(alert)

params = {
"query": query,
Expand Down
78 changes: 63 additions & 15 deletions apps/log-server/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -114,72 +114,120 @@ extraObjects:
data:
rules.yaml: |
groups:
- name: openstack-logs
- name: openstack-nodes-host-logs
interval: 30s
rules:
- alert: HighErrorLogRate
expr: >-
sum by (k8s_cluster_name) (
rate({k8s_cluster_name=~".+"}
rate({k8s_cluster_name=~"pc1|pc2", log_file_name="syslog"}
|~ "(?i)error|fatal|panic" [5m])
) > 2
for: 5m
labels:
severity: warning
cluster: '{{ `{{ $labels.k8s_cluster_name }}` }}'
source: loki
category: log-rate
annotations:
summary: >-
High error log rate on
{{ `{{ $labels.k8s_cluster_name }}` }}
description: >-
Error log lines > 2/sec sustained 5m on
Host syslog error lines > 2/sec sustained 5m on
{{ `{{ $labels.k8s_cluster_name }}` }}.
logql_context_query: >-
{k8s_cluster_name="{{ `{{ $labels.k8s_cluster_name }}` }}",
log_file_name="syslog"}
|~ "(?i)error|fatal|panic"
threshold: 'error|fatal|panic > 2 lines/sec for 5m'

- alert: OOMKillDetected
expr: >-
sum by (k8s_cluster_name) (
count_over_time({log_file_name="syslog"}
count_over_time({k8s_cluster_name=~"pc1|pc2",
log_file_name="syslog"}
|~ "(?i)out of memory|oom-killer|killed process" [2m])
) > 0
for: 0m
labels:
severity: critical
cluster: '{{ `{{ $labels.k8s_cluster_name }}` }}'
source: loki
category: oom
annotations:
summary: >-
OOM kill detected on
{{ `{{ $labels.k8s_cluster_name }}` }}
description: >-
Kernel OOM event in syslog on
{{ `{{ $labels.k8s_cluster_name }}` }}.
logql_context_query: >-
{k8s_cluster_name="{{ `{{ $labels.k8s_cluster_name }}` }}",
log_file_name="syslog"}
|~ "(?i)out of memory|oom-killer|killed process"
threshold: 'OOM signature >= 1 line in 2m'

- alert: AuthFailureSpike
- alert: FilesystemPressureLogDetected
expr: >-
sum by (k8s_cluster_name) (
rate({log_file_name="syslog"}
|~ "(?i)authentication failure|failed password" [5m])
) > 0.1
for: 5m
count_over_time({k8s_cluster_name=~"pc1|pc2",
log_file_name="syslog"}
|~ "(?i)no space left|disk full|read-only file system|filesystem.*full" [5m])
) > 0
for: 0m
labels:
severity: warning
cluster: '{{ `{{ $labels.k8s_cluster_name }}` }}'
source: loki
category: filesystem
annotations:
summary: >-
Auth failure spike on
Filesystem pressure log detected on
{{ `{{ $labels.k8s_cluster_name }}` }}
description: >-
Failed auth attempts > 0.1/sec for 5m on
Host syslog contains filesystem pressure signals on
{{ `{{ $labels.k8s_cluster_name }}` }}.
logql_context_query: >-
{k8s_cluster_name="{{ `{{ $labels.k8s_cluster_name }}` }}",
log_file_name="syslog"}
|~ "(?i)no space left|disk full|read-only file system|filesystem.*full"
threshold: 'filesystem pressure signature >= 1 line in 5m'

- alert: HostSyslogSilent
expr: >-
sum by (k8s_cluster_name) (
rate({log_file_name="syslog",
k8s_cluster_name="pc1"}[5m])
) == 0
absent_over_time({k8s_cluster_name="pc1",
log_file_name="syslog"}[5m])
for: 5m
labels:
severity: warning
cluster: pc1
source: loki
category: log-silence
annotations:
summary: 'pc1 syslog silent for 5m (host stalled?)'
description: >-
No syslog lines from pc1 in 5 minutes. Host or rsyslog
may be down.
logql_context_query: >-
{k8s_cluster_name="pc1", log_file_name="syslog"}
threshold: 'syslog == 0 lines/sec for 5m'

- alert: HostSyslogSilent
expr: >-
absent_over_time({k8s_cluster_name="pc2",
log_file_name="syslog"}[5m])
for: 5m
labels:
severity: warning
cluster: pc2
source: loki
category: log-silence
annotations:
summary: 'pc2 syslog silent for 5m (host stalled?)'
description: >-
No syslog lines from pc2 in 5 minutes. Host or rsyslog
may be down.
logql_context_query: >-
{k8s_cluster_name="pc2", log_file_name="syslog"}
threshold: 'syslog == 0 lines/sec for 5m'
Loading