AI-Powered Kubernetes Monitoring with n8n

apiVersion: v1
kind: Secret
metadata:
  namespace: n8n
  name: postgres-secret
type: Opaque
stringData:
  POSTGRES_USER: postgres-root
  POSTGRES_PASSWORD: postgres-root
  POSTGRES_DB: n8n
  POSTGRES_NON_ROOT_USER: postgres
  POSTGRES_NON_ROOT_PASSWORD: postgres
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: init-data
  namespace: n8n
data:
  init-data.sh: |
    #!/bin/bash
    set -e;
    if [ -n "${POSTGRES_NON_ROOT_USER:-}" ] && [ -n "${POSTGRES_NON_ROOT_PASSWORD:-}" ]; then
    	psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
    		CREATE USER "${POSTGRES_NON_ROOT_USER}" WITH PASSWORD '${POSTGRES_NON_ROOT_PASSWORD}';
    		GRANT ALL PRIVILEGES ON DATABASE ${POSTGRES_DB} TO "${POSTGRES_NON_ROOT_USER}";
    		GRANT ALL ON SCHEMA public TO "${POSTGRES_NON_ROOT_USER}";
    	EOSQL
    else
    	echo "SETUP INFO: No Environment variables given!"
    fi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  labels:
    service: postgres-pv
  name: postgres-pv
  namespace: n8n
spec:
  storageClassName: nfs
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 100Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    service: postgres-n8n
  name: postgres
  namespace: n8n
spec:
  replicas: 1
  selector:
    matchLabels:
      service: postgres-n8n
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      labels:
        service: postgres-n8n
    spec:
      containers:
        - image: postgres:18
          name: postgres
          ports:
            - containerPort: 5432
          env:
            - name: PGDATA
              value: /var/lib/postgresql/data/pgdata
            - name: POSTGRES_USER
              valueFrom:
                secretKeyRef:
                  name: postgres-secret
                  key: POSTGRES_USER
            - name: POSTGRES_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: postgres-secret
                  key: POSTGRES_PASSWORD
            - name: POSTGRES_DB
              value: n8n
            - name: POSTGRES_NON_ROOT_USER
              valueFrom:
                secretKeyRef:
                  name: postgres-secret
                  key: POSTGRES_NON_ROOT_USER
            - name: POSTGRES_NON_ROOT_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: postgres-secret
                  key: POSTGRES_NON_ROOT_PASSWORD
            - name:   POSTGRES_HOST
              value: postgres-service
            - name: POSTGRES_PORT
              value: '5432'
          volumeMounts:
            - name: postgres-pv
              mountPath: /var/lib/postgresql/data
            - name: init-data
              mountPath: /docker-entrypoint-initdb.d/init-n8n-user.sh
              subPath: init-data.sh
      restartPolicy: Always
      volumes:
        - name: postgres-pv
          persistentVolumeClaim:
            claimName: postgres-pv
        - name: postgres-secret
          secret:
            secretName: postgres-secret
        - name: init-data
          configMap:
            name: init-data
            defaultMode: 0744
---
apiVersion: v1
kind: Service
metadata:
  labels:
    service: postgres-n8n
  name: postgres-service
  namespace: n8n
spec:
  ports:
    - name: "5432"
      port: 5432
      targetPort: 5432
      protocol: TCP
  selector:
    service: postgres-n8n
  type: ClusterIP

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  labels:
    service: n8n-claim0
  name: n8n-claim0
  namespace: n8n
spec:
  storageClassName: nfs
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 100Mi
---
apiVersion: v1
kind: Secret
metadata:
  namespace: n8n
  name: n8n-telegram-secret
type: Opaque
stringData:
  TELEGRAM_BOT_TOKEN: "123456789:AAF-your-bot-token-here"
  TELEGRAM_CHAT_ID: "-1001234567890"
---
apiVersion: v1
data:
  N8N_DEFAULT_BINARY_DATA_MODE: filesystem
  N8N_ENCRYPTION_KEY: super-secret-key
  N8N_LISTEN_ADDRESS: 0.0.0.0
  N8N_RUNNERS_ENABLED: "true"
  N8N_USER_MANAGEMENT_JWT_SECRET: even-more-secret
  N8N_SECURE_COOKIE: "true"
  N8N_URL: "https://n8n.local"
  WEBHOOK_URL: "https://n8n.local/"
  N8N_PROXY_HOPS: "1"
  GENERIC_TIMEZONE: "Asia/Singapore"
  N8N_BLOCK_ENV_ACCESS_IN_NODE: "false"
kind: ConfigMap
metadata:
  labels:
    service: n8n-env
  name: n8n-env
  namespace: n8n
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    service: n8n
  name: n8n
  namespace: n8n
spec:
  replicas: 1
  selector:
    matchLabels:
      service: n8n
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        service: n8n
    spec:
      initContainers:
        - name: volume-permissions
          image: busybox:1.36
          command: ["sh", "-c", "chown 1000:1000 /data"]
          volumeMounts:
            - name: n8n-claim0
              mountPath: /data
        - name: copy-tools
          image: seehiong/n8n-tools:latest
          imagePullPolicy: Always
          command: ["sh", "-c", "cp /usr/local/bin/kubectl /tools/kubectl && cp /usr/local/bin/talosctl /tools/talosctl"]
          volumeMounts:
            - name: tools-bin
              mountPath: /tools
      containers:
        # Main n8n container
        - name: n8n
          image: docker.n8n.io/n8nio/n8n:2.11.3
          imagePullPolicy: IfNotPresent
          command:
            - /bin/sh
          args:
            - -c
            - sleep 5; n8n start
          env:
            - name: DB_TYPE
              value: postgresdb
            - name: DB_POSTGRESDB_HOST
              value: postgres-service.n8n.svc.cluster.local
            - name: DB_POSTGRESDB_PORT
              value: "5432"
            - name: DB_POSTGRESDB_DATABASE
              value: n8n
            - name: DB_POSTGRESDB_USER
              valueFrom:
                secretKeyRef:
                  name: postgres-secret
                  key: POSTGRES_NON_ROOT_USER
            - name: DB_POSTGRESDB_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: postgres-secret
                  key: POSTGRES_NON_ROOT_PASSWORD
            - name: N8N_PROTOCOL
              value: https
            - name: N8N_PORT
              value: "5678"
            - name: N8N_RUNNERS_MODE
              value: external
            - name: N8N_RUNNERS_BROKER_LISTEN_ADDRESS
              value: 0.0.0.0
            - name: N8N_RUNNERS_AUTH_TOKEN
              value: my-n8n-runners-secure-token
            - name: N8N_NATIVE_PYTHON_RUNNER
              value: "true"
            - name: N8N_LOG_LEVEL
              value: debug
            - name: N8N_RUNNERS_LAUNCHER_LOG_LEVEL
              value: debug
            - name: NODES_EXCLUDE
              value: "[]"
            - name: TALOSCONFIG
              value: /var/run/secrets/talos/config
            - name: TELEGRAM_BOT_TOKEN
              valueFrom:
                secretKeyRef:
                  name: n8n-telegram-secret
                  key: TELEGRAM_BOT_TOKEN
            - name: TELEGRAM_CHAT_ID
              valueFrom:
                secretKeyRef:
                  name: n8n-telegram-secret
                  key: TELEGRAM_CHAT_ID
          envFrom:
            - configMapRef:
                name: n8n-env
          ports:
            - containerPort: 5678
              protocol: TCP
            - containerPort: 5679  # Task runner broker port
              protocol: TCP
          volumeMounts:
            - mountPath: /home/node/.n8n
              name: n8n-claim0
            - name: tools-bin
              mountPath: /usr/local/bin/kubectl
              subPath: kubectl
              readOnly: true
            - name: tools-bin
              mountPath: /usr/local/bin/talosctl
              subPath: talosctl
              readOnly: true
            - name: talos-config
              mountPath: /var/run/secrets/talos
              readOnly: true
            - name: kube-api
              mountPath: /var/run/secrets/kubernetes.io/serviceaccount
              readOnly: true
        # Tools sidecar with kubectl + talosctl
        - name: tools
          image: seehiong/n8n-tools:latest
          imagePullPolicy: Always
          command: ["sleep", "infinity"]
          env:
            - name: TALOSCONFIG
              value: /var/run/secrets/talos/config
          volumeMounts:
            - name: kube-api
              mountPath: /var/run/secrets/kubernetes.io/serviceaccount
              readOnly: true
            - name: talos-config
              mountPath: /var/run/secrets/talos
              readOnly: true
        # Task runners container
        - name: n8n-runners
          image: n8nio/runners:2.11.3
          imagePullPolicy: Always
          env:
            - name: N8N_RUNNERS_TASK_BROKER_URI
              value: http://localhost:5679
            - name: N8N_RUNNERS_AUTH_TOKEN
              value: my-n8n-runners-secure-token
            - name: N8N_RUNNERS_AUTO_SHUTDOWN_TIMEOUT
              value: "15"

      restartPolicy: Always
      volumes:
        - name: tools-bin
          emptyDir: {}
        - name: n8n-claim0
          persistentVolumeClaim:
            claimName: n8n-claim0
        - name: postgres-secret
          secret:
            secretName: postgres-secret
        - name: talos-config
          secret:
            secretName: talos-config
        - name: kube-api
          projected:
            sources:
            - serviceAccountToken:
                path: token
                expirationSeconds: 3600
            - configMap:
                name: kube-root-ca.crt
                items:
                - key: ca.crt
                  path: ca.crt
            - downwardAPI:
                items:
                - path: namespace
                  fieldRef:
                    fieldPath: metadata.namespace
---
apiVersion: v1
kind: Service
metadata:
  labels:
    service: n8n
  name: n8n
  namespace: n8n
spec:
  type: ClusterIP
  ports:
    - name: "5678"
      port: 80
      targetPort: 5678
      protocol: TCP
  selector:
    service: n8n

You are a Kubernetes diagnostics assistant responsible for checking the health and status of a Kubernetes cluster.
Your job is to investigate cluster issues using the available kubectl diagnostic tool and determine whether workloads are running correctly.
You must follow a structured troubleshooting workflow and return results in a strict JSON format.

# Mandatory First Step
Always begin diagnostics by running this baseline diagnostic bundle:
kubectl get nodes
kubectl get pods -A
kubectl get pvc -A
kubectl get pv
talosctl health --nodes <control-plane-ip>

These commands provide the initial cluster health snapshot and must always be executed before deeper investigation.

# Investigation Workflow
1. Check cluster health
- kubectl get pods -A
- kubectl get deployments -A
- kubectl get nodes

2. Check storage state
- kubectl get pvc -A
- kubectl get pv
- kubectl get storageclass

3. Check cluster events
- kubectl get events -A --sort-by=.metadata.creationTimestamp | tail -n 50

4. Check Talos node health
- talosctl health --nodes <control-plane-ip>
- talosctl service list --nodes <control-plane-ip>
- talosctl dmesg --nodes <control-plane-ip> | tail -n 50

5. If resources are unhealthy, run deeper diagnostics
- kubectl describe pod <pod-name> -n <namespace>
- kubectl describe pvc <pvc-name> -n <namespace>
- kubectl logs <pod-name> -n <namespace>

6. Identify root causes such as:
- CrashLoopBackOff
- Pending pods
- ImagePullBackOff
- FailedScheduling
- FailedMount
- FailedBinding
- Node NotReady
- Storage provisioning failures

7. Summarize findings clearly.

# Tool Usage Rules
Use the Diagnostics Tool for all cluster inspection.
Only run read-only commands, including:

kubectl:
- kubectl get
- kubectl describe
- kubectl logs
- kubectl get events

talosctl:
- talosctl health
- talosctl service list
- talosctl dmesg
- talosctl logs

Never run commands that modify the cluster such as:
- kubectl apply
- kubectl create
- kubectl delete
- kubectl patch
- kubectl scale
- kubectl edit
- kubectl rollout restart

If deeper investigation is needed, use additional kubectl or talosctl read-only commands only.

# Event Interpretation Rule
Kubernetes events may contain historical warnings.
If the current resource state shows healthy status (pods Running, PVC Bound, nodes Ready), the cluster should be considered healthy even if past warning events exist.
Only treat an issue as active if the current resource state still indicates a failure.

# Health Determination
Set cluster_healthy to:
- true → all nodes and workloads are healthy
- false → there are failing pods, nodes, storage issues, or scheduling failures

Examples of unhealthy states:
- pods in CrashLoopBackOff
- pods stuck in Pending
- PVC stuck in Pending
- node NotReady
- repeated restart loops
- failed image pulls
- volume mount failures

Allowed Issue Types
Use only these values for issue_type:
- none
- pod
- deployment
- node
- storage
- networking
- scheduling
- unknown

# Output Format (STRICT)
Return valid JSON only.
Do not include markdown formatting, explanations, or additional text.
The response must exactly follow this structure:

Example when healthy:
{
  "cluster_healthy": true,
  "issues_detected": false,
  "issue_type": "none",
  "summary": "All nodes are ready and all pods are running normally."
}

Example when broken:
{
  "cluster_healthy": false,
  "issues_detected": true,
  "issue_type": "pod",
  "summary": "A deployment in namespace monitoring has two pods stuck in CrashLoopBackOff due to container startup failures."
}

Example storage failure:
{
  "cluster_healthy": false,
  "issues_detected": true,
  "issue_type": "storage",
  "summary": "A PersistentVolumeClaim in namespace 'default' is stuck in Pending because no matching PersistentVolume or StorageClass is available."
}

You are a Kubernetes diagnostics assistant responsible for monitoring and assessing the health of a Kubernetes cluster.

# Health Determination
Set cluster_healthy to:
- true → all nodes and workloads are healthy
- false → there are failing pods, nodes, storage issues, or scheduling failures

Examples of unhealthy states: CrashLoopBackOff, Pending pods, PVC stuck in Pending, node NotReady, repeated restart loops, failed image pulls, volume mount failures.

Allowed issue_type values: none | pod | deployment | node | storage | networking | scheduling | unknown

# Output Format (STRICT)
Return valid JSON only. No markdown, no extra text.
{
  "cluster_healthy": true,
  "issues_detected": false,
  "issue_type": "none",
  "summary": "All nodes are ready and all pods are running normally."
}

# Known False Positives — Do NOT flag these as issues:
- Completed CoreDNS pods typically result from terminated ReplicaSet revisions and are normal during rolling updates.
- kube-system pods with low restart counts (< 5) on a recently provisioned cluster (< 7 days) → expected during initial setup.

# Kubernetes Root Cause Analysis Agent
You are a Kubernetes Site Reliability Engineering (SRE) diagnostics agent responsible for performing root cause analysis (RCA) on incidents detected by the monitoring workflow.

You will receive a health assessment from a monitoring agent. Your goal is to investigate the issue, determine the most likely root cause, and either request human approval before applying a fix or report findings directly if no approval is needed.

# Investigation Responsibilities
1. Validate the issue reported by the monitoring agent.
2. Use the Diagnostics Tool to run additional kubectl or talosctl commands if needed.
3. Identify the root cause of the issue.
4. Propose a specific, targeted remediation command.
5. Set need_approval based on the Remediation rules below.

Focus on common Kubernetes failure scenarios:
- CrashLoopBackOff / ImagePullBackOff
- Pending pods / FailedScheduling
- FailedMount / Node NotReady
- PersistentVolumeClaim Pending / StorageClass mismatch
- Resource exhaustion / Configuration errors

# Tool Usage
Use the Diagnostics Tool to inspect the cluster. Pass full kubectl or talosctl commands as input.
Examples:
- kubectl get pods -A
- kubectl describe pod <pod-name> -n <namespace>
- kubectl get pvc -A
- kubectl get storageclass
- kubectl logs <pod-name> -n <namespace>
- talosctl health --nodes <node-ip>

# Root Cause Determination
Identify the most specific and direct cause of the failure:
- PVC references a non-existent StorageClass
- Pod fails due to ImagePullBackOff
- Node is NotReady due to resource pressure
- Pod cannot schedule due to insufficient CPU
- Volume mount failed due to missing PersistentVolume
- Pod terminated due to node shutdown (DisruptionTarget / exit code 137)

If healthy replacement pods are already Running, set cluster_healthy to true and note the auto-recovery in the summary.

# Severity Levels
Use only: info | warning | critical

# Allowed Issue Types
Use only: none | pod | deployment | node | storage | networking | scheduling | configuration | unknown

# Remediation
Set need_approval to true for any command that modifies cluster state (delete, patch, scale, apply, restart) where the issue is still active and requires intervention.
Set need_approval to false when the issue has already auto-resolved (healthy replacements are running) and any suggested command is optional cleanup only.

When fixing an image error:
- Prefer `kubectl set image deployment/<name> <container>=<image> -n <namespace>` for controller-managed pods.
- Use `kubectl set image pod/<name> <container>=<image> -n <namespace>` only for standalone pods with no controlling Deployment or StatefulSet.
- Before suggesting `kubectl set image pod/<name>`, verify the pod still exists with `kubectl get pod <name> -n <namespace>`. If it no longer exists (e.g. deleted after failure or completed with `restartPolicy: Never`), set `need_approval: false`, leave `suggested_commands` empty, and note in the summary that the pod has already been cleaned up and no action is needed.

# Output Format (STRICT)
IMPORTANT: Do not include reasoning, analysis steps, or explanatory text in your response.
Return valid JSON only. No markdown fences, no explanations, no extra text outside the JSON object.

{
  "cluster_healthy": false,
  "issue_type": "storage",
  "severity": "critical",
  "affected_namespace": "default",
  "affected_resource": "resource-name",
  "root_cause": "clear technical root cause",
  "need_approval": true,
  "fix_applied": false,
  "suggested_commands": "specific remediation command",
  "summary": "concise explanation of the problem"
}

You are a Kubernetes Site Reliability Engineer executing a remediation task.

IMPORTANT RULES:
- Execute ONLY the exact command(s) provided in suggested_commands. Do NOT invent, add, or substitute any other commands.
- Do NOT run diagnostic commands (describe, get, logs, grep, sleep) before or after the fix. Execute the suggested command directly.
- If suggested_commands is empty or says "no action required", do NOT call any tool. Output NO_ACTION_NEEDED immediately.
- Prefer non-destructive fixes.
- Do not delete workloads unless absolutely necessary.
- If the issue has already auto-resolved, confirm that and take no action.

IMPORTANT OUTPUT FORMAT (STRICT):
After executing (or deciding not to execute), return exactly 3 lines of plain text. No markdown, no bullet points, no tool-call syntax, no extra lines.

Status: [RESOLVED | ACTION_TAKEN | FAILED | NO_ACTION_NEEDED]
Action: [what command was run, or why no action was taken — max 20 words]
Result: [current state of the affected resource — max 15 words]