feat(monitoring): add prometheus-operator monitoring & alerts for n8n, influxdb, and home-assistant

- Enable Prometheus metrics endpoint in n8n, InfluxDB, and Home Assistant.
- Add ServiceMonitor configs for scraping metrics, including token authentication for Home Assistant.
- Define native PrometheusRules for health checks, restarts, volume capacities, and n8n event loop lag to prevent login freezes.
- Deploy AlertmanagerConfigs to route alerts from the new namespaces to Telegram.

k8s/README.md

@@ -365,7 +365,7 @@ helm repo add prometheus-community https://prometheus-community.github.io/helm-c
 helm repo update
 helm upgrade --install --create-namespace prometheus-operator prometheus-community/kube-prometheus-stack -n monitoring -f monitoring/prometheus-operator.secret.yml --version 56.6.1
 # alert configuration
-NAMESPACES_TO_ALERT=( kube-system monitoring cert-manager datalab ingress-nginx mailu minio velero web openebs )
+NAMESPACES_TO_ALERT=( kube-system monitoring cert-manager datalab ingress-nginx mailu minio velero web openebs n8n influxdb home-assistant )
 for i in "${NAMESPACES_TO_ALERT[@]}"; do
   kubectl apply -f monitoring/alertmanagerconfig.secret.yaml -n $i
 done

k8s/n8n/n8n.secret.yml

@@ -44,6 +44,10 @@ main:
         secretKeyRef:
           name: db-app
           key: password
+    N8N_METRICS:
+      value: "true"
+    N8N_METRICS_INCLUDE_DEFAULT_METRICS:
+      value: "true"
   # Mount the CNPG CA Cert into N8N container
   extraVolumeMounts:
     - name: db-ca-cert
@@ -111,4 +115,64 @@ extraManifests:
         limits:
           memory: "512Mi"
       storage:
-        size: 1Gi
+        size: 1Gi
+  - apiVersion: monitoring.coreos.com/v1
+    kind: ServiceMonitor
+    metadata:
+      name: mop-n8n
+      labels:
+        release: prometheus-operator
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/instance: mop-n8n
+          app.kubernetes.io/name: n8n
+      namespaceSelector:
+        matchNames:
+          - n8n
+      endpoints:
+        - port: http
+          interval: 30s
+          path: /metrics
+  - apiVersion: monitoring.coreos.com/v1
+    kind: PrometheusRule
+    metadata:
+      name: mop-n8n-alerts
+      labels:
+        release: prometheus-operator
+    spec:
+      groups:
+        - name: n8n.rules
+          rules:
+            - alert: n8nInstanceDown
+              expr: up{job="mop-n8n"} == 0
+              for: 5m
+              labels:
+                severity: critical
+              annotations:
+                summary: "n8n instance is down or unresponsive"
+                description: "n8n scraper has failed for the last 5 minutes. The application might be frozen or crashed."
+            - alert: n8nPodRestarts
+              expr: rate(kube_pod_container_status_restarts_total{container="n8n"}[15m]) * 900 > 1
+              for: 5m
+              labels:
+                severity: warning
+              annotations:
+                summary: "n8n pod is restarting frequently"
+                description: "n8n has restarted in the last 15 minutes. This might indicate liveness probe failures due to database issues or memory limit exhaustion."
+            - alert: n8nNodeEventLoopLag
+              expr: nodejs_eventloop_lag_seconds{job="mop-n8n"} > 1
+              for: 5m
+              labels:
+                severity: warning
+              annotations:
+                summary: "n8n event loop lag is high"
+                description: "n8n Node.js event loop lag has exceeded 1 second for the last 5 minutes. This can make the UI unresponsive and logins fail."
+            - alert: n8nPodNotReady
+              expr: kube_pod_status_ready{condition="true", pod=~"mop-n8n-.*"} == 0
+              for: 5m
+              labels:
+                severity: critical
+              annotations:
+                summary: "n8n pod is not ready"
+                description: "n8n pod has been in non-ready state for more than 5 minutes. This is likely due to failing readiness probes (/healthz check failing, possibly database connection issues)."