Playbook：多云告警体系合并实战 —— 从 200 条规则混战到统一治理

#!/bin/bash
# alerting-audit.sh — 拉取多集群 PrometheusRule 全量审计
# 用法：./alerting-audit.sh <output-dir>
# 输出：<dir>/rules-<context>.json + summary.csv

set -euo pipefail

OUT="${1:-/tmp/alerting-audit}"
CONTEXTS=("us-prod" "sandbox-staging" "cn-prod")

mkdir -p "$OUT"

command -v kubectl >/dev/null || { echo "需要 kubectl"; exit 1; }
command -v jq      >/dev/null || { echo "需要 jq"; exit 1; }

for ctx in "${CONTEXTS[@]}"; do
  echo "==> 拉取 context=$ctx"
  if ! kubectl --context "$ctx" version --request-timeout=5s >/dev/null 2>&1; then
    echo "    跳过：context $ctx 不可达"
    continue
  fi
  kubectl --context "$ctx" get prometheusrule -A -o json \
    | jq '{context: "'"$ctx"'", rules: [.items[] |
        {namespace: .metadata.namespace,
         name: .metadata.name,
         labels: .metadata.labels,
         groups: .spec.groups}]}' \
    > "$OUT/rules-$ctx.json"
done

# 输出对照表 CSV
python3 - "$OUT" <<'PY'
import json, glob, csv, sys, os
out_dir = sys.argv[1]
rows = []
for f in glob.glob(f"{out_dir}/rules-*.json"):
    data = json.load(open(f))
    ctx = data["context"]
    for r in data["rules"]:
        for g in (r.get("groups") or []):
            for rule in g.get("rules", []):
                if "alert" not in rule:    # 跳过 recording rule
                    continue
                rows.append({
                    "context":  ctx,
                    "ns":       r["namespace"],
                    "group":    g["name"],
                    "alert":    rule["alert"],
                    "severity": (rule.get("labels") or {}).get("severity", ""),
                    "channel":  (rule.get("labels") or {}).get("channel",  ""),
                    "for":      rule.get("for", ""),
                    "expr":     rule["expr"][:120],
                })
with open(f"{out_dir}/summary.csv", "w", newline="") as fp:
    w = csv.DictWriter(fp, fieldnames=list(rows[0].keys()))
    w.writeheader(); w.writerows(rows)
print(f"导出 {len(rows)} 条规则到 {out_dir}/summary.csv")
PY

$ ls /tmp/alerting-audit/
rules-cn-prod.json  rules-sandbox-staging.json  rules-us-prod.json  summary.csv

$ wc -l /tmp/alerting-audit/summary.csv
198 /tmp/alerting-audit/summary.csv

$ head -3 /tmp/alerting-audit/summary.csv
context,ns,group,alert,severity,channel,for,expr
us-prod,monitoring,p0-alerting-rules,ServiceAFiveXXSpike,P0,observe,5m,sum(rate(istio_requests_total{...
us-prod,monitoring,p0-alerting-rules,ContainerOOMKilled,P0,observe,3m,kube_pod_container_status_last_t...

# 业务侧 SQL 轮询规则导出（PG）
# 前置：psql 已装；DB 凭据通过 SSM 隧道，参考运维手册
psql -h "$PG_HOST" -U readonly -d business_monitor -c \
  "\copy (
     select id, name, severity, enabled, channel, expression
     from monitor_rules
     order by severity, name
   ) to '/tmp/alerting-audit/rules-biz.csv' csv header"

#!/bin/bash
# detect-overlap.sh — 用 alert 名规范化后求两两交集
set -euo pipefail
SUM="/tmp/alerting-audit/summary.csv"
BIZ="/tmp/alerting-audit/rules-biz.csv"

python3 - <<PY
import csv, re
def norm(s):
    s = re.sub(r"[_\-\s]", "", s.lower())
    return s

prom = {}
with open("$SUM") as fp:
    for r in csv.DictReader(fp):
        prom.setdefault(norm(r["alert"]), []).append((r["context"], r["alert"]))

biz = {}
with open("$BIZ") as fp:
    for r in csv.DictReader(fp):
        biz.setdefault(norm(r["name"]), []).append(r["name"])

overlap = set(prom) & set(biz)
print(f"重叠规则数：{len(overlap)}")
for k in sorted(overlap):
    print(f"  - prom: {prom[k]}  ⇄  biz: {biz[k]}")
PY

#!/bin/bash
# channel-inventory.sh — 列出所有告警相关 SNS topic + Lambda
set -euo pipefail
REGION="${REGION:-us-west-2}"
OUT="/tmp/channel-inventory"
mkdir -p "$OUT"

# 1. 找所有疑似告警 SNS topic
aws sns list-topics --region "$REGION" \
  | jq -r '.Topics[].TopicArn' | grep -iE 'alert|alarm|notif' \
  > "$OUT/sns-topics.txt"

# 2. 列出每个 topic 的订阅
while IFS= read -r topic; do
  aws sns list-subscriptions-by-topic --topic-arn "$topic" --region "$REGION" \
    | jq --arg t "$topic" '.Subscriptions[]
        | {topic:$t, protocol:.Protocol, endpoint:.Endpoint}'
done < "$OUT/sns-topics.txt" > "$OUT/sns-subs.json"

# 3. 把 Lambda endpoint 抽出来
jq -r 'select(.protocol=="lambda") | .endpoint' "$OUT/sns-subs.json" \
  | sort -u > "$OUT/alert-lambdas.txt"

# 4. 拉每个 Lambda 代码做硬编码扫描
mkdir -p "$OUT/lambda-src"
while IFS= read -r arn; do
  name=$(basename "$arn")
  url=$(aws lambda get-function --function-name "$arn" --region "$REGION" \
        --query 'Code.Location' --output text)
  curl -sL "$url" -o "$OUT/lambda-src/$name.zip"
  unzip -q -o "$OUT/lambda-src/$name.zip" -d "$OUT/lambda-src/$name"
done < "$OUT/alert-lambdas.txt"

# 5. 扫硬编码字符串
echo "==> 硬编码扫描结果"
grep -rEn 'RabbitMQ 告警|Broker: prod|access_token=[a-f0-9]{20,}|hc-ping\.com' \
     "$OUT/lambda-src/" || echo "  未发现硬编码"

$ cat /tmp/channel-inventory/sns-topics.txt
arn:aws:sns:us-west-2:<ACCOUNT_ID>:rabbitmq-prod-alerts
arn:aws:sns:us-west-2:<ACCOUNT_ID>:ops-aws-alerts
arn:aws:sns:us-west-2:<ACCOUNT_ID>:legacy-cw-notifications

$ grep -rEn 'RabbitMQ 告警' /tmp/channel-inventory/lambda-src/
rabbitmq-dingtalk-alert/index.js:42:  const title = "RabbitMQ 告警: " + alarm.AlarmName;
rabbitmq-dingtalk-alert/index.js:51:  const broker = "prod-xxx-rabbitmq-ha-v2 (m5.large)";

#!/bin/bash
# silence-audit.sh — 列出长期 silence + 检测 Watchdog 是否被误杀
# 用法：./silence-audit.sh
# 环境：AM_URL=http://alertmanager.monitoring.svc.cluster.local:9093

set -euo pipefail
AM_URL="${AM_URL:-http://localhost:9093}"

command -v amtool >/dev/null || { echo "需要 amtool"; exit 1; }
command -v jq     >/dev/null || { echo "需要 jq"; exit 1; }

echo "==> [1] 长期 silence (>7 天) 列表"
curl -s "$AM_URL/api/v2/silences" \
  | jq -r '.[] | select(.status.state=="active")
    | select((.endsAt | fromdateiso8601) - (.startsAt | fromdateiso8601) > 604800)
    | "[\(.id)] by=\(.createdBy) end=\(.endsAt) comment=\(.comment)
       matchers=\(.matchers | map("\(.name)\(if .isRegex then "=~" else "=" end)\(.value)") | join(", "))"'

echo "==> [2] 永久 silence (endsAt 距今 >365 天)"
curl -s "$AM_URL/api/v2/silences" \
  | jq -r '.[] | select(.status.state=="active")
    | select((.endsAt | fromdateiso8601) - now > 31536000)
    | "[\(.id)] by=\(.createdBy) end=\(.endsAt) comment=\(.comment)"'

echo "==> [3] 高危 matcher（.+ / .* 全匹配）"
curl -s "$AM_URL/api/v2/silences" \
  | jq -r '.[] | select(.status.state=="active")
    | select(.matchers[] | select(.isRegex and (.value=="" or .value==".+" or .value==".*")))
    | "[\(.id)] by=\(.createdBy) comment=\(.comment)
       matchers=\(.matchers | map("\(.name)\(if .isRegex then "=~" else "=" end)\(.value)"))"'

echo "==> [4] Watchdog 是否被误杀（Watchdog alert 当前是否被任何 silence 命中）"
WD_HIT=$(curl -s "$AM_URL/api/v2/alerts?filter=alertname=Watchdog" \
  | jq '[.[] | select(.status.silencedBy != null and (.status.silencedBy|length) > 0)] | length')
if [[ "$WD_HIT" -gt 0 ]]; then
  echo "  ❌ 警告：Watchdog 当前被 silence 命中！立即检查"
  curl -s "$AM_URL/api/v2/alerts?filter=alertname=Watchdog" \
    | jq '.[] | {labels, status}'
  exit 2
else
  echo "  ✓ Watchdog 未被 silence"
fi

---
apiVersion: batch/v1
kind: CronJob
metadata:
  name: silence-audit
  namespace: monitoring
spec:
  schedule: "0 9 * * 1"  # 每周一 9:00
  concurrencyPolicy: Forbid
  jobTemplate:
    spec:
      template:
        spec:
          restartPolicy: OnFailure
          containers:
          - name: audit
            image: alpine/curl-jq:3.20
            env:
            - name: AM_URL
              value: "http://alertmanager-operated:9093"
            - name: REPORT_WEBHOOK
              valueFrom:
                secretKeyRef:
                  name: dingtalk-report-webhook
                  key: url
            command: ["/bin/sh", "-c"]
            args:
            - |
              set -e
              REPORT=$(curl -s "$AM_URL/api/v2/silences" \
                | jq -r '[.[] | select(.status.state=="active")
                    | select((.endsAt|fromdateiso8601) - (.startsAt|fromdateiso8601) > 604800)
                    | "- by=\(.createdBy) end=\(.endsAt) comment=\(.comment)"] | join("\n")')
              if [ -n "$REPORT" ]; then
                BODY=$(jq -n --arg t "周巡检：长期 silence (>7 天)\n$REPORT" \
                  '{msgtype:"text", text:{content:$t}}')
                curl -s -H 'Content-Type: application/json' \
                  -d "$BODY" "$REPORT_WEBHOOK"
              fi

amtool silence add --alertmanager.url="$AM_URL" \
  alertname!=Watchdog \
  --comment="上线窗口 silence（排除 Watchdog）" \
  --duration=2h \
  --author="$USER"

curl -X POST "$AM_URL/api/v2/silences" \
  -H 'Content-Type: application/json' \
  -d '{
    "matchers": [
      {"name":"alertname","value":"Watchdog","isRegex":false,"isEqual":false}
    ],
    "startsAt": "2026-04-30T09:00:00Z",
    "endsAt":   "2026-04-30T11:00:00Z",
    "createdBy":"alice",
    "comment":  "上线窗口排除 Watchdog 的 silence"
  }'

---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: watchdog-heartbeat
  namespace: monitoring
  labels:
    app.kubernetes.io/name: kube-prometheus-stack
    release: monitoring                # 必须与 Prometheus.spec.ruleSelector 匹配，否则不生效
spec:
  groups:
    - name: watchdog
      interval: 30s
      rules:
        - alert: Watchdog
          expr: vector(1)
          for: 0s
          labels:
            severity: none
            channel: heartbeat
          annotations:
            summary: "Watchdog heartbeat — 永远 firing，无心跳即链路异常"
            description: "如果你看到这条告警 resolved，说明 Alertmanager → 钉钉链路活着但 Prometheus 评估异常。如果 healthchecks.io 30s 收不到，说明链路死了。"

---
# alertmanager-config.yaml — 完整可部署版本
apiVersion: v1
kind: Secret
metadata:
  name: alertmanager-main
  namespace: monitoring
type: Opaque
stringData:
  alertmanager.yaml: |
    global:
      resolve_timeout: 5m
      smtp_from: 'ops@example.aws.com'
      smtp_smarthost: 'email-smtp.us-west-2.amazonaws.com:587'
      smtp_auth_username: 'AKIA<redacted>'
      smtp_auth_password: '<smtp-password>'

    templates:
      - '/etc/alertmanager/config/*.tmpl'

    route:
      receiver: webhook-dingtalk-prod
      group_by: ['alertname', 'cluster', 'namespace']
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 30m
      routes:
        # Watchdog 走独立通道：healthchecks.io
        - matchers:
            - alertname = "Watchdog"
          receiver: dead-mans-switch
          group_wait: 10s
          group_interval: 1m
          repeat_interval: 1m
          continue: false

        # P0 灾难性告警 → 飞书 oncall 群（带 @所有人）
        - matchers:
            - severity = "P0"
          receiver: feishu-oncall
          group_wait: 10s
          repeat_interval: 15m
          continue: true

        # staging / sandbox 流量分到 staging 钉钉群
        - matchers:
            - cluster =~ "sandbox.*|staging.*"
          receiver: webhook-dingtalk-staging
          continue: false

        # 邮件备份给 P0/P1
        - matchers:
            - severity =~ "P0|P1"
          receiver: email-fallback
          continue: true

    inhibit_rules:
      # 集群级故障时抑制具体节点/Pod 告警
      - source_matchers:
          - alertname = "KubeAPIDown"
        target_matchers:
          - severity =~ "warning|info"
        equal: ['cluster']

      # 节点 NotReady 时抑制其上 Pod 的 Restart/Pending 告警
      - source_matchers:
          - alertname = "NodeNotReady"
        target_matchers:
          - alertname =~ "Pod.*Restart|Pod.*Pending"
        equal: ['cluster', 'instance']

      # 高 severity 抑制低 severity 同名告警
      - source_matchers:
          - severity = "P0"
        target_matchers:
          - severity =~ "P1|P2"
        equal: ['alertname', 'cluster', 'namespace']

    receivers:
      - name: dead-mans-switch
        webhook_configs:
          - url: 'https://hc-ping.com/<uuid>'
            send_resolved: false
            max_alerts: 0

      - name: webhook-dingtalk-prod
        webhook_configs:
          - url: 'http://prometheus-webhook-dingtalk:8060/dingtalk/prod/send'
            send_resolved: true
            max_alerts: 10

      - name: webhook-dingtalk-staging
        webhook_configs:
          - url: 'http://prometheus-webhook-dingtalk:8060/dingtalk/staging/send'
            send_resolved: true
            max_alerts: 10

      - name: feishu-oncall
        webhook_configs:
          - url: 'http://feishu-bridge.monitoring:8080/oncall'
            send_resolved: true
            max_alerts: 5

      - name: email-fallback
        email_configs:
          - to: 'oncall@example.aws.com'
            send_resolved: true
            headers:
              Subject: '[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}'

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: webhook-dingtalk-config
  namespace: monitoring
data:
  config.yml: |
    timeout: 5s
    targets:
      prod:
        url: https://oapi.dingtalk.com/robot/send?access_token=<prod-token>
        secret: SEC<prod-secret>
        message:
          title: '{{ template "default.title" . }}'
          text:  '{{ template "default.content" . }}'
      staging:
        url: https://oapi.dingtalk.com/robot/send?access_token=<staging-token>
        secret: SEC<staging-secret>
        message:
          title: '{{ template "default.title" . }}'
          text:  '{{ template "default.content" . }}'
  default.tmpl: |
    {{ define "default.title" }}{{ if eq .Status "firing" }}【告警】{{ else }}【告警恢复】{{ end }}{{ .CommonLabels.alertname }}{{ end }}
    {{ define "default.content" }}
    {{ if eq .Status "firing" }}#### 【告警】{{ else }}#### 【告警恢复】{{ end }}（关键词：告警 / 运维 / 恢复）
    - **集群**：{{ .CommonLabels.cluster | default "n/a" }}
    - **命名空间**：{{ .CommonLabels.namespace | default "n/a" }}
    - **严重度**：{{ .CommonLabels.severity | default "n/a" }}
    - **触发数**：{{ .Alerts.Firing | len }} firing / {{ .Alerts.Resolved | len }} resolved
    {{ range .Alerts -}}
    - {{ .Labels.alertname }}: {{ .Annotations.summary }}
    {{ end -}}
    {{ end }}
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus-webhook-dingtalk
  namespace: monitoring
  labels:
    app: prometheus-webhook-dingtalk
spec:
  replicas: 2
  selector:
    matchLabels:
      app: prometheus-webhook-dingtalk
  template:
    metadata:
      labels:
        app: prometheus-webhook-dingtalk
    spec:
      containers:
        - name: webhook-dingtalk
          image: timonwong/prometheus-webhook-dingtalk:v2.1.0
          args:
            - --config.file=/etc/webhook-dingtalk/config.yml
            - --web.listen-address=:8060
          ports:
            - name: http
              containerPort: 8060
          readinessProbe:
            httpGet:
              path: /health
              port: 8060
            periodSeconds: 5
          livenessProbe:
            httpGet:
              path: /health
              port: 8060
            periodSeconds: 30
          resources:
            requests: { cpu: 50m, memory: 64Mi }
            limits:   { cpu: 500m, memory: 256Mi }
          volumeMounts:
            - name: config
              mountPath: /etc/webhook-dingtalk
      volumes:
        - name: config
          configMap:
            name: webhook-dingtalk-config
---
apiVersion: v1
kind: Service
metadata:
  name: prometheus-webhook-dingtalk
  namespace: monitoring
spec:
  selector:
    app: prometheus-webhook-dingtalk
  ports:
    - name: http
      port: 8060
      targetPort: 8060

#!/bin/bash
# heartbeat-down.sh — healthchecks.io 配置 webhook 调它（grace=2m）
# 在 healthchecks.io check 的 Integration → Webhook 里填：
# POST https://<self-host>/heartbeat-down.sh?status=$CHECK_STATUS
set -euo pipefail
DT_URL="${DT_PAGER_URL:?未配反向告警钉钉 webhook}"
STATUS="${1:-down}"

curl -s -H 'Content-Type: application/json' -X POST "$DT_URL" -d "$(jq -n \
  --arg s "$STATUS" \
  '{msgtype:"markdown", markdown:{
     title:"【告警】Watchdog 心跳丢失",
     text: "## 【告警】Watchdog 心跳丢失（关键词：告警/运维/恢复）\n\n- 状态：\($s)\n- 时间：\(now | strftime(\"%Y-%m-%d %H:%M:%S\"))\n- 含义：Alertmanager → 钉钉主链路异常，立即检查\n- 排查清单：\n  1. kubectl -n monitoring get pod\n  2. amtool silence query --alertmanager.url=$AM_URL\n  3. 检查 prometheus-webhook-dingtalk 日志"
  }}')"

# 1. 规则被加载
curl -s http://prometheus.monitoring:9090/api/v1/rules \
  | jq '.data.groups[] | select(.name=="watchdog") | .rules[].name'
# 期望输出：
# "Watchdog"

# 2. Watchdog 当前在 firing
curl -s 'http://alertmanager.monitoring:9093/api/v2/alerts?filter=alertname=Watchdog' \
  | jq '.[] | {state: .status.state, startsAt}'
# 期望：state="active"

# 3. healthchecks.io 显示 1 分钟前刚收到 ping
# UI：https://healthchecks.io/projects/<id>/checks/

# 4. 主动模拟挂掉：kill 一个 alertmanager pod，2 分钟后应收到反向告警
kubectl -n monitoring delete pod -l app.kubernetes.io/name=alertmanager

kubectl -n monitoring delete prometheusrule watchdog-heartbeat
kubectl -n monitoring delete deploy prometheus-webhook-dingtalk
kubectl -n monitoring delete svc prometheus-webhook-dingtalk
kubectl -n monitoring delete configmap webhook-dingtalk-config
# alertmanager-main Secret 改回旧版（git revert + ArgoCD sync）

#!/usr/bin/env python3
# biz-alert-bridge.py — 业务侧告警 → Alertmanager 桥接
# 部署：作为 K8s Deployment，开 8080 端口
# 业务侧 webhook 配置 → POST http://biz-alert-bridge.monitoring:8080/

import json, os, sys, time
from datetime import datetime, timezone, timedelta
from http.server import BaseHTTPRequestHandler, HTTPServer
import urllib.request

AM_URL = os.environ.get("AM_URL", "http://alertmanager-operated:9093")
RESOLVE_AFTER = int(os.environ.get("RESOLVE_AFTER", "300"))  # 5 min

def to_am_format(biz_alert):
    """业务侧 alert dict → Alertmanager API v2 格式"""
    now = datetime.now(timezone.utc).isoformat()
    end = (datetime.now(timezone.utc) + timedelta(seconds=RESOLVE_AFTER)).isoformat()
    return {
        "labels": {
            "alertname": biz_alert["name"],
            "severity":  biz_alert.get("severity", "P2"),
            "source":    "biz-monitor",
            "rule_id":   str(biz_alert.get("id", "")),
            "channel":   "observe",
        },
        "annotations": {
            "summary":     biz_alert.get("summary", ""),
            "description": biz_alert.get("description", ""),
            "runbook":     biz_alert.get("runbook", ""),
        },
        "startsAt": biz_alert.get("triggered_at", now),
        "endsAt":   end,   # 5 分钟后未续推则自动 resolved
    }

class Handler(BaseHTTPRequestHandler):
    def do_POST(self):
        n = int(self.headers.get("Content-Length", 0))
        try:
            payload = json.loads(self.rfile.read(n))
            alerts = payload if isinstance(payload, list) else [payload]
            am_alerts = [to_am_format(a) for a in alerts]
            req = urllib.request.Request(
                f"{AM_URL}/api/v2/alerts",
                data=json.dumps(am_alerts).encode(),
                headers={"Content-Type": "application/json"},
                method="POST",
            )
            with urllib.request.urlopen(req, timeout=5) as resp:
                self.send_response(resp.status)
                self.end_headers()
                self.wfile.write(resp.read())
        except Exception as e:
            self.send_response(500)
            self.end_headers()
            self.wfile.write(f"err={e}".encode())

if __name__ == "__main__":
    HTTPServer(("0.0.0.0", 8080), Handler).serve_forever()

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: biz-alert-bridge
  namespace: monitoring
spec:
  replicas: 2
  selector:
    matchLabels: { app: biz-alert-bridge }
  template:
    metadata:
      labels: { app: biz-alert-bridge }
    spec:
      containers:
        - name: bridge
          image: python:3.12-alpine
          command: ["python3", "/app/bridge.py"]
          env:
            - name: AM_URL
              value: "http://alertmanager-operated:9093"
            - name: RESOLVE_AFTER
              value: "300"
          ports:
            - containerPort: 8080
          volumeMounts:
            - name: code
              mountPath: /app
      volumes:
        - name: code
          configMap:
            name: biz-alert-bridge-code
---
apiVersion: v1
kind: Service
metadata:
  name: biz-alert-bridge
  namespace: monitoring
spec:
  selector: { app: biz-alert-bridge }
  ports:
    - port: 8080
      targetPort: 8080

#!/bin/bash
# rules-reconcile.sh — git yaml 与集群实际加载对账
set -euo pipefail
PROM="${PROM:-http://prometheus.monitoring:9090}"
GIT_DIR="${GIT_DIR:-./clusters/us-prod/monitoring}"

git_count=$(grep -rh '^[[:space:]]*-[[:space:]]*alert:[[:space:]]' "$GIT_DIR" \
            | wc -l | tr -d ' ')
api_count=$(curl -s "$PROM/api/v1/rules" \
  | jq '[.data.groups[].rules[] | select(.type=="alerting")] | length')

echo "git rules:    $git_count"
echo "loaded rules: $api_count"

if [[ "$git_count" -ne "$api_count" ]]; then
  echo "❌ DRIFT: git=$git_count vs api=$api_count"
  echo "→ 检查 PrometheusRule.metadata.labels 是否匹配 Prometheus.spec.ruleSelector"
  exit 1
fi
echo "✓ rules in sync"

#!/bin/bash
# fix-prometheusalert-url.sh
set -euo pipefail

DD_URL_RAW='https://oapi.dingtalk.com/robot/send?access_token=XXXX'
DD_URL_ENC=$(python3 -c "import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1], safe=''))" "$DD_URL_RAW")
echo "encoded: $DD_URL_ENC"

# 改 alertmanager.yaml receiver 里：
# url: 'http://prometheusalert:8080/prometheusalert?type=dd&tpl=prom-dd&ddurl='"$DD_URL_ENC"

# 验证
curl -s "http://prometheusalert:8080/prometheusalert?type=dd&tpl=prom-dd&ddurl=$DD_URL_ENC" \
  -H 'Content-Type: application/json' \
  -d '{"alerts":[{"labels":{"alertname":"test"},"annotations":{"summary":"test"},"status":"firing"}]}'
# 期望：钉钉群收到 test 消息

# webhook-dingtalk config.yml — 完全没有 URL 嵌套问题
targets:
  prod:
    url: https://oapi.dingtalk.com/robot/send?access_token=XXXX
    secret: SECxxx
    message:
      title: '{{ template "default.title" . }}'
      text:  '{{ template "default.content" . }}'

# alertmanager 引用
receivers:
- name: webhook-dingtalk-prod
  webhook_configs:
  - url: 'http://prometheus-webhook-dingtalk:8060/dingtalk/prod/send'

# 安全 silence 脚本
#!/bin/bash
# safe-silence-all.sh — 全 silence 但保留 Watchdog
set -euo pipefail
AM_URL="${AM_URL:-http://localhost:9093}"
DURATION="${1:-2h}"
COMMENT="${2:?用法: $0 <duration> <comment>}"

amtool silence add --alertmanager.url="$AM_URL" \
  alertname!=Watchdog \
  --comment="$COMMENT (auto-exclude Watchdog)" \
  --duration="$DURATION" \
  --author="$(whoami)"

#!/bin/bash
# fix-aurora-protection.sh
set -euo pipefail
CLUSTER_ID="${1:?aurora cluster id}"

# Aurora 改 cluster 层
aws rds modify-db-cluster \
  --db-cluster-identifier "$CLUSTER_ID" \
  --deletion-protection

# 验证
aws rds describe-db-clusters \
  --db-cluster-identifier "$CLUSTER_ID" \
  --query 'DBClusters[0].DeletionProtection'
# 期望：true

aws rds modify-db-instance \
  --db-instance-identifier <pg-instance-id> \
  --deletion-protection

#!/bin/bash
# delete-argocd-app-safe.sh
set -euo pipefail
APP="${1:?argocd app name}"
NS="${2:-argocd}"

# 1. 检查是否 ApplicationSet 生成
OWNER=$(kubectl -n "$NS" get application "$APP" -o jsonpath='{.metadata.ownerReferences[0].kind}' 2>/dev/null || true)
if [[ "$OWNER" == "ApplicationSet" ]]; then
  APPSET=$(kubectl -n "$NS" get application "$APP" -o jsonpath='{.metadata.ownerReferences[0].name}')
  echo "❌ $APP 由 ApplicationSet=$APPSET 生成，必须改 git"
  echo "→ 改方式："
  echo "   1) 删除 git 仓库中对应 generator 列表项 (如 clusters/<env>/applications/<path>/)"
  echo "   2) 或改 ApplicationSet generator 的 filter 规则"
  echo "   3) 等 ApplicationSet 重新 reconcile，child app 自动消失"
  exit 1
fi

# 2. 不属于 ApplicationSet 才能直接删
echo "✓ $APP 是独立 Application，可以直接删"
argocd app delete "$APP" --yes

#!/bin/bash
# verify-prometheusrule-loaded.sh
set -euo pipefail
NS="${NS:-monitoring}"

# 1. 找 Prometheus 实际的 ruleSelector
SELECTOR=$(kubectl -n "$NS" get prometheus -o jsonpath='{.items[0].spec.ruleSelector}')
echo "Prometheus.spec.ruleSelector = $SELECTOR"

# 2. 列出所有 PrometheusRule + labels
echo "==> 所有 PrometheusRule labels:"
kubectl -n "$NS" get prometheusrule -o json \
  | jq -r '.items[] | "\(.metadata.name): \(.metadata.labels)"'

# 3. 对比 Prometheus 实际加载的规则数 vs PrometheusRule yaml 总条数
yaml_count=$(kubectl -n "$NS" get prometheusrule -o json \
  | jq '[.items[].spec.groups[].rules[] | select(.alert)] | length')
api_count=$(curl -s "http://prometheus.${NS}:9090/api/v1/rules" \
  | jq '[.data.groups[].rules[] | select(.type=="alerting")] | length')

echo "yaml=$yaml_count api=$api_count"
[[ "$yaml_count" -eq "$api_count" ]] || {
  echo "❌ DRIFT — 修 metadata.labels 匹配 ruleSelector 后 kubectl apply"
  exit 1
}

{{ define "default.title" }}{{ if eq .Status "firing" }}【告警】{{ else }}【告警恢复】{{ end }}{{ .CommonLabels.alertname }}{{ end }}
{{ define "default.content" }}
{{ if eq .Status "firing" }}#### 【告警】{{ else }}#### 【告警恢复】{{ end }}（关键词：告警 / 运维 / 恢复）
...
{{ end }}

#!/bin/bash
# verify-dingtalk-keywords.sh — 同时测 firing + resolved 两类消息
set -euo pipefail
WD_URL="${WD_URL:-http://prometheus-webhook-dingtalk:8060/dingtalk/prod/send}"

echo "==> 测 firing"
curl -s -H 'Content-Type: application/json' "$WD_URL" -d '{
  "version":"4","status":"firing","groupKey":"test",
  "commonLabels":{"alertname":"DingtalkKeywordTest","severity":"P2"},
  "alerts":[{"status":"firing","labels":{"alertname":"DingtalkKeywordTest"},
             "annotations":{"summary":"test firing"},"startsAt":"2026-04-30T10:00:00Z"}]
}' | jq

echo "==> 测 resolved"
curl -s -H 'Content-Type: application/json' "$WD_URL" -d '{
  "version":"4","status":"resolved","groupKey":"test",
  "commonLabels":{"alertname":"DingtalkKeywordTest","severity":"P2"},
  "alerts":[{"status":"resolved","labels":{"alertname":"DingtalkKeywordTest"},
             "annotations":{"summary":"test resolved"},
             "startsAt":"2026-04-30T10:00:00Z","endsAt":"2026-04-30T10:05:00Z"}]
}' | jq
# 两次都期望 钉钉 API errcode=0；任一报 310000 → 加关键词

#!/bin/bash
# detect-hardcoded-lambdas.sh
set -euo pipefail
REGION="${REGION:-us-west-2}"
DIR=$(mktemp -d)

aws lambda list-functions --region "$REGION" \
  | jq -r '.Functions[] | select(.FunctionName | test("alert|alarm|notif"; "i")) | .FunctionName' \
  | while read -r fn; do
    url=$(aws lambda get-function --function-name "$fn" --region "$REGION" \
          --query 'Code.Location' --output text)
    curl -sL "$url" -o "$DIR/$fn.zip"
    unzip -q -o "$DIR/$fn.zip" -d "$DIR/$fn"
    BAD=$(grep -rEn 'RabbitMQ 告警|Broker: prod|m5\.large|prod-[a-z-]+-rabbitmq' "$DIR/$fn/" || true)
    if [[ -n "$BAD" ]]; then
      echo "❌ Lambda $fn 含硬编码字段："
      echo "$BAD" | head -5
      echo "→ 行动：禁用该 Lambda 复用，新建独立 SNS + Lambda"
    fi
  done

// generic-cw-to-dingtalk.js — 通用 CloudWatch → 钉钉
exports.handler = async (event) => {
  const sns = JSON.parse(event.Records[0].Sns.Message);
  const namespace = sns.Trigger?.Namespace ?? 'unknown';
  const metric    = sns.Trigger?.MetricName ?? 'unknown';
  const dim       = (sns.Trigger?.Dimensions ?? [])
                      .map(d => `${d.name}=${d.value}`).join(', ');
  const status    = sns.NewStateValue;
  const prefix    = status === 'OK' ? '【告警恢复】' : '【告警】';

  const text =
    `## ${prefix} ${sns.AlarmName}（关键词：告警/运维/恢复）\n` +
    `- 命名空间：${namespace}\n` +
    `- 指标：${metric}\n` +
    `- 维度：${dim}\n` +
    `- 触发原因：${sns.NewStateReason}\n` +
    `- 时间：${sns.StateChangeTime}`;

  const webhook = process.env.DT_WEBHOOK;   // 注入到 env，不在代码里
  const resp = await fetch(webhook, {
    method: 'POST',
    headers: {'Content-Type': 'application/json'},
    body: JSON.stringify({
      msgtype: 'markdown',
      markdown: { title: prefix + sns.AlarmName, text },
    }),
  });
  return { statusCode: resp.status };
};

kubectl -n monitoring rollout restart deploy prometheus-webhook-dingtalk
kubectl -n monitoring rollout status deploy prometheus-webhook-dingtalk --timeout=2m