Playbook：K8s 集群三合一实战——QA / PRE / AI Sandbox 合并的完整可执行手册

mkdir -p ~/cluster-merge-report
cat > ~/cluster-merge-report/collect-usage.py <<'PYEOF'
#!/usr/bin/env python3
# collect-usage.py - 拉三个集群的节点 / Pod / 资源使用量
# 用法：./collect-usage.py <ctx_qa> <ctx_pre> <ctx_ai>
import json
import subprocess
import sys
from pathlib import Path

if len(sys.argv) != 4:
    print("用法：collect-usage.py <ctx_qa> <ctx_pre> <ctx_ai>")
    sys.exit(1)

CTXS = {"qa": sys.argv[1], "pre": sys.argv[2], "ai": sys.argv[3]}
OUT = Path.home() / "cluster-merge-report"
OUT.mkdir(parents=True, exist_ok=True)


def kc(ctx, *args):
    cmd = ["kubectl", "--context", ctx] + list(args)
    return subprocess.run(cmd, capture_output=True, text=True, check=True).stdout


for env, ctx in CTXS.items():
    print(f"==> 收集 {env} (context={ctx})")
    nodes = json.loads(kc(ctx, "get", "nodes", "-o", "json"))
    pods = json.loads(kc(ctx, "get", "pods", "-A", "-o", "json"))

    summary = {
        "env": env,
        "node_count": len(nodes["items"]),
        "node_instance_types": {},
        "pod_count_by_ns": {},
        "cpu_request_total_milli": 0,
        "mem_request_total_mi": 0,
    }
    for n in nodes["items"]:
        it = n["metadata"]["labels"].get("node.kubernetes.io/instance-type", "unknown")
        summary["node_instance_types"][it] = summary["node_instance_types"].get(it, 0) + 1

    for p in pods["items"]:
        ns = p["metadata"]["namespace"]
        summary["pod_count_by_ns"][ns] = summary["pod_count_by_ns"].get(ns, 0) + 1
        for c in p["spec"].get("containers", []):
            req = c.get("resources", {}).get("requests", {})
            cpu = req.get("cpu", "0")
            mem = req.get("memory", "0")
            # 简化解析：只处理 m/Mi 单位
            if cpu.endswith("m"):
                summary["cpu_request_total_milli"] += int(cpu[:-1])
            elif cpu and cpu != "0":
                summary["cpu_request_total_milli"] += int(float(cpu) * 1000)
            if mem.endswith("Mi"):
                summary["mem_request_total_mi"] += int(mem[:-2])
            elif mem.endswith("Gi"):
                summary["mem_request_total_mi"] += int(mem[:-2]) * 1024

    out_file = OUT / f"usage-{env}.json"
    out_file.write_text(json.dumps(summary, indent=2))
    print(f"    写入 {out_file}")
    print(f"    nodes={summary['node_count']} pods={sum(summary['pod_count_by_ns'].values())} "
          f"cpu_req={summary['cpu_request_total_milli']}m mem_req={summary['mem_request_total_mi']}Mi")
PYEOF
chmod +x ~/cluster-merge-report/collect-usage.py
~/cluster-merge-report/collect-usage.py eks-qa eks-pre eks-ai

ls -la ~/cluster-merge-report/usage-*.json
# 期望看到 usage-qa.json / usage-pre.json / usage-ai.json
jq '.node_count, .pod_count_by_ns | length' ~/cluster-merge-report/usage-qa.json

cat > ~/cluster-merge-report/list-middleware.sh <<'BASH'
#!/bin/bash
# list-middleware.sh - 扫三个环境的 Nacos / ConfigMap / Secret 找中间件 endpoint
set -euo pipefail
OUT=~/cluster-merge-report/middleware.tsv
echo -e "env\tservice\ttype\tendpoint" > "$OUT"

for env in qa pre ai; do
  ctx="eks-$env"
  # 扫 ConfigMap 里的 endpoint 关键字
  kubectl --context "$ctx" get configmap -A -o json | \
    jq -r --arg env "$env" '
      .items[] |
      select(.data != null) |
      .metadata as $m |
      .data | to_entries[] |
      select(.value | tostring | test("rds|rabbitmq|kafka|redis|valkey|aurora"; "i")) |
      [$env, $m.namespace + "/" + $m.name, .key, (.value | tostring | .[0:120])] |
      @tsv' >> "$OUT" || true
done

echo "==> 输出 $OUT，共 $(wc -l < "$OUT") 行"
echo "==> 重点关注 endpoint 在多个 env 行里出现 = 共享中间件"
sort -k4 "$OUT" | awk -F'\t' '{print $4}' | sort | uniq -c | sort -rn | head -20
BASH
chmod +x ~/cluster-merge-report/list-middleware.sh
~/cluster-merge-report/list-middleware.sh

-- 在每个环境的 RDS 上跑一次，对比 max(id) 与 min(id)
-- 重点关注 "新环境 max(id)" 是否落在 "老环境历史 id" 区间内
SELECT
    'qa' AS env,
    'm_project' AS table_name,
    MIN(id) AS min_id,
    MAX(id) AS max_id,
    COUNT(*) AS rows
FROM service_a_qa.m_project
UNION ALL
SELECT 'pre', 'm_project', MIN(id), MAX(id), COUNT(*) FROM service_a_pre.m_project
UNION ALL
SELECT 'ai',  'm_project', MIN(id), MAX(id), COUNT(*) FROM service_a_ai.m_project;

---
apiVersion: karpenter.k8s.aws/v1
kind: EC2NodeClass
metadata:
  name: gvisor-pre
spec:
  amiFamily: AL2
  amiSelectorTerms:
    - alias: al2@latest
  subnetSelectorTerms:
    - tags:
        karpenter.sh/discovery: "merged-cluster"
        Tier: private
  securityGroupSelectorTerms:
    - tags:
        karpenter.sh/discovery: "merged-cluster"
  role: "KarpenterNodeRole-merged-cluster"
  tags:
    Environment: pre
    NodePool: gvisor-pre
    karpenter.sh/discovery: "merged-cluster"
  blockDeviceMappings:
    - deviceName: /dev/xvda
      ebs:
        volumeType: gp3
        volumeSize: 100Gi
        encrypted: true
        deleteOnTermination: true
  userData: |
    #!/bin/bash
    set -euo pipefail

    # 挂载本环境独占 EFS（每个环境独立 EFS，禁止跨 VPC 复用，详见踩坑 4）
    EFS_DNS="fs-054af543525c9e06a.efs.ap-southeast-1.amazonaws.com"
    mkdir -p /mnt/user
    cat >> /etc/fstab <<FSEOF
    ${EFS_DNS}:/ /mnt/user nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,_netdev 0 0
    FSEOF

    # 防 mnt-user.mount hard-hang（已知坑：默认 30s timeout 不够，调到 180s）
    mkdir -p /etc/systemd/system/mnt-user.mount.d
    cat > /etc/systemd/system/mnt-user.mount.d/override.conf <<UNITEOF
    [Mount]
    TimeoutSec=180
    UNITEOF
    systemctl daemon-reload
    mount -a || true

---
apiVersion: karpenter.sh/v1
kind: NodePool
metadata:
  name: gvisor-pre
spec:
  template:
    metadata:
      labels:
        sandbox.env: pre
        sandbox.gvisor/enabled: "true"
    spec:
      nodeClassRef:
        group: karpenter.k8s.aws
        kind: EC2NodeClass
        name: gvisor-pre
      taints:
        - key: sandbox.env
          value: pre
          effect: NoSchedule
      requirements:
        - key: kubernetes.io/arch
          operator: In
          values: ["amd64"]
        - key: karpenter.sh/capacity-type
          operator: In
          values: ["on-demand"]
        - key: node.kubernetes.io/instance-type
          operator: In
          values: ["m6i.xlarge", "m6i.2xlarge", "m6a.xlarge", "m6a.2xlarge"]
        - key: topology.kubernetes.io/zone
          operator: In
          values: ["ap-southeast-1a", "ap-southeast-1b"]
      expireAfter: 720h
  limits:
    cpu: 16
    memory: 64Gi
  disruption:
    consolidationPolicy: WhenEmpty
    consolidateAfter: 10m

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: service-foo
  namespace: app-pre
spec:
  replicas: 2
  selector:
    matchLabels:
      app: service-foo
  template:
    metadata:
      labels:
        app: service-foo
        sandbox.env: pre
    spec:
      tolerations:
        - key: sandbox.env
          operator: Equal
          value: pre
          effect: NoSchedule
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: sandbox.env
                    operator: In
                    values: ["pre"]
      containers:
        - name: app
          image: <ACCOUNT_ID>.dkr.ecr.ap-southeast-1.amazonaws.com/service-foo:v1.2.3
          resources:
            requests:
              cpu: 200m
              memory: 256Mi
            limits:
              cpu: 1000m
              memory: 1Gi

# ❌ 错误示范：nodeSelector 写宽泛 label，会同时匹配 qa / pre / ai 三个 NodePool
spec:
  nodeSelector:
    sandbox.gvisor/enabled: "true"   # 这个 label 三个 NodePool 都有
  # 没有 toleration 时调度直接失败；
  # 加了通配 toleration 时业务 Pod 可能跑到 qa 节点上消费 pre 数据 → 灾难
  tolerations:
    - operator: Exists                # ❌ 容忍所有 taint，相当于裸奔

kubectl --context merged get nodes -L sandbox.env
# 期望：每个节点 SANDBOX.ENV 列分别显示 qa / pre / ai
kubectl --context merged get pod -n app-pre -o wide | awk '{print $7}' | xargs -I{} kubectl --context merged get node {} -L sandbox.env --no-headers
# 期望：所有 pod 所在节点的 sandbox.env 都是 pre

kubectl --context merged delete nodepool gvisor-pre
kubectl --context merged delete ec2nodeclass gvisor-pre
# Karpenter 会自动回收节点，业务 Pod 转 Pending 直到换 NodePool

---
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: default-deny-cross-ns
  namespace: app-pre
spec:
  endpointSelector: {}        # 匹配 ns 内所有 endpoint
  ingress:
    # 仅允许同 ns 流量
    - fromEndpoints:
        - matchLabels:
            k8s:io.kubernetes.pod.namespace: app-pre
    # 允许 infra ns（监控 / DNS 探针）
    - fromEndpoints:
        - matchLabels:
            k8s:io.kubernetes.pod.namespace: infra
  egress:
    # 允许同 ns
    - toEndpoints:
        - matchLabels:
            k8s:io.kubernetes.pod.namespace: app-pre
    # 允许 kube-system 的 CoreDNS（DNS 解析必备）
    - toEndpoints:
        - matchLabels:
            k8s:io.kubernetes.pod.namespace: kube-system
            k8s-app: kube-dns
      toPorts:
        - ports:
            - port: "53"
              protocol: UDP
            - port: "53"
              protocol: TCP
    # 允许 infra ns（向 Prom 上报 metrics）
    - toEndpoints:
        - matchLabels:
            k8s:io.kubernetes.pod.namespace: infra
    # 允许出站到外部 RDS / Kafka / Valkey（按 CIDR 白名单）
    - toCIDRSet:
        - cidr: 10.51.0.0/16     # pre 环境中间件 VPC

---
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: service-foo-allow-bar
  namespace: app-pre
spec:
  endpointSelector:
    matchLabels:
      app: service-foo
  ingress:
    # 仅允许 service-bar 调 service-foo 的 8080
    - fromEndpoints:
        - matchLabels:
            app: service-bar
            k8s:io.kubernetes.pod.namespace: app-pre
      toPorts:
        - ports:
            - port: "8080"
              protocol: TCP

# pre 的 Pod 不应该能 ping 通 ai 的 Service
kubectl --context merged exec -n app-pre deploy/service-foo -- \
  curl -m 3 http://service-foo.app-ai.svc.cluster.local:8080/healthz
# 期望：超时或 connection refused
# Hubble 看流量被 deny 的明细
hubble observe --namespace app-pre --verdict DROPPED --last 50

kubectl --context merged delete cnp default-deny-cross-ns -n app-pre
# 立即恢复跨 ns 默认放行

#!/bin/bash
# create-aurora-pre.sh - 给 pre 环境建独立 Aurora MySQL cluster
set -euo pipefail

REGION=ap-southeast-1
CLUSTER_ID=service_a-pre-aurora
SG_ID=sg-xxxxxxxxxxxxx
SUBNET_GROUP=service_a-pre-db-subnet
MASTER_USER=admin
MASTER_PASS_SECRET_ARN=arn:aws:secretsmanager:${REGION}:<ACCOUNT_ID>:secret:rds-pre-master

# 1. 建 cluster
aws rds create-db-cluster \
  --region "$REGION" \
  --db-cluster-identifier "$CLUSTER_ID" \
  --engine aurora-mysql \
  --engine-version 8.0.mysql_aurora.3.06.0 \
  --master-username "$MASTER_USER" \
  --manage-master-user-password \
  --master-user-secret-kms-key-id alias/aws/secretsmanager \
  --vpc-security-group-ids "$SG_ID" \
  --db-subnet-group-name "$SUBNET_GROUP" \
  --backup-retention-period 7 \
  --storage-encrypted \
  --tags Key=Environment,Value=pre

# 2. 建 instance
aws rds create-db-instance \
  --region "$REGION" \
  --db-cluster-identifier "$CLUSTER_ID" \
  --db-instance-identifier "${CLUSTER_ID}-instance-1" \
  --db-instance-class db.t4g.medium \
  --engine aurora-mysql

# 3. 等待就绪并打印 endpoint
aws rds wait db-instance-available \
  --region "$REGION" \
  --db-instance-identifier "${CLUSTER_ID}-instance-1"

aws rds describe-db-clusters \
  --region "$REGION" \
  --db-cluster-identifier "$CLUSTER_ID" \
  --query 'DBClusters[0].{Endpoint:Endpoint,Reader:ReaderEndpoint}' \
  --output table

mysql -h <CLUSTER_ID>.cluster-xxx.ap-southeast-1.rds.amazonaws.com -u admin -p \
  -e "SELECT @@version, @@hostname;"
# 期望返回 8.0.x 版本

aws rds delete-db-instance --db-instance-identifier service_a-pre-aurora-instance-1 --skip-final-snapshot
aws rds delete-db-cluster --db-cluster-identifier service_a-pre-aurora --skip-final-snapshot

#!/bin/bash
# kafka-topic-rename.sh - 给共享 MSK 集群的 topic 加环境前缀
# 前置：本机已装 kafka-topics.sh，KAFKA_BOOTSTRAP 已 export
set -euo pipefail

ENV="${1:-}"
[[ -z "$ENV" ]] && { echo "用法：$0 <qa|pre|ai>"; exit 1; }

KAFKA_BOOTSTRAP="${KAFKA_BOOTSTRAP:?需要 export KAFKA_BOOTSTRAP=broker:9092}"

# 1. 列现有 topic
existing=$(kafka-topics.sh --bootstrap-server "$KAFKA_BOOTSTRAP" --list)

# 2. 找没有前缀的 topic（潜在共享风险）
echo "==> 没有环境前缀的 topic（需要重命名或迁移）："
echo "$existing" | grep -vE '^(qa|pre|ai)\.' || echo "（无）"

# 3. 给本环境 topic 建带前缀的 + 6 partitions + replication=3
for raw in agent-msg dispatch-event meter-stat; do
  new="${ENV}.${raw}"
  if echo "$existing" | grep -q "^${new}$"; then
    echo "  ${new} 已存在，跳过"
  else
    kafka-topics.sh --bootstrap-server "$KAFKA_BOOTSTRAP" \
      --create --topic "$new" \
      --partitions 6 --replication-factor 3 \
      --config retention.ms=604800000
    echo "  建好 ${new}"
  fi
done

# 4. 检查 consumer group 是否带前缀
echo "==> consumer group 前缀检查："
kafka-consumer-groups.sh --bootstrap-server "$KAFKA_BOOTSTRAP" --list | \
  grep -vE "^${ENV}\." | grep -E '^(qa|pre|ai)\.' && \
  echo "  ⚠️ 发现跨环境 consumer group，业务代码 group.id 必须带 ${ENV}. 前缀"

package cache

import (
    "fmt"
    "os"
    "strings"

    "github.com/redis/go-redis/v9"
)

// envPrefix 从环境变量读，启动时强制校验
func envPrefix() string {
    p := os.Getenv("CACHE_KEY_PREFIX")
    if p == "" || !strings.HasSuffix(p, ":") {
        panic("CACHE_KEY_PREFIX must be set and end with ':' (e.g. 'pre:')")
    }
    return p
}

// 统一的 client wrapper，所有 Get/Set 强制加前缀
type SafeClient struct {
    cli    *redis.Client
    prefix string
}

func NewSafeClient(cli *redis.Client) *SafeClient {
    return &SafeClient{cli: cli, prefix: envPrefix()}
}

func (s *SafeClient) Set(ctx context.Context, key string, val any, ttl time.Duration) error {
    return s.cli.Set(ctx, s.prefix+key, val, ttl).Err()
}

func (s *SafeClient) Get(ctx context.Context, key string) (string, error) {
    return s.cli.Get(ctx, s.prefix+key).Result()
}

// 启动检查：扫一遍 Redis，发现没前缀的旧 key 直接 panic
func (s *SafeClient) AssertNoLegacyKeys(ctx context.Context) error {
    iter := s.cli.Scan(ctx, 0, "*", 1000).Iterator()
    for iter.Next(ctx) {
        k := iter.Val()
        if !strings.HasPrefix(k, s.prefix) {
            return fmt.Errorf("legacy key without prefix found: %s (env=%s)", k, s.prefix)
        }
    }
    return iter.Err()
}

#!/bin/bash
# nacos-prefix-audit.sh - 扫所有服务的 Nacos 配置，确认 cache.key_prefix 字段存在
set -euo pipefail

NACOS_ADDR=${NACOS_ADDR:?need NACOS_ADDR}
NAMESPACE=$1     # us-pre / us-qa / us-ai
EXPECTED_PREFIX=$2   # pre: / qa: / ai:

services=$(curl -s "${NACOS_ADDR}/nacos/v1/cs/configs?dataId=&group=&search=blur&pageNo=1&pageSize=200&tenant=${NAMESPACE}" | \
  jq -r '.pageItems[].dataId')

for svc in $services; do
  cfg=$(curl -s "${NACOS_ADDR}/nacos/v1/cs/configs?dataId=${svc}&group=DEFAULT_GROUP&tenant=${NAMESPACE}")
  if ! echo "$cfg" | grep -q "key_prefix.*${EXPECTED_PREFIX}"; then
    echo "❌ ${svc} 缺少 cache.key_prefix=${EXPECTED_PREFIX}"
  fi
done

-- 在 AI 环境（id 起点低的那个）执行
-- 把后续新行的 id 起点提到 10000000，与 QA 历史 id 区间错开
USE service_a_ai;

-- 1. 备份当前 max(id)
SELECT 'before', MAX(id), AUTO_INCREMENT
  FROM information_schema.TABLES
  JOIN service_a_ai.m_project ON TABLE_NAME='m_project'
  WHERE TABLE_SCHEMA='service_a_ai' AND TABLE_NAME='m_project';

-- 2. 抬高 AUTO_INCREMENT
ALTER TABLE service_a_ai.m_project AUTO_INCREMENT = 10000000;
ALTER TABLE service_a_ai.m_realtime_messages AUTO_INCREMENT = 10000000;
ALTER TABLE service_a_ai.m_fastagent_messages AUTO_INCREMENT = 10000000;
ALTER TABLE service_a_ai.m_context_messages AUTO_INCREMENT = 10000000;

-- 3. 验证
SELECT TABLE_NAME, AUTO_INCREMENT
  FROM information_schema.TABLES
  WHERE TABLE_SCHEMA='service_a_ai'
    AND TABLE_NAME IN ('m_project','m_realtime_messages','m_fastagent_messages','m_context_messages');
-- 期望：AUTO_INCREMENT >= 10000000

-- 4. 插入一行测试
INSERT INTO service_a_ai.m_project (user_id, name, created_at)
  VALUES (99, '__id_check__', NOW());
SELECT id FROM service_a_ai.m_project WHERE name='__id_check__';
-- 期望：id >= 10000000
DELETE FROM service_a_ai.m_project WHERE name='__id_check__' LIMIT 1;

-- 给所有跨环境消息表加 dispatch_env 字段，作为防御层
-- 即使 broker 撞了，消费端也能按 dispatch_env 过滤

ALTER TABLE service_a_ai.m_realtime_messages
  ADD COLUMN dispatch_env VARCHAR(16) NOT NULL DEFAULT 'ai' AFTER project_id,
  ADD INDEX idx_dispatch_env (dispatch_env);

ALTER TABLE service_a_ai.m_fastagent_messages
  ADD COLUMN dispatch_env VARCHAR(16) NOT NULL DEFAULT 'ai' AFTER project_id,
  ADD INDEX idx_dispatch_env (dispatch_env);

-- QA 环境同步加，默认值 'qa'
ALTER TABLE service_a_qa.m_realtime_messages
  ADD COLUMN dispatch_env VARCHAR(16) NOT NULL DEFAULT 'qa' AFTER project_id,
  ADD INDEX idx_dispatch_env (dispatch_env);

-- 验证
SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, COLUMN_DEFAULT, IS_NULLABLE
  FROM information_schema.COLUMNS
  WHERE COLUMN_NAME='dispatch_env';
-- 期望：IS_NULLABLE=NO，COLUMN_DEFAULT 是 'ai' 或 'qa'

-- 老数据没有 dispatch_env，先用 created_at 推断
-- 业务上线前的历史消息全部回填为本环境标识
UPDATE service_a_ai.m_realtime_messages
   SET dispatch_env = 'ai'
   WHERE dispatch_env = '' OR dispatch_env IS NULL;
-- 影响行数应等于历史总行数

UPDATE service_a_qa.m_realtime_messages
   SET dispatch_env = 'qa'
   WHERE dispatch_env = '' OR dispatch_env IS NULL;

-- 验证：每个表 dispatch_env 唯一值数量 = 1
SELECT dispatch_env, COUNT(*) FROM service_a_ai.m_realtime_messages GROUP BY dispatch_env;
SELECT dispatch_env, COUNT(*) FROM service_a_qa.m_realtime_messages GROUP BY dispatch_env;

# 1. 在新集群建 namespace
kubectl --context merged apply -f - <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
  name: app-sandbox-pre
  labels:
    sandbox.env: pre
    pod-security.kubernetes.io/enforce: baseline
EOF

# 2. 应用 GitOps overlay
kubectl --context merged apply -f gitops/clusters/merged/applications/sandbox-pre/

# 3. 等业务 Pod 全 Running
kubectl --context merged wait --for=condition=Available --timeout=600s \
  -n app-sandbox-pre deploy --all

# 删 Application 触发资源回收
kubectl --context merged delete -f gitops/clusters/merged/applications/sandbox-pre/
# DNS 切回旧集群
aws route53 change-resource-record-sets --hosted-zone-id ZXXX --change-batch file://rollback-dns.json

---
# 用 Argo Rollouts 做 canary
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: service-foo
  namespace: app-qa
spec:
  replicas: 5
  strategy:
    canary:
      steps:
        - setWeight: 10
        - pause: { duration: 10m }
        - setWeight: 30
        - pause: { duration: 30m }
        - setWeight: 60
        - pause: { duration: 1h }
        - setWeight: 100
      analysis:
        templates:
          - templateName: error-rate
        startingStep: 1
        args:
          - name: service-name
            value: service-foo
  selector:
    matchLabels:
      app: service-foo
  template:
    metadata:
      labels:
        app: service-foo
    spec:
      tolerations:
        - key: sandbox.env
          operator: Equal
          value: qa
          effect: NoSchedule
      containers:
        - name: app
          image: <ACCOUNT_ID>.dkr.ecr.ap-southeast-1.amazonaws.com/service-foo:v1.2.3

#!/bin/bash
# pre-cutover.sh - PRE 集群切换，预计 T-0 停机 30-60 秒
set -euo pipefail

# T-30min: 把旧集群业务流量降到 30%（DNS TTL 已改 60s）
# T-10min: 应用层 readonly
kubectl --context old-pre annotate deploy -n app-pre service-foo readonly=true --overwrite

# T-5min: rsync EFS 增量
rsync -avz --delete /mnt/old-efs/usersandbox-pvc-/ /mnt/new-efs/usersandbox-pvc-/

# T-0: DNS 切换
aws route53 change-resource-record-sets --hosted-zone-id ZXXX --change-batch file://cutover-dns.json

# T+1min: 新集群业务起来
kubectl --context merged scale -n app-pre deploy/service-foo --replicas=5

# T+5min: 回写权限放开
kubectl --context merged annotate deploy -n app-pre service-foo readonly-

echo "==> Cutover 完成，开始 1-2 周观察期"

# DNS 切回旧 NLB
aws route53 change-resource-record-sets --hosted-zone-id ZXXX --change-batch file://rollback-dns.json
# 旧集群业务恢复
kubectl --context old-pre scale deploy -n app-pre service-foo --replicas=5

# 1. 合并完成后立即把旧集群 nodegroup 缩到 0（control plane 保留）
aws eks update-nodegroup-config \
  --cluster-name old-pre \
  --nodegroup-name app \
  --scaling-config minSize=0,maxSize=0,desiredSize=0 \
  --region ap-southeast-1

# 2. 14 天观察期内每天检查
for day in $(seq 1 14); do
  date
  kubectl --context merged get pod -n app-pre | grep -v Running && echo "⚠️ 有非 Running Pod" || echo "OK"
  # 抓客户工单 / Sentry 错误率
done

# 3. 观察期通过后下线旧集群
aws eks delete-nodegroup --cluster-name old-pre --nodegroup-name app
aws eks delete-cluster --name old-pre
# 旧 RDS / Valkey 留快照后再删
aws rds delete-db-cluster --db-cluster-identifier old-pre-aurora \
  --final-db-snapshot-identifier old-pre-aurora-final-$(date +%Y%m%d)

-- Step 1：备份脏数据（CTAS）
CREATE TABLE service_a_qa.m_realtime_messages_bak_20260418 AS
SELECT * FROM service_a_qa.m_realtime_messages
 WHERE project_id BETWEEN 1 AND 274
   AND created_at > '2026-03-25 00:00:00'   -- AI 环境上线日
   AND project_id IN (
       SELECT id FROM service_a_qa.m_project
        WHERE created_at < '2026-03-25 00:00:00'
   );

-- 备份表加主键，便于后续中转 DELETE
ALTER TABLE service_a_qa.m_realtime_messages_bak_20260418
  ADD PRIMARY KEY (id);

SELECT COUNT(*) AS dirty_rows FROM service_a_qa.m_realtime_messages_bak_20260418;
-- 期望：约 91,403 行

-- Step 2：临时表中转，避免一次大事务锁表
CREATE TABLE service_a_qa.dirty_ids_tmp (id BIGINT PRIMARY KEY);
INSERT INTO service_a_qa.dirty_ids_tmp (id)
  SELECT id FROM service_a_qa.m_realtime_messages_bak_20260418;

-- Step 3：分批 DELETE（每批 1000 行，避免长事务）
DELIMITER $$
CREATE PROCEDURE service_a_qa.cleanup_dirty()
BEGIN
  DECLARE done INT DEFAULT 0;
  REPEAT
    DELETE FROM service_a_qa.m_realtime_messages
     WHERE id IN (SELECT id FROM service_a_qa.dirty_ids_tmp LIMIT 1000);
    SET done = ROW_COUNT();
    DELETE FROM service_a_qa.dirty_ids_tmp LIMIT 1000;
    DO SLEEP(0.1);   -- 让从库追上
  UNTIL done = 0 END REPEAT;
END$$
DELIMITER ;

CALL service_a_qa.cleanup_dirty();
DROP PROCEDURE service_a_qa.cleanup_dirty;
DROP TABLE service_a_qa.dirty_ids_tmp;

-- Step 4：验证清干净
SELECT COUNT(*) FROM service_a_qa.m_realtime_messages
 WHERE project_id BETWEEN 1 AND 274
   AND created_at > '2026-03-25 00:00:00'
   AND project_id IN (
       SELECT id FROM service_a_qa.m_project
        WHERE created_at < '2026-03-25 00:00:00'
   );
-- 期望：0

# 1. 起独立 RabbitMQ broker（约 71 USD/月）
aws mq create-broker \
  --broker-name ai-service_a-rabbitmq \
  --engine-type RabbitMQ \
  --engine-version 3.13 \
  --host-instance-type mq.m7g.medium \
  --deployment-mode SINGLE_INSTANCE \
  --auto-minor-version-upgrade \
  --publicly-accessible false \
  --subnet-ids subnet-xxx \
  --security-groups sg-xxx \
  --users Username=admin,Password=<PASS>

# 2. 起独立 Valkey（约 11 USD/月）
aws elasticache create-cache-cluster \
  --cache-cluster-id valkey-ai \
  --cache-node-type cache.t4g.micro \
  --engine valkey \
  --engine-version 8.0 \
  --num-cache-nodes 1 \
  --cache-subnet-group-name ai-cache-subnet \
  --security-group-ids sg-xxx

-- 3. AI 库 ID 起点提到 10000000+（步骤 4.1）
ALTER TABLE service_a_ai.m_project AUTO_INCREMENT = 10000000;

-- 4. 加 dispatch_env 字段（步骤 4.2）
ALTER TABLE service_a_ai.m_realtime_messages
  ADD COLUMN dispatch_env VARCHAR(16) NOT NULL DEFAULT 'ai' AFTER project_id,
  ADD INDEX idx_dispatch_env (dispatch_env);

#!/bin/bash
# merge-aux-checklist.sh - 集群合并后扫所有辅助组件
set -euo pipefail

CTX_NEW="${1:?需要新集群 context}"
CTX_OLD="${2:?需要旧集群 context}"
NS="${3:?需要 namespace}"

echo "==> 1. 比对 Deployment 清单"
diff <(kubectl --context "$CTX_OLD" get deploy -n "$NS" -o name | sort) \
     <(kubectl --context "$CTX_NEW" get deploy -n "$NS" -o name | sort) || \
     echo "    ⚠️ Deployment 不一致"

echo "==> 2. 比对 DaemonSet"
diff <(kubectl --context "$CTX_OLD" get ds -n "$NS" -o name | sort) \
     <(kubectl --context "$CTX_NEW" get ds -n "$NS" -o name | sort) || \
     echo "    ⚠️ DaemonSet 不一致"

echo "==> 3. 比对 CronJob"
diff <(kubectl --context "$CTX_OLD" get cj -n "$NS" -o name | sort) \
     <(kubectl --context "$CTX_NEW" get cj -n "$NS" -o name | sort) || \
     echo "    ⚠️ CronJob 不一致"

echo "==> 4. 比对 ServiceAccount + RBAC"
diff <(kubectl --context "$CTX_OLD" get sa,role,rolebinding -n "$NS" -o name | sort) \
     <(kubectl --context "$CTX_NEW" get sa,role,rolebinding -n "$NS" -o name | sort) || \
     echo "    ⚠️ RBAC 不一致"

echo "==> 5. 检查 placeholder Deployment replicas"
kubectl --context "$CTX_NEW" get deploy -n "$NS" -l app=placeholder \
  -o jsonpath='{.items[*].spec.replicas}' | \
  grep -qE '^[1-9]' || echo "    ❌ placeholder replicas=0，NodePool 起不来"

echo "==> 6. 端到端冒烟（业务自定义）"
echo "    跑 ./e2e-smoke.sh 创建沙箱 → 使用 → 销毁，确认全链路通"

# 1. PRE 单独建 Valkey
aws elasticache create-serverless-cache \
  --serverless-cache-name valkey-meter-pre \
  --engine valkey \
  --major-engine-version 8 \
  --subnet-ids subnet-xxx \
  --security-group-ids sg-xxx

# 2. 业务代码强制 key prefix（步骤 3.3）

# 3. 用 SCAN 把老 key 迁过去
redis-cli -h old-shared-valkey --tls --insecure --scan --pattern 'gw:*' | while read k; do
  v=$(redis-cli -h old-shared-valkey --tls --insecure get "$k")
  redis-cli -h valkey-meter-pre --tls --insecure set "pre:$k" "$v"
done

# 4. 配置中心切到新 endpoint，rollout 业务
kubectl --context merged rollout restart deploy/sandbox-meter -n app-pre

---
# Loki Helm values 关键片段
loki:
  schemaConfig:
    configs:
      - from: "2026-04-30"
        store: tsdb
        object_store: s3
        schema: v13
        index:
          prefix: loki_index_
          period: 24h
  storage:
    type: s3
    s3:
      region: ap-southeast-1
      bucketnames: example-loki-merged
      s3forcepathstyle: false
  storage_config:
    aws:
      bucketnames: example-loki-merged
      region: ap-southeast-1
      s3: s3://ap-southeast-1/example-loki-merged

#!/bin/bash
# verify-storage-backend.sh - 操作有状态服务前必须跑
set -euo pipefail

NS="$1"
STS="$2"

echo "==> 1. 看 PVC retention policy"
kubectl get sts "$STS" -n "$NS" -o jsonpath='{.spec.persistentVolumeClaimRetentionPolicy}'
echo
# 期望：whenDeleted=Retain, whenScaled=Retain
# 如果是 Delete，立即改成 Retain：
# kubectl patch sts "$STS" -n "$NS" --type=merge -p \
#   '{"spec":{"persistentVolumeClaimRetentionPolicy":{"whenDeleted":"Retain","whenScaled":"Retain"}}}'

echo "==> 2. 看真实存储后端配置"
kubectl get configmap -n "$NS" -o yaml | grep -E "object_store|backend|s3:" || \
  echo "    ⚠️ 未发现 S3/对象存储配置，可能是 filesystem"

echo "==> 3. 看 PVC 数据量"
kubectl get pvc -n "$NS" -l app="$STS" -o custom-columns=NAME:.metadata.name,SIZE:.spec.resources.requests.storage,BOUND:.spec.volumeName

echo "==> 4. 确认操作风险"
read -p "确认存储后端是 S3 / filesystem？继续操作请输入 'i-checked-backend': " CONFIRM
[[ "$CONFIRM" == "i-checked-backend" ]] || { echo "未确认，退出"; exit 1; }