Playbook：自建 Headscale 零信任 Mesh，混合云内网访问的可执行落地方案

~/db-tunnel.sh prod
mysql -h 127.0.0.1 -P 13306 -u <user> -p

#!/bin/bash
# 01-create-headscale-ecs.sh
set -euo pipefail

REGION="cn-beijing"
VSWITCH_ID="vsw-xxxxxxxxxxxxxxxxx"      # ArgoCD VPC 的 cn-beijing-i 交换机
VPC_ID="vpc-xxxxxxxxxxxxxxxxx"
INSTANCE_TYPE="ecs.e-c1m2.large"        # 2c4g 经济型
IMAGE_ID="ubuntu_22_04_x64_20G_alibase_20240228.vhd"
KEY_NAME="ops-keypair"                  # 提前在 ECS 控制台导入的公钥
INSTANCE_NAME="headscale-prod-cn"

# 1.1 创建独立安全组
SG_ID=$(aliyun ecs CreateSecurityGroup \
  --RegionId "$REGION" \
  --VpcId "$VPC_ID" \
  --SecurityGroupName "headscale-prod-sg" \
  --Description "Headscale control plane + DERP" \
  --query 'SecurityGroupId' --output text)
echo "Created SG: $SG_ID"

# 1.2 加入站规则
# SSH 仅运维办公出口 IP（按实际填）
aliyun ecs AuthorizeSecurityGroup \
  --RegionId "$REGION" --SecurityGroupId "$SG_ID" \
  --IpProtocol tcp --PortRange "22/22" \
  --SourceCidrIp "203.0.113.0/24" \
  --Description "Ops office SSH"

# 80/443 开公网（Caddy 跑 ACME + Headscale HTTPS）
for port in "80/80" "443/443"; do
  aliyun ecs AuthorizeSecurityGroup \
    --RegionId "$REGION" --SecurityGroupId "$SG_ID" \
    --IpProtocol tcp --PortRange "$port" \
    --SourceCidrIp "0.0.0.0/0" \
    --Description "HTTP/HTTPS"
done

# 3478 UDP（DERP STUN）
aliyun ecs AuthorizeSecurityGroup \
  --RegionId "$REGION" --SecurityGroupId "$SG_ID" \
  --IpProtocol udp --PortRange "3478/3478" \
  --SourceCidrIp "0.0.0.0/0" \
  --Description "DERP STUN"

# 41641 UDP（WireGuard 直连 fallback）
aliyun ecs AuthorizeSecurityGroup \
  --RegionId "$REGION" --SecurityGroupId "$SG_ID" \
  --IpProtocol udp --PortRange "41641/41641" \
  --SourceCidrIp "0.0.0.0/0" \
  --Description "WireGuard direct"

# 1.3 创建 ECS（按量付费，1Mbps 按使用量计）
INSTANCE_ID=$(aliyun ecs RunInstances \
  --RegionId "$REGION" \
  --ImageId "$IMAGE_ID" \
  --InstanceType "$INSTANCE_TYPE" \
  --SecurityGroupId "$SG_ID" \
  --VSwitchId "$VSWITCH_ID" \
  --InstanceName "$INSTANCE_NAME" \
  --HostName "$INSTANCE_NAME" \
  --InstanceChargeType "PostPaid" \
  --InternetChargeType "PayByTraffic" \
  --InternetMaxBandwidthOut 1 \
  --SystemDisk.Category "cloud_essd" \
  --SystemDisk.Size 40 \
  --DataDisk.1.Category "cloud_essd" \
  --DataDisk.1.Size 20 \
  --DataDisk.1.DeleteWithInstance true \
  --KeyPairName "$KEY_NAME" \
  --Amount 1 \
  --query 'InstanceIdSets.InstanceIdSet[0]' --output text)
echo "Created instance: $INSTANCE_ID"

# 1.4 等待 Running，分配公网 IP
sleep 30
aliyun ecs AllocatePublicIpAddress \
  --InstanceId "$INSTANCE_ID"

PUBLIC_IP=$(aliyun ecs DescribeInstances \
  --RegionId "$REGION" --InstanceIds "[\"$INSTANCE_ID\"]" \
  --query 'Instances.Instance[0].PublicIpAddress.IpAddress[0]' --output text)
echo "Public IP: $PUBLIC_IP"
echo "$INSTANCE_ID $PUBLIC_IP" > /tmp/headscale-ecs-info.txt

# 查实例状态
aliyun ecs DescribeInstanceStatus --RegionId cn-beijing --InstanceId.1 "$INSTANCE_ID"
# 期望：Status=Running

# 测试 SSH
ssh -i ~/.ssh/ops-keypair root@$PUBLIC_IP "uname -a"
# 期望：Linux headscale-prod-cn 5.15.0-... Ubuntu

# 删除实例（包含数据盘）
aliyun ecs DeleteInstance --InstanceId "$INSTANCE_ID" --Force true
# 删除 SG
aliyun ecs DeleteSecurityGroup --RegionId cn-beijing --SecurityGroupId "$SG_ID"

#!/bin/bash
# 02-setup-dns-and-disk.sh
set -euo pipefail

DOMAIN="example.cn"
RR="headscale"
PUBLIC_IP=$(awk '{print $2}' /tmp/headscale-ecs-info.txt)

# 2.1 加 A 记录
aliyun alidns AddDomainRecord \
  --DomainName "$DOMAIN" \
  --RR "$RR" \
  --Type A \
  --Value "$PUBLIC_IP" \
  --TTL 600

# 2.2 验证 DNS（等 60 秒生效）
sleep 60
dig +short "$RR.$DOMAIN" @223.5.5.5
# 期望：输出 PUBLIC_IP

# 2.3 SSH 到 ECS 上格式化数据盘
ssh root@$PUBLIC_IP <<'EOF'
set -euo pipefail
DEVICE=$(lsblk -ndo NAME,TYPE,MOUNTPOINT | awk '$2=="disk" && $3=="" {print "/dev/"$1; exit}')
mkfs.ext4 -L data "$DEVICE"
mkdir -p /data
echo "LABEL=data /data ext4 defaults 0 2" >> /etc/fstab
mount /data
df -hT /data
EOF

# DNS 全国生效检查
for ns in 223.5.5.5 8.8.8.8 114.114.114.114; do
  echo -n "$ns: "; dig +short headscale.example.cn @$ns
done
# 期望三个 NS 都返回同一个 IP

# 数据盘挂载
ssh root@$PUBLIC_IP "df -hT /data"
# 期望：ext4，挂载在 /data

# 删 DNS 记录
RECORD_ID=$(aliyun alidns DescribeDomainRecords --DomainName example.cn \
  --RRKeyWord headscale --query 'DomainRecords.Record[0].RecordId' --output text)
aliyun alidns DeleteDomainRecord --RecordId "$RECORD_ID"

ssh root@$PUBLIC_IP
mkdir -p /data/headscale/{config,data,certs,logs}
cd /data/headscale

{
    email ops@example.cn
    # 启用 ACME HTTP-01 验证（80 端口必须开放给 Let's Encrypt 验证服务器）
    acme_ca https://acme-v02.api.letsencrypt.org/directory
}

headscale.example.cn {
    encode gzip
    log {
        output file /var/log/caddy/access.log {
            roll_size 100mb
            roll_keep 7
        }
        format json
    }

    # gRPC 长连接需要的 buffer 调大
    reverse_proxy headscale:8080 {
        flush_interval -1
        transport http {
            read_timeout 300s
            write_timeout 300s
        }
    }
}

# DERP STUN 不走 Caddy（UDP 3478 直接由 Headscale 接），这里只占位

# Headscale 0.26.x 完整配置
server_url: https://headscale.example.cn
listen_addr: 0.0.0.0:8080
metrics_listen_addr: 127.0.0.1:9090

grpc_listen_addr: 0.0.0.0:50443
grpc_allow_insecure: false

# 节点身份私钥（首次启动自动生成）
private_key_path: /var/lib/headscale/private.key
noise:
  private_key_path: /var/lib/headscale/noise_private.key

# IPv4 池：100.64.0.0/10 是 Tailscale 标准段
prefixes:
  v4: 100.64.0.0/10
  v6: fd7a:115c:a1e0::/48
  allocation: sequential

# 嵌入式 DERP（生产建议至少再起一个独立 DERP，本配置先单点起步）
derp:
  server:
    enabled: true
    region_id: 999
    region_code: "self-hosted-cn"
    region_name: "Headscale Self-hosted CN-Beijing"
    stun_listen_addr: "0.0.0.0:3478"
  urls: []
  paths: []
  auto_update_enabled: false  # 禁用从 Tailscale 官方拉默认 DERP 列表
  update_frequency: 24h

# 节点过期：未活跃节点 180 天后自动 expire
node_update_check_interval: 10s
ephemeral_node_inactivity_timeout: 30m

# 数据库：起步 SQLite（< 100 节点都够），后续上 Litestream 备份
database:
  type: sqlite3
  sqlite:
    path: /var/lib/headscale/db.sqlite

# 日志
log:
  format: text
  level: info

# 策略文件模式（git 管理 ACL）
policy:
  mode: file
  path: /etc/headscale/acl.json

# DNS：让 mesh 内部 hostname 自动解析（暂不开 split-DNS，后续可加）
dns:
  override_local_dns: false
  nameservers:
    global:
      - 1.1.1.1
      - 8.8.8.8
  magic_dns: true
  base_domain: mesh.example.cn

unix_socket: /var/run/headscale/headscale.sock
unix_socket_permission: "0770"

# 关闭非必要的 metric：保留 prometheus
disable_check_updates: true

services:
  headscale:
    image: headscale/headscale:0.26.1
    container_name: headscale
    restart: unless-stopped
    command: serve
    user: "1000:1000"
    volumes:
      - ./config:/etc/headscale
      - ./data:/var/lib/headscale
      - ./logs:/var/log/headscale
      - /var/run/headscale:/var/run/headscale
    ports:
      # 控制面 HTTP（仅 127.0.0.1，外网走 Caddy）
      - "127.0.0.1:8080:8080"
      # DERP STUN（UDP 3478）
      - "0.0.0.0:3478:3478/udp"
      # gRPC（远程 CLI 用，先不开公网）
      - "127.0.0.1:50443:50443"
      # Metrics（Prometheus 拉）
      - "127.0.0.1:9090:9090"
    healthcheck:
      test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/health"]
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 30s
    deploy:
      resources:
        limits:
          cpus: "1.0"
          memory: 512M
        reservations:
          cpus: "0.1"
          memory: 128M
    networks:
      - headscale-net
    logging:
      driver: json-file
      options:
        max-size: "100m"
        max-file: "5"

  caddy:
    image: caddy:2.8-alpine
    container_name: caddy
    restart: unless-stopped
    ports:
      - "0.0.0.0:80:80"
      - "0.0.0.0:443:443"
    volumes:
      - ./config/Caddyfile:/etc/caddy/Caddyfile:ro
      - ./certs:/data
      - caddy-config:/config
      - ./logs:/var/log/caddy
    depends_on:
      headscale:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "wget", "-q", "--spider", "http://localhost:80"]
      interval: 30s
      timeout: 5s
      retries: 3
    deploy:
      resources:
        limits:
          cpus: "0.5"
          memory: 256M
    networks:
      - headscale-net

volumes:
  caddy-config:

networks:
  headscale-net:
    driver: bridge
    ipam:
      config:
        # 显式指定网段，避免与 mesh advertise 段冲突（坑 1）
        - subnet: 10.250.0.0/24

cd /data/headscale
# 起步用一个最严格的 ACL（拒绝所有），后面再开
cat > config/acl.json <<'EOF'
{
  "groups": {},
  "tagOwners": {},
  "acls": []
}
EOF

# 设置目录所有者（Headscale 容器以 UID 1000 跑）
chown -R 1000:1000 data logs
chmod 0700 data

# 启动
docker compose up -d
docker compose logs -f --tail=50 headscale
# 等到日志出现 "listening on [::]:8080"，按 Ctrl-C 退出

# 3.1 容器健康
docker compose ps
# 期望两个 service 都是 Up (healthy)

# 3.2 Caddy 自动取证书
curl -sI https://headscale.example.cn/health
# 期望：HTTP/2 200，证书由 Let's Encrypt R10/R11 签发
openssl s_client -connect headscale.example.cn:443 -servername headscale.example.cn </dev/null 2>/dev/null | openssl x509 -noout -issuer -dates
# 期望：issuer = C=US, O=Let's Encrypt, CN=R10
#       validity 90 天

# 3.3 Headscale API 通
docker compose exec headscale headscale version
# 期望：v0.26.1

# 3.4 证书续期监控（写入 cron）
cat > /etc/cron.d/cert-monitor <<'EOF'
0 6 * * * root /usr/bin/openssl s_client -connect headscale.example.cn:443 -servername headscale.example.cn </dev/null 2>/dev/null | /usr/bin/openssl x509 -noout -checkend 1209600 || /usr/bin/curl -X POST -H 'Content-Type: application/json' -d '{"msgtype":"text","text":{"content":"[ALERT] headscale.example.cn 证书 14 天内到期"}}' '<DingTalk Webhook>'
EOF

docker compose down -v
rm -rf /data/headscale

cd /data/headscale

# 4.1 建 infra 服务账号（挂 Subnet Router）
docker compose exec headscale headscale users create infra@example.cn

# 4.2 建运维账号（首批 1-2 个种子）
docker compose exec headscale headscale users create alice@example.cn
docker compose exec headscale headscale users create bob@example.cn

# 4.3 列出验证
docker compose exec headscale headscale users list
# 期望输出三行：infra@example.cn / alice@example.cn / bob@example.cn

{
  "groups": {
    "group:ops": ["alice@example.cn", "bob@example.cn"]
  },
  "tagOwners": {
    "tag:subnet-router": ["group:ops"]
  },
  "hosts": {
    "aurora-prod": "10.x.y.z",
    "redis-prod":  "10.x.y.21"
  },
  "acls": [
    {
      "action": "accept",
      "src": ["group:ops"],
      "dst": ["*:*"]
    },
    {
      "action": "accept",
      "src": ["tag:subnet-router"],
      "dst": ["tag:subnet-router:*"]
    }
  ]
}

{
  "groups": {
    "group:ops": ["alice@example.cn"],
    "group:dba": ["bob@example.cn", "carol@example.cn"],
    "group:dev": ["dave@example.cn", "eve@example.cn"]
  },
  "tagOwners": {
    "tag:subnet-router": ["group:ops"],
    "tag:db-prod":       ["group:ops"],
    "tag:db-qa":         ["group:ops"],
    "tag:k8s-prod":      ["group:ops"]
  },
  "hosts": {
    "aurora-prod":  "10.x.y.z",
    "aurora-qa":    "10.2.5.98",
    "redis-prod":   "10.x.y.21",
    "grafana-prod": "10.x.y.50"
  },
  "acls": [
    {
      "action": "accept",
      "src": ["group:ops"],
      "dst": ["*:*"]
    },
    {
      "action": "accept",
      "src": ["group:dba"],
      "dst": [
        "aurora-prod:3306,5432",
        "aurora-qa:3306,5432",
        "redis-prod:6379"
      ]
    },
    {
      "action": "accept",
      "src": ["group:dev"],
      "dst": [
        "aurora-qa:3306,5432",
        "grafana-prod:443"
      ]
    },
    {
      "action": "accept",
      "src": ["tag:subnet-router"],
      "dst": ["tag:subnet-router:*"]
    }
  ]
}

{
  "groups": {
    "group:ops": ["alice@example.cn"],
    "group:dba": ["bob@example.cn"],
    "group:dev": ["dave@example.cn"]
  },
  "tagOwners": {
    "tag:env-prod":      ["group:ops"],
    "tag:env-qa":        ["group:ops"],
    "tag:role-db":       ["group:ops"],
    "tag:role-cache":    ["group:ops"],
    "tag:subnet-router": ["group:ops"]
  },
  "acls": [
    { "action": "accept", "src": ["group:ops"], "dst": ["*:*"] },
    { "action": "accept", "src": ["group:dba"],
      "dst": ["tag:role-db:3306,5432", "tag:env-qa:*"] },
    { "action": "accept", "src": ["group:dev"],
      "dst": ["tag:env-qa:*"] },
    { "action": "accept", "src": ["tag:subnet-router"],
      "dst": ["tag:env-prod:*", "tag:env-qa:*"] }
  ],
  "ssh": [
    { "action": "accept",
      "src": ["group:ops"],
      "dst": ["tag:env-prod"],
      "users": ["root", "ubuntu"] }
  ]
}

docker compose kill -s SIGHUP headscale
docker compose logs --tail=20 headscale
# 期望日志：reloaded policy from /etc/headscale/acl.json

docker compose exec headscale headscale policy check --policy-file /etc/headscale/acl.json
# 期望：policy is valid

docker compose exec headscale headscale users list
docker compose exec headscale headscale nodes list

git -C /data/headscale checkout config/acl.json
docker compose kill -s SIGHUP headscale

#!/bin/bash
# join-mesh-mac.sh
set -euo pipefail
LOGIN_SERVER="https://headscale.example.cn"
AUTHKEY="${1:-}"
[ -z "$AUTHKEY" ] && { echo "Usage: $0 <preauthkey>"; exit 1; }

# 1. 安装（cask 装的是带 GUI 的版本，但走命令行登录）
brew install --cask tailscale

# 2. 启动后台 daemon（首次安装会弹权限框，要求允许 VPN profile）
open -a "Tailscale" || true
sleep 5

# 3. 命令行登录到自建控制面（不要点 GUI 的 Sign in）
sudo /Applications/Tailscale.app/Contents/MacOS/Tailscale up \
  --login-server="$LOGIN_SERVER" \
  --auth-key="$AUTHKEY" \
  --accept-routes \
  --accept-dns=false

# 4. 验证
/Applications/Tailscale.app/Contents/MacOS/Tailscale status

#!/bin/bash
# join-mesh-linux.sh
set -euo pipefail
LOGIN_SERVER="https://headscale.example.cn"
AUTHKEY="${1:-}"
[ -z "$AUTHKEY" ] && { echo "Usage: sudo $0 <preauthkey>"; exit 1; }
[ "$EUID" -ne 0 ] && { echo "Run as root"; exit 1; }

# 1. 安装
curl -fsSL https://tailscale.com/install.sh | sh

# 2. 开 IP forwarding（如果该机器要做 Subnet Router；纯客户端跳过）
# echo 'net.ipv4.ip_forward=1' >> /etc/sysctl.d/99-tailscale.conf
# sysctl -p /etc/sysctl.d/99-tailscale.conf

# 3. 启动 systemd 服务
systemctl enable --now tailscaled

# 4. 登录
tailscale up \
  --login-server="$LOGIN_SERVER" \
  --auth-key="$AUTHKEY" \
  --accept-routes \
  --accept-dns=false

# 5. 验证
tailscale status
tailscale ip -4

# join-mesh-windows.ps1
$ErrorActionPreference = "Stop"
$loginServer = "https://headscale.example.cn"
$authKey = $args[0]
if (-not $authKey) { Write-Error "Usage: .\join-mesh-windows.ps1 <preauthkey>"; exit 1 }

# 1. 下载并静默安装（最新稳定版）
$installer = "$env:TEMP\tailscale-setup.exe"
Invoke-WebRequest -Uri "https://pkgs.tailscale.com/stable/tailscale-setup-latest.exe" -OutFile $installer
Start-Process -FilePath $installer -ArgumentList "/S" -Wait

# 2. 注册到自建控制面（注意：必须用命令行 up，GUI 默认连官方）
$tailscale = "C:\Program Files\Tailscale\tailscale.exe"
& $tailscale up --login-server=$loginServer --auth-key=$authKey --accept-routes --accept-dns=false --unattended

# 3. 验证
& $tailscale status
& $tailscale ip -4

docker compose exec headscale headscale preauthkeys create \
  --user alice@example.cn \
  --expiration 24h
# 输出形如：abc123def456...（40 字符 hex）

# 5.1 mesh 状态
tailscale status
# 期望：本机有 100.64.x.x IP；至少一个 peer 是 Subnet Router

# 5.2 ICMP（macOS 注意：utun 接口对 ICMP 有特殊处理，ping 不通不代表网络不通）
tailscale ping 10.x.y.z
# 期望：pong via DERP/direct，<= 200ms

# 5.3 实际 TCP（更可靠的判定）
nc -zv 10.x.y.z 3306
# 期望：Connection to 10.x.y.z port 3306 [tcp/*] succeeded!

# Mac
sudo /Applications/Tailscale.app/Contents/MacOS/Tailscale logout
sudo /Applications/Tailscale.app/Contents/MacOS/Tailscale down
# Linux
sudo tailscale down
sudo tailscale logout
sudo apt remove --purge tailscale
# 控制面侧也清掉
docker compose exec headscale headscale nodes expire --identifier <node-id>

docker compose exec headscale headscale preauthkeys create \
  --user infra@example.cn --reusable --expiration 8760h \
  --tags tag:subnet-router
# 期望输出 40 字符 hex key，记下后填入下方 Secret

---
apiVersion: v1
kind: Namespace
metadata:
  name: tailscale-system
  labels:
    pod-security.kubernetes.io/enforce: privileged
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: tailscale
  namespace: tailscale-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: tailscale
  namespace: tailscale-system
rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["create", "get", "list", "watch", "update", "patch", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: tailscale
  namespace: tailscale-system
subjects:
  - kind: ServiceAccount
    name: tailscale
    namespace: tailscale-system
roleRef:
  kind: Role
  name: tailscale
  apiGroup: rbac.authorization.k8s.io
---
# Subnet Router 用 infra@ 的 reusable PreAuthKey 自注册
# 生产建议从 ExternalSecrets / Sealed Secrets 拉，避免明文进 git
apiVersion: v1
kind: Secret
metadata:
  name: tailscale-auth
  namespace: tailscale-system
type: Opaque
stringData:
  TS_AUTHKEY: "PASTE_YOUR_REUSABLE_PREAUTHKEY_HERE"
---
# 状态 secret（Tailscale 自己写）
apiVersion: v1
kind: Secret
metadata:
  name: tailscale-state
  namespace: tailscale-system
type: Opaque
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: subnet-router
  namespace: tailscale-system
  labels:
    app: subnet-router
    cluster: us-prod  # 改成实际集群名
spec:
  # 起步 1 副本（Tailscale 同 CIDR 多副本主备切换 30-60s 抖动可见）
  # 真有跨 AZ 故障担忧时再调到 2，并接受切换抖动
  replicas: 1
  strategy:
    type: Recreate  # Subnet Router 不能 RollingUpdate（单 mesh 节点身份）
  selector:
    matchLabels:
      app: subnet-router
  template:
    metadata:
      labels:
        app: subnet-router
      annotations:
        # 让每日 CronJob 改这个 annotation 触发滚动重启
        kubectl.kubernetes.io/restartedAt: "2026-04-30T03:00:00+08:00"
    spec:
      serviceAccountName: tailscale
      # 调度到内核 6.2+ 节点开 rx-udp-gro-forwarding（吞吐显著提升）
      nodeSelector:
        kubernetes.io/os: linux
      tolerations:
        - key: "node.kubernetes.io/not-ready"
          operator: "Exists"
          effect: "NoExecute"
          tolerationSeconds: 60
      containers:
        - name: tailscale
          image: tailscale/tailscale:v1.84.0
          imagePullPolicy: IfNotPresent
          env:
            - name: TS_KUBE_SECRET
              value: "tailscale-state"
            - name: TS_USERSPACE
              value: "false"            # 用内核 WireGuard，性能比 userspace 高 5x
            - name: TS_HOSTNAME
              value: "subnet-router-us-prod"   # 改成 cluster + region
            - name: TS_ROUTES
              value: "10.3.0.0/18"      # 仅 advertise VPC CIDR，不要广播 ClusterIP
            - name: TS_EXTRA_ARGS
              value: "--login-server=https://headscale.example.cn --accept-routes --accept-dns=false --advertise-tags=tag:subnet-router"
            - name: TS_AUTHKEY
              valueFrom:
                secretKeyRef:
                  name: tailscale-auth
                  key: TS_AUTHKEY
            # 性能调优（内核 6.2+ 才生效）
            - name: TS_DEBUG_FIREWALL_MODE
              value: "auto"
          securityContext:
            capabilities:
              add:
                - NET_ADMIN
                - NET_RAW
          resources:
            requests:
              cpu: 50m
              memory: 64Mi
            limits:
              cpu: 500m
              memory: 256Mi
          livenessProbe:
            exec:
              command:
                - tailscale
                - status
                - --self=false
            initialDelaySeconds: 60
            periodSeconds: 30
            failureThreshold: 3
          readinessProbe:
            exec:
              command:
                - tailscale
                - status
                - --json
            initialDelaySeconds: 15
            periodSeconds: 10
---
# 每日凌晨 3 点滚动重启（深夜低峰，强制刷新所有 peer 状态，规避坑 2）
apiVersion: batch/v1
kind: CronJob
metadata:
  name: subnet-router-daily-restart
  namespace: tailscale-system
spec:
  schedule: "0 19 * * *"   # UTC 19:00 = 北京 03:00
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      backoffLimit: 2
      template:
        spec:
          serviceAccountName: subnet-router-restart
          restartPolicy: OnFailure
          containers:
            - name: kubectl
              image: bitnami/kubectl:1.29
              command:
                - /bin/sh
                - -c
                - |
                  kubectl -n tailscale-system rollout restart deployment/subnet-router
                  kubectl -n tailscale-system rollout status deployment/subnet-router --timeout=180s
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: subnet-router-restart
  namespace: tailscale-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: subnet-router-restart
  namespace: tailscale-system
rules:
  - apiGroups: ["apps"]
    resources: ["deployments"]
    verbs: ["get", "list", "patch"]
  - apiGroups: ["apps"]
    resources: ["deployments/status"]
    verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: subnet-router-restart
  namespace: tailscale-system
subjects:
  - kind: ServiceAccount
    name: subnet-router-restart
    namespace: tailscale-system
roleRef:
  kind: Role
  name: subnet-router-restart
  apiGroup: rbac.authorization.k8s.io

# 6.1 应用 manifest
kubectl --context us-prod apply -f subnet-router.yaml

# 6.2 等 Pod 起来
kubectl --context us-prod -n tailscale-system rollout status deploy/subnet-router --timeout=120s

# 6.3 看日志确认注册成功
kubectl --context us-prod -n tailscale-system logs -l app=subnet-router --tail=50
# 期望出现：Success. ... Tailscale started

# 6.4 在 Headscale 控制面手动批准路由（首次必须批准）
ssh root@headscale-ecs
cd /data/headscale
docker compose exec headscale headscale nodes list
# 找到对应 hostname 的 ID
docker compose exec headscale headscale nodes approve-routes \
  --identifier <node-id> --routes 10.3.0.0/18

spec:
  replicas: 2
  template:
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchLabels:
                  app: subnet-router
              topologyKey: topology.kubernetes.io/zone

# Pod ready
kubectl --context us-prod get pod -n tailscale-system
# 期望：1/1 Running

# 路由广播
docker compose exec headscale headscale nodes list-routes
# 期望：subnet-router-us-prod | 10.3.0.0/18 | true | true (advertised + enabled)

# 端到端：从 Mac 客户端测
nc -zv 10.x.y.z 3306
# 期望：succeeded
mysql -h 10.x.y.z -u readonly -p -e "SELECT 1"
# 期望：返回 1

kubectl --context us-prod delete -f subnet-router.yaml
# 控制面 expire 节点（不要 delete，见坑 2）
docker compose exec headscale headscale nodes expire --identifier <node-id>

#!/bin/bash
# mesh-grant.sh - 给同事开通 mesh 访问权限
# 用法：./mesh-grant.sh <email> <group:dev|dba|ops>
# 前置：在 Headscale 控制面 ECS 上跑，cwd 为 /data/headscale

set -euo pipefail

EMAIL="${1:-}"
GROUP="${2:-}"

# 输入校验
if [[ -z "$EMAIL" || -z "$GROUP" ]]; then
  cat <<EOF >&2
Usage: $0 <email> <group:dev|dba|ops>
Example: $0 carol@example.cn dba
EOF
  exit 1
fi

if [[ ! "$EMAIL" =~ ^[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]+$ ]]; then
  echo "ERROR: invalid email format: $EMAIL" >&2
  exit 1
fi

if [[ ! "$GROUP" =~ ^(dev|dba|ops)$ ]]; then
  echo "ERROR: group must be one of: dev / dba / ops" >&2
  exit 1
fi

# 依赖检查
for cmd in jq docker; do
  command -v "$cmd" >/dev/null || { echo "ERROR: missing $cmd"; exit 1; }
done

ACL_FILE="config/acl.json"
[[ -f "$ACL_FILE" ]] || { echo "ERROR: $ACL_FILE not found"; exit 1; }

# 1. 创建用户（已存在则跳过）
EXISTING=$(docker compose exec -T headscale headscale users list --output json \
  | jq -r --arg e "$EMAIL" '.[] | select(.name==$e) | .id' || true)

if [[ -n "$EXISTING" ]]; then
  echo "[INFO] user $EMAIL already exists (id=$EXISTING)"
  USERID="$EXISTING"
else
  echo "[INFO] creating user $EMAIL"
  docker compose exec -T headscale headscale users create "$EMAIL"
  USERID=$(docker compose exec -T headscale headscale users list --output json \
    | jq -r --arg e "$EMAIL" '.[] | select(.name==$e) | .id')
fi

# 2. 加入 ACL group（幂等）
TMP=$(mktemp)
trap 'rm -f $TMP' EXIT

jq --arg e "$EMAIL" --arg g "group:$GROUP" '
  .groups[$g] = ((.groups[$g] // []) + [$e] | unique)
' "$ACL_FILE" > "$TMP"

# 校验生成的 ACL 合法
docker compose exec -T headscale headscale policy check --policy-file - < "$TMP" \
  || { echo "ERROR: ACL validation failed"; exit 1; }

# 备份 + 替换
cp "$ACL_FILE" "${ACL_FILE}.bak.$(date +%Y%m%d-%H%M%S)"
mv "$TMP" "$ACL_FILE"
trap - EXIT

# 3. 重载
docker compose kill -s SIGHUP headscale
sleep 2

# 4. 生成 90 天一次性 PreAuthKey
KEY=$(docker compose exec -T headscale headscale preauthkeys create \
  --user "$EMAIL" --expiration 2160h --output json \
  | jq -r '.key')

if [[ -z "$KEY" || "$KEY" == "null" ]]; then
  echo "ERROR: preauthkey generation failed"
  exit 1
fi

# 5. 输出标准话术（直接复制给同事）
cat <<EOF

==========================================
[mesh 接入信息 — 请妥善保管，仅 24h 内有效]
==========================================
邮箱：       $EMAIL
分组：       group:$GROUP
控制面：     https://headscale.example.cn
PreAuthKey： $KEY
有效期：     90 天

接入命令（macOS / Linux）：
  curl -O https://devops-bucket.example.cn/tools/join-mesh.sh
  chmod +x join-mesh.sh
  sudo ./join-mesh.sh $KEY

接入后验证：
  tailscale status
  nc -zv aurora-prod 3306

文档：https://wiki.example.cn/zerotrust-mesh
==========================================
EOF

#!/bin/bash
# mesh-revoke.sh - 回收用户访问权限（不删用户/不删节点）
# 用法：./mesh-revoke.sh <email>

set -euo pipefail
EMAIL="${1:-}"
[[ -z "$EMAIL" ]] && { echo "Usage: $0 <email>"; exit 1; }

command -v jq >/dev/null || { echo "need jq"; exit 1; }

ACL_FILE="config/acl.json"
TMP=$(mktemp)
trap 'rm -f $TMP' EXIT

# 1. 从所有 group 中移除
jq --arg e "$EMAIL" '
  .groups |= with_entries(.value |= (. - [$e]))
' "$ACL_FILE" > "$TMP"

# 2. 校验
docker compose exec -T headscale headscale policy check --policy-file - < "$TMP" \
  || { echo "ERROR: ACL validation failed"; exit 1; }

cp "$ACL_FILE" "${ACL_FILE}.bak.$(date +%Y%m%d-%H%M%S)"
mv "$TMP" "$ACL_FILE"
trap - EXIT

# 3. 重载（亚秒级生效）
docker compose kill -s SIGHUP headscale
sleep 2

# 4. 把该用户所有节点标记为过期（不删，避免触发坑 2 的 peer 同步 bug）
NODE_IDS=$(docker compose exec -T headscale headscale nodes list --output json \
  | jq -r --arg e "$EMAIL" '.[] | select(.user.name==$e) | .id')

for nid in $NODE_IDS; do
  echo "[INFO] expiring node $nid"
  docker compose exec -T headscale headscale nodes expire --identifier "$nid"
done

# 5. 验证
echo
echo "[VERIFY] $EMAIL 已从所有 group 移除，所有节点已 expire"
docker compose exec -T headscale headscale nodes list --output json \
  | jq -r --arg e "$EMAIL" '.[] | select(.user.name==$e) | "  node \(.id): expired=\(.expiry)"'

#!/bin/bash
# mesh-list.sh - 列出当前所有用户和节点（含 group / 路由 / 在线状态）
set -euo pipefail

echo "================ Users ================"
docker compose exec -T headscale headscale users list

echo
echo "================ ACL Groups ================"
jq -r '.groups | to_entries[] | "  \(.key):\n" + (.value | map("    - " + .) | join("\n"))' \
  config/acl.json

echo
echo "================ Nodes ================"
docker compose exec -T headscale headscale nodes list

echo
echo "================ Routes ================"
docker compose exec -T headscale headscale nodes list-routes

echo
echo "================ Online Peers (last 60s) ================"
docker compose exec -T headscale headscale nodes list --output json \
  | jq -r '.[] | select(.online==true) | "  \(.id) \(.given_name) \(.user.name) \(.ip_addresses[0])"'

chmod +x bin/*.sh
git init && git add . && git commit -m "headscale: initial config + ops scripts"
# 推到内部 git（建议加 .gitignore 排除 data/、certs/、logs/）

{
  "Version": "1",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "ecs:DescribeInstances",
        "ecs:DescribeInstanceStatus",
        "ecs:RebootInstance",
        "ecs:DescribeSecurityGroups",
        "ecs:DescribeSecurityGroupAttribute",
        "ecs:AuthorizeSecurityGroup",
        "ecs:RevokeSecurityGroup"
      ],
      "Resource": [
        "acs:ecs:cn-beijing:*:instance/i-xxxxxxxxxxxxxxxxx",
        "acs:ecs:cn-beijing:*:securitygroup/sg-xxxxxxxxxxxxxxxxx"
      ]
    },
    {
      "Effect": "Allow",
      "Action": [
        "alidns:DescribeDomainRecords",
        "alidns:UpdateDomainRecord"
      ],
      "Resource": "acs:alidns:*:*:domain/example.cn"
    }
  ]
}

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "AllowSSMSessionToBastion",
      "Effect": "Allow",
      "Action": [
        "ssm:StartSession",
        "ssm:TerminateSession",
        "ssm:ResumeSession"
      ],
      "Resource": [
        "arn:aws:ec2:us-west-2:<ACCOUNT_ID>:instance/i-xxxxxxxxxxxxx",
        "arn:aws:ssm:us-west-2:<ACCOUNT_ID>:document/AWS-StartPortForwardingSessionToRemoteHost"
      ]
    },
    {
      "Sid": "AllowDescribeForListing",
      "Effect": "Allow",
      "Action": [
        "ec2:DescribeInstances",
        "ssm:DescribeSessions"
      ],
      "Resource": "*"
    },
    {
      "Sid": "DenyEverythingElse",
      "Effect": "Deny",
      "Action": [
        "ssm:SendCommand",
        "ssm:StartAutomationExecution",
        "ec2:RunInstances"
      ],
      "Resource": "*"
    }
  ]
}

# 1. 客户端本地 mesh 状态
tailscale status
# 期望：peer 列表里有目标 Subnet Router，状态为 active 或 idle（不是 -）

# 2. 网络栈连通性
tailscale netcheck
# 期望：UDP 可用 + 至少一个 DERP region 可达
# 如果输出 "UDP: false" + "IPv4: blocked"，说明本地防火墙挡 UDP，必须走 DERP

# 3. 直连 vs 中继
tailscale ping --c 5 100.64.0.10
# 输出 "via DERP" → 走中继；"via X.X.X.X:Y" → P2P 成功

# 4. 控制面拉到的 packetfilter
tailscale status --json | jq '.Self.Capabilities, .CurrentTailnet'
# 检查是否拿到了预期的能力

# 5. 实际 TCP（绕开 ICMP 干扰）
nc -zv <target> <port>
# 在 macOS 上 ICMP 可能被 utun 过滤，永远不要只看 ping 结果

# 现象：新设备登录失败，但已建立 mesh 仍能跑（WireGuard 无状态）

# 1. ECS 上检查
ssh root@headscale-ecs
cd /data/headscale
docker compose ps
docker compose logs --tail=100 headscale

# 2. 重启
docker compose restart headscale
docker compose logs -f headscale  # 看到 "listening on" 即恢复

# 3. 如果 SQLite 损坏（看到 "database disk image is malformed"）
# 见场景 C

dbs:
  - path: /data/headscale/data/db.sqlite
    replicas:
      - type: s3
        bucket: example-headscale-backup
        path: prod/db.sqlite
        region: oss-cn-beijing
        endpoint: https://oss-cn-beijing.aliyuncs.com
        access-key-id: ${OSS_AK}
        secret-access-key: ${OSS_SK}
        retention: 720h           # 30 天
        retention-check-interval: 1h
        snapshot-interval: 1h
        validation-interval: 12h

  litestream:
    image: litestream/litestream:0.3
    container_name: litestream
    restart: unless-stopped
    command: ["replicate", "-config", "/etc/litestream.yml"]
    volumes:
      - ./data:/data/headscale/data
      - ./litestream.yml:/etc/litestream.yml:ro
    environment:
      OSS_AK: ${OSS_AK}
      OSS_SK: ${OSS_SK}
    depends_on:
      - headscale

# 1. 停 Headscale
cd /data/headscale
docker compose stop headscale

# 2. 备份当前损坏库（万一）
mv data/db.sqlite data/db.sqlite.broken.$(date +%s)

# 3. 从 OSS 恢复最新快照
docker run --rm \
  -e OSS_AK=$OSS_AK -e OSS_SK=$OSS_SK \
  -v $(pwd)/data:/restore \
  -v $(pwd)/litestream.yml:/etc/litestream.yml:ro \
  litestream/litestream:0.3 \
  restore -config /etc/litestream.yml -o /restore/db.sqlite \
  /data/headscale/data/db.sqlite

# 4. 验证完整性
sqlite3 data/db.sqlite "PRAGMA integrity_check;"
# 期望：ok

# 5. 起 Headscale
docker compose start headscale
docker compose logs -f headscale

# 6. 客户端重连不需要重新注册（节点身份基于公钥，存在 SQLite 里恢复了）

# 1. 退出官方账号
sudo tailscale logout

# 2. 重新指定自建控制面
sudo tailscale up \
  --login-server=https://headscale.example.cn \
  --auth-key=<新 PreAuthKey> \
  --accept-routes \
  --reset

# 3. 验证 login server（macOS 路径不同）
tailscale debug prefs | grep -i 'ControlURL\|loginserver'
# 期望：ControlURL=https://headscale.example.cn

kubectl --context sandbox -n tailscale-system set env deploy/subnet-router \
  TS_ROUTES=10.x.0.0/18

# Headscale 控制面同步撤销路由
docker compose exec headscale headscale nodes approve-routes \
  --identifier <node-id> --routes 10.x.0.0/18

# 错误做法（触发 peer 同步 bug）
docker compose exec headscale headscale nodes delete --identifier <id>

# 正确做法 — 见第 7 步 mesh-revoke.sh
./bin/mesh-revoke.sh dave@example.cn

# 1. 重命名所有老用户
docker compose exec headscale headscale users list
# 假设老用户：alice, bob, infra

for old in alice bob infra; do
  docker compose exec headscale headscale users rename "$old" "$old@example.cn"
done

# 2. 同步改 ACL 里的引用
sed -i.bak 's/"alice"/"alice@example.cn"/g; s/"bob"/"bob@example.cn"/g; s/"infra"/"infra@example.cn"/g' config/acl.json

# 3. 重载
docker compose kill -s SIGHUP headscale

# 错误的诊断（ICMP 受 utun 影响）
ping 10.x.y.z

# 正确的诊断
tailscale ping 10.x.y.z       # mesh 层连通
nc -zv 10.x.y.z 3306          # 实际 TCP 服务
mysql -h 10.x.y.z -u xxx -p   # 应用层