Playbook：中等规模公司的完整 DevOps 流程——从代码提交到生产部署的全链路设计

# Gitea 1.22 helm install，docker-compose 也可以但 K8s 内更省心
helm repo add gitea-charts https://dl.gitea.com/charts/
helm repo update

helm upgrade --install gitea gitea-charts/gitea \
  --version 10.4.0 \
  --namespace gitea --create-namespace \
  --set persistence.size=50Gi \
  --set postgresql-ha.enabled=false \
  --set postgresql.enabled=true \
  --set redis-cluster.enabled=false \
  --set redis.enabled=true \
  --set gitea.config.server.ROOT_URL=https://git.example.cn \
  --set gitea.config.repository.DEFAULT_BRANCH=main

# .gitea/branch-protection.yaml（示意，实际通过 API 创建）
branch: main
required_approvals: 1
enable_status_check: true
status_check_contexts:
  - "ci/build"
  - "ci/schema-check-pre"
block_on_outdated_branch: true
dismiss_stale_approvals: true
require_signed_commits: false

sources:
  - name: "main_repo"
    type: "codeup"
    data:
      repo: "https://codeup.aliyun.com/<org>/<repo>.git"
      branchesFilter: "feature/*,main"   # 必须字符串，不能嵌套对象
      cloneDepth: "0"                    # 必须字符串 "0"，整数 0 会被忽略
      triggerEvents:
        - "push"
        - "merge_request"

# us-only-template.yaml 简化版
sources:
  - name: "main"
    type: "codeup"
    data:
      repo: "https://codeup.aliyun.com/<org>/${PROJECT}.git"
      branchesFilter: "main"
      cloneDepth: "0"
      triggerEvents: ["push"]

stages:
  - name: build
    jobs:
      - name: build-image
        runsOn: "private/bZI4pCEcUjK5OwqK"
        steps:
          - component: "DockerBuildPush"
            inputs:
              dockerfilePath: "dockerfiles/${PROJECT}/${SERVICE}.Dockerfile"
              registryType: "ECR"
              imageName: "${PROJECT}/${SERVICE}"
              imageTag: "${COMMIT_SHORT}-${TIMESTAMP}"

  - name: schema-check-pre
    jobs:
      - name: warn-only
        steps:
          - run: "python3 scripts/schema_check.py --mode warning"
            continueOnError: true

  - name: deploy-qa
    jobs:
      - name: gitops-update
        steps:
          - run: |
              python3 scripts/deploy.py \
                --region us --env qa \
                --project ${PROJECT} --service ${SERVICE} \
                --tag ${COMMIT_SHORT}-${TIMESTAMP} \
                --action all

  - name: approval-pre
    jobs:
      - name: validate
        steps:
          - component: "ManualValidation"
            inputs:
              validators: ["alice@example.cn", "bob@example.cn"]
              mode: "OR"

  - name: deploy-pre
    jobs:
      - name: gitops-update
        steps:
          - run: |
              python3 scripts/deploy.py \
                --region us --env pre \
                --project ${PROJECT} --service ${SERVICE} \
                --tag ${COMMIT_SHORT}-${TIMESTAMP} \
                --action all

  - name: schema-check-post
    jobs:
      - name: fail-on-missing
        steps:
          - run: "python3 scripts/schema_check.py --mode fail"
            continueOnError: false

# .github/workflows/ci.yaml
name: CI
on:
  push:
    branches: [main]
  pull_request:

jobs:
  build:
    runs-on: ubuntu-22.04
    permissions:
      id-token: write   # 用 OIDC 拿 ECR token
      contents: read
    steps:
      - uses: actions/checkout@v4
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::<ACCOUNT_ID>:role/gh-actions-ecr
          aws-region: us-west-2
      - uses: aws-actions/amazon-ecr-login@v2
      - name: Build & push
        run: |
          IMAGE=<ACCOUNT_ID>.dkr.ecr.us-west-2.amazonaws.com/${{ github.event.repository.name }}
          TAG=${GITHUB_SHA::7}-$(date +%Y-%m-%d-%H-%M-%S)
          docker build -t $IMAGE:$TAG .
          docker push $IMAGE:$TAG
          echo "TAG=$TAG" >> $GITHUB_ENV

  schema-check-pre:
    needs: build
    runs-on: ubuntu-22.04
    continue-on-error: true   # warning 模式
    steps:
      - uses: actions/checkout@v4
      - run: python3 scripts/schema_check.py --mode warning

  deploy-qa:
    needs: schema-check-pre
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
        with:
          repository: <org>/gitops
          token: ${{ secrets.GITOPS_BOT_TOKEN }}
      - run: |
          python3 scripts/deploy.py --region us --env qa \
            --project ${{ github.event.repository.name }} \
            --service main --tag $TAG --action all

apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
  - deployment.yaml
  - service.yaml
  - pdb.yaml
commonLabels:
  app.kubernetes.io/name: service-foo
  app.kubernetes.io/part-of: product-foo
images:
  - name: service-foo
    newName: <ACCOUNT_ID>.dkr.ecr.us-west-2.amazonaws.com/product-foo/service-foo
    newTag: PLACEHOLDER   # overlay 覆盖

apiVersion: apps/v1
kind: Deployment
metadata:
  name: service-foo
spec:
  replicas: 2
  selector:
    matchLabels:
      app.kubernetes.io/name: service-foo
  template:
    metadata:
      labels:
        app.kubernetes.io/name: service-foo
    spec:
      containers:
        - name: app
          image: service-foo
          ports:
            - containerPort: 8080
          readinessProbe:
            httpGet: { path: /healthz, port: 8080 }
            initialDelaySeconds: 5
          resources:
            requests: { cpu: 100m, memory: 256Mi }
            limits:   { cpu: 1000m, memory: 1Gi }

apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: product-foo
resources:
  - ../../../../../../base/product-foo/service-foo
images:
  - name: service-foo
    newTag: f37386f-2026-04-29-19-05-17   # CI 写入
patches:
  - target:
      kind: Deployment
      name: service-foo
    patch: |
      - op: replace
        path: /spec/replicas
        value: 6
      - op: replace
        path: /spec/template/spec/containers/0/resources/requests/cpu
        value: 500m
      - op: replace
        path: /spec/template/spec/containers/0/resources/limits/memory
        value: 4Gi

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: product-foo
  namespace: argocd
spec:
  goTemplate: true
  goTemplateOptions: ["missingkey=error"]
  generators:
    - matrix:
        generators:
          - list:
              elements:
                - cluster: us-prod
                  url: https://eks-us-prod.example.aws.com
                  envOverlay: us-prod
                - cluster: cn-prod
                  url: https://ack-cn-prod.example.aws.com
                  envOverlay: cn-prod
                - cluster: us-qa
                  url: https://eks-us-qa.example.aws.com
                  envOverlay: us-qa
          - git:
              repoURL: ssh://git@codeup.aliyun.com/<org>/gitops.git
              revision: main
              directories:
                - path: "clusters/{{ .envOverlay }}/applications/product-foo/*"
  template:
    metadata:
      name: 'product-foo-{{ .path.basename }}-{{ .cluster }}'
      labels:
        product: product-foo
        cluster: '{{ .cluster }}'
    spec:
      project: default
      source:
        repoURL: ssh://git@codeup.aliyun.com/<org>/gitops.git
        targetRevision: main
        path: '{{ .path.path }}'
      destination:
        server: '{{ .url }}'
        namespace: product-foo
      syncPolicy:
        automated:
          prune: true
          selfHeal: false
        syncOptions:
          - CreateNamespace=true
          - ServerSideApply=true
        retry:
          limit: 5
          backoff:
            duration: 30s
            factor: 2
            maxDuration: 5m
      ignoreDifferences:
        - group: apps
          kind: Deployment
          jsonPointers:
            - /spec/replicas   # HPA 接管

# 一步到位：更新 tag → git push → ArgoCD sync → 等待 healthy
python3 deploy.py --region us --env qa \
  --project product-foo --service service-foo \
  --tag f37386f-2026-04-29-19-05-17 \
  --action all

#!/usr/bin/env python3
# scripts/deploy.py — CI 与 GitOps 的胶水
# 用法：deploy.py --region us --env qa --project foo --service bar --tag T --action all
import argparse, os, subprocess, sys, time, requests

ARGOCD_DEFAULT = ("ARGOCD_SERVER", "ARGOCD_TOKEN")        # 主 ArgoCD（阿里云）
ARGOCD_US = ("ARGOCD_SERVER_US", "ARGOCD_TOKEN_US")        # 辅 ArgoCD（AWS）

# 服务到 ArgoCD 实例的显式映射，避免按 region 推断出错
SERVICE_ARGOCD_MAP = {
    "p2s-api": "us",
    "infra-coredns": "us",
    # 其他服务都走 default
}

def overlay_path(args):
    if args.path_mode == "p2s":
        return f"clusters/{args.cluster}/applications/{args.service}"
    return f"clusters/{args.region}-{args.env}/applications/{args.project}/{args.service}"

def pick_argocd(args):
    if SERVICE_ARGOCD_MAP.get(args.service) == "us":
        return ARGOCD_US
    if args.path_mode == "p2s" and args.env in ("qa", "pre"):
        return ARGOCD_US
    return ARGOCD_DEFAULT

def update_overlay(args):
    path = overlay_path(args)
    kfile = f"{path}/kustomization.yaml"
    subprocess.check_call(["sed", "-i",
        f"s|newTag:.*|newTag: {args.tag}|", kfile])
    subprocess.check_call(["git", "add", kfile])
    subprocess.check_call(["git", "commit", "-m",
        f"deploy({args.service}): {args.tag} → {args.region}-{args.env}"])
    subprocess.check_call(["git", "push", "origin", "main"])

def argocd_sync(args):
    server_var, token_var = pick_argocd(args)
    server = os.environ[server_var]
    token = os.environ[token_var]
    app = f"{args.project}-{args.service}-{args.region}-{args.env}"
    r = requests.post(
        f"https://{server}/api/v1/applications/{app}/sync",
        headers={"Authorization": f"Bearer {token}"},
        json={"prune": True}, timeout=30)
    r.raise_for_status()
    print(f"[sync] {app} via {server_var}")

def wait_healthy(args, timeout=900):
    server_var, token_var = pick_argocd(args)
    server, token = os.environ[server_var], os.environ[token_var]
    app = f"{args.project}-{args.service}-{args.region}-{args.env}"
    deadline = time.time() + timeout
    while time.time() < deadline:
        r = requests.get(f"https://{server}/api/v1/applications/{app}",
                         headers={"Authorization": f"Bearer {token}"}, timeout=10)
        s = r.json()["status"]
        if (s["sync"]["status"] == "Synced"
                and s["health"]["status"] == "Healthy"
                and s.get("operationState", {}).get("syncResult", {}).get("revision", "")[:7] == args.tag.split("-")[0]):
            print("[wait] healthy"); return
        time.sleep(10)
    sys.exit(f"[wait] timeout after {timeout}s")

if __name__ == "__main__":
    p = argparse.ArgumentParser()
    p.add_argument("--region", required=True)
    p.add_argument("--env", required=True)
    p.add_argument("--project", required=True)
    p.add_argument("--service", required=True)
    p.add_argument("--tag", required=True)
    p.add_argument("--action", choices=["gitops", "sync", "wait", "all"], default="all")
    p.add_argument("--path-mode", choices=["standard", "p2s"], default="standard")
    p.add_argument("--cluster")  # path-mode=p2s 时必填
    args = p.parse_args()
    if args.action in ("gitops", "all"): update_overlay(args)
    if args.action in ("sync", "all"):   argocd_sync(args)
    if args.action in ("wait", "all"):   wait_healthy(args)

helm repo add argo https://argoproj.github.io/argo-helm
helm repo update

helm upgrade --install argocd argo/argo-cd \
  --version 7.7.0 \
  --namespace argocd --create-namespace \
  --set global.domain=argocd.example.cn \
  --set configs.params."server\.insecure"=true \
  --set server.ingress.enabled=false \
  --set redis-ha.enabled=true \
  --set controller.replicas=2 \
  --set repoServer.replicas=2 \
  --set applicationSet.replicas=2 \
  --set notifications.enabled=true \
  --wait --timeout 10m

# 拿到初始 admin 密码
kubectl -n argocd get secret argocd-initial-admin-secret \
  -o jsonpath='{.data.password}' | base64 -d ; echo

# 用 argocd CLI 登录
argocd login argocd.example.cn --username admin --password '<password>' --insecure

# 添加 GitOps 仓库（SSH key）
argocd repo add ssh://git@codeup.aliyun.com/<org>/gitops.git \
  --ssh-private-key-path ~/.ssh/argocd-deploy-key

# 添加目标集群（在 us-prod kubectl context 下）
argocd cluster add arn:aws:eks:us-west-2:<ACCOUNT_ID>:cluster/us-prod \
  --name us-prod --upsert

# infra/argocd-bootstrap/root-app.yaml
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: argocd-self
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "-10"
spec:
  project: default
  source:
    repoURL: ssh://git@codeup.aliyun.com/<org>/gitops.git
    targetRevision: main
    path: argocd/applicationsets
    directory:
      recurse: true
  destination:
    server: https://kubernetes.default.svc
    namespace: argocd
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - ServerSideApply=true

# 一次性 bootstrap：
kubectl apply -f infra/argocd-bootstrap/root-app.yaml

helm registry logout public.ecr.aws 2>/dev/null || true
helm upgrade --install karpenter \
  oci://public.ecr.aws/karpenter/karpenter \
  --version 1.0.6 \
  --namespace kube-system \
  --set settings.clusterName=us-prod \
  --set settings.interruptionQueue=karpenter-us-prod \
  --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=arn:aws:iam::<ACCOUNT_ID>:role/KarpenterController \
  --set replicas=2 \
  --wait

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

cat > /tmp/kps-values.yaml <<'EOF'
prometheus:
  prometheusSpec:
    retention: 15d
    retentionSize: 100GiB
    storageSpec:
      volumeClaimTemplate:
        spec:
          accessModes: ["ReadWriteOnce"]
          storageClassName: gp3
          resources:
            requests:
              storage: 200Gi
    resources:
      requests: { cpu: 1, memory: 4Gi }
      limits:   { cpu: 4, memory: 16Gi }
    externalLabels:
      cluster: us-prod
      region: us-west-2
    remoteWrite:
      - url: https://prom-central.example.aws.com/api/v1/write
        writeRelabelConfigs:
          - sourceLabels: [__name__]
            regex: 'up|kube_.*|node_.*|container_.*'
            action: keep

alertmanager:
  alertmanagerSpec:
    storage:
      volumeClaimTemplate:
        spec:
          accessModes: ["ReadWriteOnce"]
          storageClassName: gp3
          resources: { requests: { storage: 10Gi } }
  config:
    global:
      resolve_timeout: 5m
    route:
      receiver: dingtalk-default
      group_by: [alertname, cluster, service]
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 4h
      routes:
        - matchers: [severity="P0"]
          receiver: dingtalk-p0
          continue: true
        - matchers: [severity="P1"]
          receiver: dingtalk-p1
        - matchers: [severity="P2"]
          receiver: dingtalk-p2
          repeat_interval: 24h
        - matchers: [alertname="Watchdog"]
          receiver: 'null'
    receivers:
      - name: 'null'
      - name: dingtalk-default
        webhook_configs:
          - url: http://prometheus-alert.monitoring/prometheusalert?type=dd&tpl=default&ddurl=<DINGTALK_DEFAULT>
      - name: dingtalk-p0
        webhook_configs:
          - url: http://prometheus-alert.monitoring/prometheusalert?type=dd&tpl=p0&ddurl=<DINGTALK_P0>&phone=<ONCALL_PHONE>
      - name: dingtalk-p1
        webhook_configs:
          - url: http://prometheus-alert.monitoring/prometheusalert?type=dd&tpl=p1&ddurl=<DINGTALK_P1>
      - name: dingtalk-p2
        webhook_configs:
          - url: http://prometheus-alert.monitoring/prometheusalert?type=dd&tpl=p2&ddurl=<DINGTALK_P2>

grafana:
  adminPassword: <CHANGEME>
  persistence:
    enabled: true
    size: 10Gi
  ingress:
    enabled: true
    hosts: [grafana.example.cn]
EOF

helm upgrade --install kps prometheus-community/kube-prometheus-stack \
  --version 62.7.0 \
  --namespace monitoring --create-namespace \
  -f /tmp/kps-values.yaml \
  --wait --timeout 15m

helm repo add grafana https://grafana.github.io/helm-charts
helm upgrade --install loki grafana/loki \
  --version 6.16.0 \
  --namespace monitoring \
  --set deploymentMode=SimpleScalable \
  --set loki.schemaConfig.configs[0].from=2024-04-01 \
  --set loki.schemaConfig.configs[0].store=tsdb \
  --set loki.schemaConfig.configs[0].object_store=s3 \
  --set loki.schemaConfig.configs[0].schema=v13 \
  --set loki.storage.bucketNames.chunks=loki-chunks-us-prod \
  --set loki.storage.bucketNames.ruler=loki-ruler-us-prod \
  --set loki.storage.s3.region=us-west-2 \
  --wait

# 1. 改源 + push
cd /data/repos/gitops
vim argocd/applicationsets/product-foo.yaml
git commit -am "chore(argocd): adjust retry policy"
git push

# 2. 关键：手动 apply 到 cluster（在 ArgoCD 所在集群）
kubectl --context argocd-cluster apply \
  -f argocd/applicationsets/product-foo.yaml -n argocd

# 3. 验证 template 已更新
kubectl --context argocd-cluster get applicationset \
  -n argocd product-foo \
  -o jsonpath='{.spec.template.spec.syncPolicy.retry}'

#!/bin/bash
# scripts/dora-deploy-freq.sh
# 用法：./dora-deploy-freq.sh <owner/repo> <since-iso8601>
set -euo pipefail
REPO="${1:?repo}"; SINCE="${2:?since}"
gh api -X GET "repos/$REPO/deployments" \
  -f environment=production \
  --paginate \
  --jq ".[] | select(.created_at > \"$SINCE\") | .created_at" \
  | wc -l

#!/bin/bash
# scripts/dora-lead-time.sh
# 用法：./dora-lead-time.sh <gitops-repo-path> <last-N-deploys>
set -euo pipefail
REPO_PATH="${1:-/data/repos/gitops}"
N="${2:-20}"

cd "$REPO_PATH"
git log --grep '^deploy(' -n "$N" --pretty=format:'%H|%ct|%s' \
| while IFS='|' read -r sha ts subj; do
    # subj 例：deploy(service-foo): f37386f-2026-04-29-19-05-17 → us-prod
    img_tag=$(echo "$subj" | grep -oE '[0-9a-f]{7}-[0-9-]+' | head -1)
    img_sha=${img_tag%%-*}
    # 在源码仓库找对应 commit 的提交时间
    src_repo=$(echo "$subj" | grep -oE 'deploy\(([^)]+)' | sed 's/deploy(//')
    src_ts=$(cd "/data/repos/$src_repo" 2>/dev/null && git show -s --format=%ct "$img_sha" 2>/dev/null) || continue
    delta=$(( ts - src_ts ))
    printf "%s\t%dh%dm\n" "$img_tag" $((delta/3600)) $(((delta%3600)/60))
done | awk '{
  # 统计中位数
  n[NR]=$2;
  split($2, parts, /[hm]/); secs=parts[1]*3600+parts[2]*60;
  total+=secs; count++
}
END { printf "avg lead time: %dh%dm (n=%d)\n", int(total/count/3600), int((total/count)%3600/60), count }'

# Prometheus query: 变更失败率
sum(
  count_over_time(
    ALERTS{severity=~"P0|P1", alertstate="firing"}[24h]
  ) > 0
) / sum(
  count_over_time(
    deployment_event{environment="production"}[24h]
  )
)

# Alertmanager 告警恢复时间分布
histogram_quantile(0.5,
  sum by (le) (
    rate(alertmanager_alerts_resolved_duration_seconds_bucket{severity=~"P0|P1"}[7d])
  )
)