“如何设计永不宕机的K8s集群？”这套生产级方案让面试官当场发offer！

1、多Master节点部署

# etcd配置（/etc/etcd/etcd.conf）ETCD_HEARTBEAT_INTERVAL="500ms"  ETCD_ELECTION_TIMEOUT="2500ms"  ETCD_MAX_REQUEST_BYTES="157286400"  # 提高大请求吞吐量  

# Nginx配置示例（健康检查与熔断）upstream kube-apiserver {  server 10.0.1.10:6443 max_fails=3 fail_timeout=10s;  server 10.0.2.10:6443 max_fails=3 fail_timeout=10s;  check interval=5000 rise=2 fall=3 timeout=3000 type=http;  check_http_send "GET /readyz HTTP/1.0\r\n\r\n";  check_http_expect_alive http_2xx http_3xx;}

所需etcd节点数 = (预期写入QPS × 平均请求大小) / (单节点最大吞吐量) + 冗余系数 

# /etc/etcd/etcd.conf  # 增加网络和磁盘吞吐  ETCD_HEARTBEAT_INTERVAL="500ms"  ETCD_ELECTION_TIMEOUT="2500ms"  ETCD_SNAPSHOT_COUNT="10000"  # 提高快照频率  

# 主节点切换频繁告警increase(etcd_server_leader_changes_seen_total[1h]) > 3  # 写入延迟过高告警  histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) > 1s  

# 从快照恢复etcd  ETCDCTL_API=3 etcdctl snapshot restore snapshot.db --data-dir /var/lib/etcd-new

3、Cluster Autoscaler高级策略

# 节点组配置（AWS EKS）- name: gpu-nodegroup    instanceTypes: ["p3.2xlarge"]    labels: { node.kubernetes.io/accelerator: "nvidia" }    taints: { dedicated=gpu:NoSchedule }    scalingConfig: { minSize: 1, maxSize: 5 }  

# 基于Prometheus的QPS扩缩容  metrics:  - type: Pods    pods:      metric:        name: http_requests_per_second      target:        type: AverageValue        averageValue: 500  

spec:    topologySpreadConstraints:    - maxSkew: 1      topologyKey: topology.kubernetes.io/zone      whenUnsatisfiable: DoNotSchedule  

# 节点打标签  kubectl label nodes gpu-node1 accelerator=nvidia  # 设置污点  kubectl taint nodes gpu-node1 dedicated=ai:NoSchedule  # Pod配置容忍度 + 资源请求  spec:    tolerations:      - key: "dedicated"        operator: "Equal"        value: "ai"        effect: "NoSchedule"    containers:      - resources:          limits:            nvidia.com/gpu: 1  

6、Cilium eBPF网络加速

helm install cilium cilium/cilium --namespace kube-system \    --set kubeProxyReplacement=strict \    --set k8sServiceHost=API_SERVER_IP \    --set k8sServicePort=6443  

cilium status  # 应显示 "KubeProxyReplacement: Strict"  

resource "aws_globalaccelerator_endpoint_group" "ingress" {    listener_arn = aws_globalaccelerator_listener.ingress.arn    endpoint_configuration {      endpoint_id = aws_lb.ingress.arn      weight      = 100    }  }  

8、Rook/Ceph生产级配置

apiVersion: ceph.rook.io/v1  kind: CephCluster  metadata:    name: rook-ceph  spec:    dataDirHostPath: /var/lib/rook    mon:      count: 3      allowMultiplePerNode: false    storage:      useAllNodes: false      nodes:      - name: "storage-node-1"        devices:        - name: "nvme0n1"  

velero schedule create daily-backup --schedule="0 3 * * *" \    --include-namespaces=production \    --ttl 168h  velero backup-location create secondary --provider aws \    --bucket velero-backup-dr \    --config region=eu-west-1  

velero install \    --provider aws \    --plugins velero/velero-plugin-for-aws:v1.5.0 \    --bucket velero-backups \    --backup-location-config region=us-west-2 \    --snapshot-location-config region=us-west-2 \    --use-volume-snapshots=false \    --secret-file ./credentials-velero  # 添加跨区域复制规则  velero backup-location create secondary \    --provider aws \    --bucket velero-backups \    --config region=us-east-1  

11、Thanos长期存储优化

# thanos-store.yaml  args:    - --retention.resolution-raw=14d    - --retention.resolution-5m=180d    - --objstore.config-file=/etc/thanos/s3.yml 

thanos query \    --http-address 0.0.0.0:10902 \    --store=thanos-store-01:10901 \    --store=thanos-store-02:10901  

# Fluentd配置（提取Kubernetes元数据）<filter kubernetes.**>    @type parser    key_name log    reserve_data true    <parse>      @type json    </parse>  </filter>  

13、OPA Gatekeeper策略库

apiVersion: constraints.gatekeeper.sh/v1beta1  kind: K8sPSPPrivilegedContainer  spec:    match:      kinds: [{ apiGroups: [""], kinds: ["Pod"] }]    parameters:      privileged: false

# Falco检测特权容器启动  falco -r /etc/falco/falco_rules.yaml \    -o json_output=true \    -o "webserver.enabled=true"  

# image_scan.rego  package kubernetes.admission  deny[msg] {    input.request.kind.kind == "Pod"    image := input.request.object.spec.containers[_].image    vuln_score := data.vulnerabilities[image].maxScore    vuln_score >= 7.0    msg := sprintf("镜像 %v 存在高危漏洞（CVSS评分 %.1f）", [image, vuln_score])  }  

apiVersion: types.kubefed.io/v1beta1  kind: FederatedService  metadata:    name: frontend  spec:    placement:      clusters:        - name: cluster-us        - name: cluster-eu    trafficSplit:      - cluster: cluster-us        weight: 70      - cluster: cluster-eu        weight: 30  

apiVersion: chaos-mesh.org/v1alpha1  kind: NetworkChaos  metadata:    name: simulate-az-failure  spec:    action: partition    mode: all    selector:      namespaces: [production]      labelSelectors:        "app": "frontend"    direction: both    duration: "10m"  

apiVersion: chaos-mesh.org/v1alpha1  kind: PodChaos  metadata:    name: kill-master  spec:    action: pod-kill    mode: one    selector:      namespaces: [kube-system]      labelSelectors:        "component": "kube-apiserver"    scheduler:      cron: "@every 10m"    duration: "5m" 

19、Kubecost多集群预算分配

apiVersion: kubecost.com/v1alpha1  kind: Budget  metadata:    name: team-budget  spec:    target:      namespace: team-a    amount:      value: 5000      currency: USD    period: monthly    notifications:      - threshold: 80%        message: "团队A的云资源消耗已达预算80%"  

20、Argo Rollouts金丝雀发布

apiVersion: argoproj.io/v1alpha1  kind: Rollout  spec:    strategy:      canary:        steps:          - setWeight: 10%          - pause: { duration: 5m }  # 监控业务指标          - setWeight: 50%          - pause: { duration: 30m } # 观察日志和性能          - setWeight: 100%    analysis:      templates:        - templateName: success-rate      args:        - name: service-name          value: my-service