# 创建故意崩溃的 Pod
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: crash-demo
spec:
  containers:
  - name: bad-app
    image: busybox:1.36
    command: ["sh", "-c", "echo 'Starting...'; sleep 5; exit 1"]
EOF

# 排查流程：
# Step 1: 确认状态
kubectl get pod crash-demo
# STATUS: CrashLoopBackOff

# Step 2: 查看事件
kubectl describe pod crash-demo | grep -A15 Events
# Back-off restarting failed container

# Step 3: 查看日志（包括上一次）
kubectl logs crash-demo
kubectl logs crash-demo --previous
# Starting...

# Step 4: 结论 → 应用启动后 exit 1，需要检查应用配置
kubectl delete pod crash-demo

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: oom-demo
spec:
  containers:
  - name: mem-hog
    image: busybox:1.36
    command:
    - sh
    - -c
    - dd if=/dev/zero of=/dev/shm/big bs=50M count=10
    resources:
      limits:
        memory: "100Mi"
EOF

# 排查
kubectl get pod oom-demo -w
# STATUS: OOMKilled

kubectl describe pod oom-demo | grep -A5 "State"
# Reason: OOMKilled

kubectl delete pod oom-demo

kubectl run bad-image --image=notexist/image:v99 --restart=Never

kubectl get pod bad-image
# STATUS: ImagePullBackOff / ErrImagePull

kubectl describe pod bad-image | grep -A5 Events
# Failed to pull image: notexist/image:v99

kubectl delete pod bad-image

# 模拟 Service 不通的场景
# 1. 创建后端但故意 label 不匹配
kubectl create deploy web-backend --image=nginx:alpine --replicas=2

# 2. 创建 Service 选错误的 label
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: bad-svc
spec:
  selector:
    app: non-existent     # 故意不匹配！
  ports:
  - port: 80
EOF

# 3. 排查
kubectl get endpoints bad-svc
# ENDPOINTS: <none>  ← 问题！

kubectl get pod -l app=web-backend --show-labels
# 实际 label: app=web-backend

kubectl get svc bad-svc -o jsonpath='{.spec.selector}'
# {"app":"non-existent"}

# 4. 修复
kubectl patch svc bad-svc -p '{"spec":{"selector":{"app":"web-backend"}}}'
kubectl get endpoints bad-svc
# 现在有 ENDPOINTS 了

# 5. 清理
kubectl delete deploy web-backend
kubectl delete svc bad-svc

# 模拟节点问题
# 1. 对节点加污点使其不可调度
kubectl taint node k8s-node1 test=block:NoSchedule

# 2. 创建 Pod 观察现象
kubectl run taint-test --image=nginx:alpine
kubectl get pod taint-test -o wide
# 被调度到其他节点

# 3. 如果所有节点都有污点？
kubectl describe pod taint-test | grep -A10 Events
# Warning  FailedScheduling  ...

# 4. 排查节点状况
kubectl describe node k8s-node1 | grep -A10 Taints
kubectl describe node k8s-node1 | grep -A5 Conditions
# MemoryPressure / DiskPressure / PIDPressure

# 5. 恢复
kubectl taint node k8s-node1 test-
kubectl delete pod taint-test

# 模拟 PVC 无法绑定
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: impossible-pvc
spec:
  accessModes: ["ReadWriteMany"]
  resources:
    requests:
      storage: 100Ti    # 不可能有的容量
EOF

# 排查
kubectl get pvc impossible-pvc
# STATUS: Pending

kubectl describe pvc impossible-pvc | grep -A5 Events
# "no persistent volumes available..."

kubectl get pv
# 没有 100Ti 的 PV

kubectl delete pvc impossible-pvc

# 1. 创建一个多容器 Pod
cat <<'EOF' | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: log-demo
spec:
  containers:
  - name: app
    image: busybox:1.36
    command:
    - sh
    - -c
    - |
      i=0
      while true; do
        echo "[$(date +%T)][INFO] Application log entry $i"
        i=$((i+1))
        sleep 2
      done
  - name: sidecar
    image: busybox:1.36
    command:
    - sh
    - -c
    - |
      while true; do
        echo "[$(date +%T)][DEBUG] Sidecar heartbeat"
        sleep 5
      done
EOF

# 2. 基础操作
kubectl logs log-demo -c app --tail=5
kubectl logs log-demo -c sidecar --tail=3

# 3. 跟踪日志
kubectl logs log-demo -c app -f &
# Ctrl+C 停止

# 4. 查看所有容器日志
kubectl logs log-demo --all-containers --tail=10

# 5. 查看最近 2 分钟的日志
kubectl logs log-demo -c app --since=2m

# 6. 带时间戳
kubectl logs log-demo -c app --timestamps --tail=5

# 7. 查看上一次崩溃的日志
# 先模拟崩溃
kubectl run crash-log --image=busybox:1.36 --restart=Always -- /bin/sh -c 'echo "About to crash"; exit 1'

sleep 15
kubectl logs crash-log --previous
# 输出：About to crash

# 清理
kubectl delete pod log-demo crash-log

# stern 是比 kubectl logs 更好的多 Pod 日志工具

# 1. 安装 stern
# macOS
brew install stern
# Linux
wget https://github.com/stern/stern/releases/download/v1.28.0/stern_1.28.0_linux_amd64.tar.gz
tar -xzf stern_1.28.0_linux_amd64.tar.gz
sudo mv stern /usr/local/bin/

# 2. 创建多副本 Deployment
kubectl create deploy stern-test --image=busybox:1.36 --replicas=3 -- \
  sh -c 'while true; do echo "[$(hostname)] log at $(date +%T)"; sleep 3; done'

# 3. stern 同时查看所有副本日志
stern stern-test --tail=5
# 按颜色区分不同 Pod！

# 4. 按标签过滤
stern -l app=stern-test --since=1m

# 5. 清理
kubectl delete deploy stern-test

# 1. 添加 Fluent Helm 仓库
helm repo add fluent https://fluent.github.io/helm-charts
helm repo update

# 2. 安装 Fluent Bit
helm install fluent-bit fluent/fluent-bit \
  --namespace logging \
  --create-namespace \
  --set config.outputs.es.enabled=false \
  --set config.outputs.stdout.enabled=true \
  --set config.outputs.stdout.match="*"

# 3. 查看 Fluent Bit Pod
kubectl get pods -n logging
# fluent-bit-xxx（每个节点一个）

# 4. 查看 Fluent Bit 采集的日志（输出到 stdout）
kubectl logs -n logging -l app.kubernetes.io/name=fluent-bit --tail=20

# 5. 验证日志 Pipeline
kubectl get daemonset -n logging
kubectl get configmap -n logging
kubectl describe configmap -n logging fluent-bit

# 1. 所有容器的 stdout/stderr 日志在这里
# SSH 到任一 worker 节点：
ls /var/log/containers/
# 每个容器一个 symlink → /var/log/pods/

# 2. 容器日志存储格式
ls /var/log/pods/
# <namespace>_<pod-name>_<pod-uid>/<container-name>/

# 3. 查看 kubelet 日志
sudo journalctl -u kubelet -n 50 --no-pager

# 4. 查看容器运行时日志（containerd）
sudo journalctl -u containerd -n 30 --no-pager

# 5. 查看内核日志
sudo dmesg | tail -30

# 可能原因：
# 1. 应用没有输出到 stdout/stderr（写到了文件）
kubectl exec <pod> -- ls /var/log/app/

# 2. 日志被轮转删除了
# 3. 应用还没开始输出
kubectl logs <pod> -f

# 检查节点磁盘
df -h

# 检查容器日志大小
du -sh /var/log/containers/* | sort -rh | head -10

# 配置 kubelet 日志轮转（/var/lib/kubelet/config.yaml）
# containerLogMaxSize: "10Mi"
# containerLogMaxFiles: 5

# 1. 添加 Helm 仓库
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

# 2. 安装（使用 NodePort 暴露 Grafana）
helm install monitoring prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --create-namespace \
  --set grafana.service.type=NodePort \
  --set grafana.service.nodePort=30300 \
  --set prometheus.service.type=NodePort \
  --set prometheus.service.nodePort=30900 \
  --set alertmanager.service.type=NodePort \
  --set alertmanager.service.nodePort=30903

# 3. 等待所有 Pod 就绪
kubectl get pods -n monitoring -w
# prometheus-xxx, grafana-xxx, alertmanager-xxx, operator-xxx, node-exporter-xxx, kube-state-metrics-xxx

# 4. 查看 Service
kubectl get svc -n monitoring

# 1. 获取 Grafana 登录密码
kubectl get secret -n monitoring monitoring-grafana \
  -o jsonpath='{.data.admin-password}' | base64 -d
echo

# 2. 访问 Grafana（NodePort 30300）
echo "Grafana: http://<任意节点IP>:30300"
echo "User: admin"
echo "Password: <上面获取的密码>"

# 3. 访问 Prometheus（NodePort 30900）
echo "Prometheus: http://<任意节点IP>:30900"

# 4. 在 Prometheus 中查询一些指标：
# - up（所有目标状态）
# - kube_node_info（节点信息）
# - container_memory_usage_bytes（容器内存）

# 1. 部署一个带 metrics 的应用
kubectl create ns app-metrics

cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: metrics-app
  namespace: app-metrics
  labels:
    app: metrics-app
spec:
  replicas: 2
  selector:
    matchLabels:
      app: metrics-app
  template:
    metadata:
      labels:
        app: metrics-app
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "9113"
    spec:
      containers:
      - name: nginx
        image: nginx:alpine
        ports:
        - containerPort: 80
      - name: exporter
        image: nginx/nginx-prometheus-exporter:0.11
        args:
        - -nginx.scrape-uri=http://localhost/nginx_status
        ports:
        - containerPort: 9113
          name: metrics
EOF

kubectl expose deploy metrics-app -n app-metrics --port=9113 --name=metrics-app-svc

# 2. 创建 ServiceMonitor
cat <<EOF | kubectl apply -f -
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: metrics-app-monitor
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: metrics-app
  namespaceSelector:
    matchNames:
    - app-metrics
  endpoints:
  - port: metrics
    interval: 30s
EOF

# 3. 在 Prometheus Targets 中验证新目标已出现
kubectl port-forward -n monitoring svc/monitoring-prometheus 9090:9090 &
# 浏览器打开 http://localhost:9090/targets

# 创建 PrometheusRule
cat <<EOF | kubectl apply -f -
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: custom-alerts
  namespace: monitoring
spec:
  groups:
  - name: pod-alerts
    rules:
    - alert: HighPodRestarts
      expr: rate(kube_pod_container_status_restarts_total[15m]) > 0.05
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "Pod {{ \$labels.pod }} has high restart rate"
        description: "Pod {{ \$labels.pod }} in {{ \$labels.namespace }} restarted {{ \$value }} times in 15min"
    
    - alert: PodCrashLooping
      expr: kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"} > 0
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: "Pod {{ \$labels.pod }} is crash looping"
EOF

# 验证规则
kubectl get PrometheusRule -n monitoring
kubectl describe PrometheusRule custom-alerts -n monitoring

# 1. 检查 ServiceMonitor 是否创建
kubectl get servicemonitor -A

# 2. 检查 Prometheus 配置是否已加载
kubectl port-forward -n monitoring svc/monitoring-prometheus 9090:9090
# 访问 http://localhost:9090/config 查看 scrape_configs

# 3. 检查 Target 状态
# http://localhost:9090/targets

# 4. 标签是否匹配
kubectl get servicemonitor <name> -o yaml | grep -A10 selector
kubectl get svc <name> -o yaml | grep -A5 labels

# 1. 查看 Pod 默认挂载的 SA Token
kubectl run sa-test --image=busybox:1.36 --restart=Never -- sleep 3600

kubectl exec sa-test -- ls /var/run/secrets/kubernetes.io/serviceaccount/
# ca.crt  namespace  token

kubectl exec sa-test -- cat /var/run/secrets/kubernetes.io/serviceaccount/token | head -3

# 2. 禁用自动挂载
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: no-sa-mount
spec:
  automountServiceAccountToken: false
  containers:
  - name: app
    image: busybox:1.36
    command: ["sleep", "3600"]
EOF

kubectl exec no-sa-mount -- ls /var/run/secrets/kubernetes.io/serviceaccount/ 2>&1
# ls: /var/run/secrets/kubernetes.io/serviceaccount/: No such file or directory

# 3. 创建自定义 SA
kubectl create sa custom-sa
kubectl run custom-sa-pod --image=busybox:1.36 --restart=Never --overrides='
{
  "spec": {
    "serviceAccountName": "custom-sa"
  }
}' -- sleep 3600

kubectl get pod custom-sa-pod -o jsonpath='{.spec.serviceAccountName}'
# custom-sa

# 清理
kubectl delete pod sa-test no-sa-mount custom-sa-pod
kubectl delete sa custom-sa

# 1. 非 root 用户运行
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: non-root-pod
spec:
  securityContext:
    runAsUser: 1000
    runAsGroup: 3000
    fsGroup: 2000
  containers:
  - name: app
    image: nginx:alpine
    securityContext:
      allowPrivilegeEscalation: false
      readOnlyRootFilesystem: false
EOF

# 验证用户
kubectl exec non-root-pod -- id
# uid=1000 gid=3000

# 2. 设置 ReadOnlyRootFilesystem
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: readonly-pod
spec:
  containers:
  - name: app
    image: nginx:alpine
    securityContext:
      readOnlyRootFilesystem: true
    volumeMounts:
    - name: nginx-run
      mountPath: /var/run
    - name: nginx-cache
      mountPath: /var/cache/nginx
  volumes:
  - name: nginx-run
    emptyDir: {}
  - name: nginx-cache
    emptyDir: {}
EOF

kubectl exec readonly-pod -- touch /test-file
# touch: /test-file: Read-only file system

# 3. Capabilities 管理
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: caps-pod
spec:
  containers:
  - name: app
    image: busybox:1.36
    command: ["sleep", "3600"]
    securityContext:
      capabilities:
        drop:
        - ALL
        add:
        - NET_BIND_SERVICE
        - CHOWN
EOF

# 清理
kubectl delete pod non-root-pod readonly-pod caps-pod

# 1. 创建 Docker Registry Secret
kubectl create secret docker-registry private-registry \
  --docker-server=registry.cn-hangzhou.aliyuncs.com \
  --docker-username=your-username \
  --docker-password=your-password \
  --docker-email=your@email.com

# 2. 方式一：在 Pod 中指定
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: private-image-pod
spec:
  imagePullSecrets:
  - name: private-registry
  containers:
  - name: app
    image: registry.cn-hangzhou.aliyuncs.com/your-ns/your-image:tag
EOF

# 3. 方式二：绑定到 ServiceAccount（推荐）
kubectl patch serviceaccount default -p '{"imagePullSecrets":[{"name":"private-registry"}]}'

# 验证
kubectl get sa default -o yaml | grep -A3 imagePullSecrets

# 4. 清理
kubectl delete pod private-image-pod
kubectl delete secret private-registry
# 恢复默认 SA
kubectl patch serviceaccount default --type=json \
  -p='[{"op":"remove","path":"/imagePullSecrets"}]'

# 排查步骤
# 1. 查看错误详情
kubectl describe pod <pod-name> | grep -A10 Events

# 2. 是否是私有镜像？
# → 检查是否配置了 imagePullSecrets

# 3. Secret 是否正确？
kubectl get secret <secret-name> -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d | python3 -m json.tool

# 4. 镜像名是否正确？
# registry/namespace/image:tag

# 可能是因为 SecurityContext 限制了权限
kubectl logs <pod-name>
# Permission denied

# 检查 SecurityContext
kubectl get pod <pod-name> -o yaml | grep -A15 securityContext

kubectl create ns secure-stack

# === 1. ServiceAccount ===
kubectl create sa app-runner -n secure-stack

# === 2. RBAC（最小权限） ===
cat <<EOF | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: app-role
  namespace: secure-stack
rules:
- apiGroups: [""]
  resources: ["configmaps"]
  verbs: ["get", "list"]
  resourceNames: ["app-config"]     # 只能访问特定 ConfigMap
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: app-runner-binding
  namespace: secure-stack
subjects:
- kind: ServiceAccount
  name: app-runner
  namespace: secure-stack
roleRef:
  kind: Role
  name: app-role
  apiGroup: rbac.authorization.k8s.io
EOF

# === 3. Deploy + SecurityContext ===
kubectl create configmap app-config -n secure-stack \
  --from-literal=ENV=production

cat <<'EOF' | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: secure-web
  namespace: secure-stack
spec:
  replicas: 2
  selector:
    matchLabels:
      app: secure-web
  template:
    metadata:
      labels:
        app: secure-web
    spec:
      serviceAccountName: app-runner
      automountServiceAccountToken: false
      securityContext:
        runAsUser: 1001
        runAsNonRoot: true
        fsGroup: 1001
      containers:
      - name: web
        image: nginx:alpine
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: false
        ports:
        - containerPort: 80
        resources:
          requests:
            cpu: "100m"
            memory: "128Mi"
          limits:
            cpu: "200m"
            memory: "256Mi"
        livenessProbe:
          httpGet:
            path: /
            port: 80
        readinessProbe:
          httpGet:
            path: /
            port: 80
EOF

kubectl expose deploy secure-web -n secure-stack --port=80

# === 4. NetworkPolicy ===
cat <<EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: secure-web-policy
  namespace: secure-stack
spec:
  podSelector:
    matchLabels:
      app: secure-web
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: ingress-nginx
    ports:
    - protocol: TCP
      port: 80
  egress:
  - to:
    - namespaceSelector: {}
      podSelector:
        matchLabels:
          k8s-app: kube-dns
    ports:
    - protocol: UDP
      port: 53
EOF

# 验证部署
kubectl get all -n secure-stack
kubectl get networkpolicy -n secure-stack

# 1. 检查特权容器
kubectl get pods --all-namespaces -o json | \
  jq '.items[] | select(.spec.containers[].securityContext.privileged==true) | .metadata.name'

# 2. 检查以 root 运行的容器
kubectl get pods --all-namespaces -o json | \
  jq '[.items[] | select(.spec.containers[].securityContext.runAsNonRoot!=true)] | length'

# 3. 检查未设资源限制的 Pod
kubectl get pods --all-namespaces -o json | \
  jq '.items[] | select(.spec.containers[].resources.requests==null) | "\(.metadata.namespace)/\(.metadata.name)"'

# 4. 检查过于宽泛的 RBAC 权限
kubectl get clusterrolebindings -o json | \
  jq '.items[] | select(.subjects[]?.kind=="User" and .subjects[]?.name=="system:anonymous")'

# 5. 检查 Secret 是否明文（检查 etcd 加密状态）
kubectl get --raw=/api/v1/secrets | jq '.items[].type' | sort | uniq -c

# 6. 检查是否有 Pod 挂载了 hostPath
kubectl get pods --all-namespaces -o json | \
  jq '.items[] | select(.spec.volumes[]?.hostPath!=null) | "\(.metadata.namespace)/\(.metadata.name)"'

# 1. 创建命名空间
kubectl create ns rbac-demo

# 2. 创建 ServiceAccount
kubectl create sa developer -n rbac-demo

# 3. 创建 Role（只读 Pod）
cat <<EOF | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: pod-reader
  namespace: rbac-demo
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["get", "list", "watch"]
- apiGroups: [""]
  resources: ["pods/log"]
  verbs: ["get", "list"]
EOF

# 4. 创建 RoleBinding
cat <<EOF | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: developer-pod-reader
  namespace: rbac-demo
subjects:
- kind: ServiceAccount
  name: developer
  namespace: rbac-demo
roleRef:
  kind: Role
  name: pod-reader
  apiGroup: rbac.authorization.k8s.io
EOF

# 5. 验证权限
kubectl auth can-i get pods -n rbac-demo --as=system:serviceaccount:rbac-demo:developer
# yes

kubectl auth can-i create pods -n rbac-demo --as=system:serviceaccount:rbac-demo:developer
# no

kubectl auth can-i delete pods -n rbac-demo --as=system:serviceaccount:rbac-demo:developer
# no

# 6. 查看完整权限列表
kubectl auth can-i --list -n rbac-demo --as=system:serviceaccount:rbac-demo:developer

# 1. 创建 ClusterRole（集群级别只读）
cat <<EOF | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: cluster-viewer
rules:
- apiGroups: [""]
  resources: ["nodes", "namespaces", "pods", "services"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
  resources: ["deployments", "statefulsets", "daemonsets"]
  verbs: ["get", "list", "watch"]
EOF

# 2. 创建 ServiceAccount
kubectl create sa auditor -n rbac-demo

# 3. ClusterRoleBinding
cat <<EOF | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: auditor-viewer
subjects:
- kind: ServiceAccount
  name: auditor
  namespace: rbac-demo
roleRef:
  kind: ClusterRole
  name: cluster-viewer
  apiGroup: rbac.authorization.k8s.io
EOF

# 4. 验证集群级别权限
kubectl auth can-i get nodes --as=system:serviceaccount:rbac-demo:auditor
# yes

kubectl auth can-i get deploy --as=system:serviceaccount:rbac-demo:auditor
# yes

kubectl auth can-i delete nodes --as=system:serviceaccount:rbac-demo:auditor
# no