# 1. 添加 Helm 仓库
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

# 2. 安装（使用 NodePort 暴露 Grafana）
helm install monitoring prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --create-namespace \
  --set grafana.service.type=NodePort \
  --set grafana.service.nodePort=30300 \
  --set prometheus.service.type=NodePort \
  --set prometheus.service.nodePort=30900 \
  --set alertmanager.service.type=NodePort \
  --set alertmanager.service.nodePort=30903

# 3. 等待所有 Pod 就绪
kubectl get pods -n monitoring -w
# prometheus-xxx, grafana-xxx, alertmanager-xxx, operator-xxx, node-exporter-xxx, kube-state-metrics-xxx

# 4. 查看 Service
kubectl get svc -n monitoring

# 1. 获取 Grafana 登录密码
kubectl get secret -n monitoring monitoring-grafana \
  -o jsonpath='{.data.admin-password}' | base64 -d
echo

# 2. 访问 Grafana（NodePort 30300）
echo "Grafana: http://<任意节点IP>:30300"
echo "User: admin"
echo "Password: <上面获取的密码>"

# 3. 访问 Prometheus（NodePort 30900）
echo "Prometheus: http://<任意节点IP>:30900"

# 4. 在 Prometheus 中查询一些指标：
# - up（所有目标状态）
# - kube_node_info（节点信息）
# - container_memory_usage_bytes（容器内存）

# 1. 部署一个带 metrics 的应用
kubectl create ns app-metrics

cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: metrics-app
  namespace: app-metrics
  labels:
    app: metrics-app
spec:
  replicas: 2
  selector:
    matchLabels:
      app: metrics-app
  template:
    metadata:
      labels:
        app: metrics-app
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "9113"
    spec:
      containers:
      - name: nginx
        image: nginx:alpine
        ports:
        - containerPort: 80
      - name: exporter
        image: nginx/nginx-prometheus-exporter:0.11
        args:
        - -nginx.scrape-uri=http://localhost/nginx_status
        ports:
        - containerPort: 9113
          name: metrics
EOF

kubectl expose deploy metrics-app -n app-metrics --port=9113 --name=metrics-app-svc

# 2. 创建 ServiceMonitor
cat <<EOF | kubectl apply -f -
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: metrics-app-monitor
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: metrics-app
  namespaceSelector:
    matchNames:
    - app-metrics
  endpoints:
  - port: metrics
    interval: 30s
EOF

# 3. 在 Prometheus Targets 中验证新目标已出现
kubectl port-forward -n monitoring svc/monitoring-prometheus 9090:9090 &
# 浏览器打开 http://localhost:9090/targets

# 创建 PrometheusRule
cat <<EOF | kubectl apply -f -
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: custom-alerts
  namespace: monitoring
spec:
  groups:
  - name: pod-alerts
    rules:
    - alert: HighPodRestarts
      expr: rate(kube_pod_container_status_restarts_total[15m]) > 0.05
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "Pod {{ \$labels.pod }} has high restart rate"
        description: "Pod {{ \$labels.pod }} in {{ \$labels.namespace }} restarted {{ \$value }} times in 15min"
    
    - alert: PodCrashLooping
      expr: kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"} > 0
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: "Pod {{ \$labels.pod }} is crash looping"
EOF

# 验证规则
kubectl get PrometheusRule -n monitoring
kubectl describe PrometheusRule custom-alerts -n monitoring

# 1. 检查 ServiceMonitor 是否创建
kubectl get servicemonitor -A

# 2. 检查 Prometheus 配置是否已加载
kubectl port-forward -n monitoring svc/monitoring-prometheus 9090:9090
# 访问 http://localhost:9090/config 查看 scrape_configs

# 3. 检查 Target 状态
# http://localhost:9090/targets

# 4. 标签是否匹配
kubectl get servicemonitor <name> -o yaml | grep -A10 selector
kubectl get svc <name> -o yaml | grep -A5 labels