Inactive → Pending → Firing → Resolved
  ↑          ↑          ↑          ↑
未触发    条件满足    持续满足    条件恢复
         但未超时    超过阈值    告警解除

┌─────────────────────────────────────────────────────────────┐
│                    kube-prometheus-stack                    │
├─────────────────────────────────────────────────────────────┤
│  Prometheus Operator    │  Prometheus CRD 控制器            │
│  Prometheus Server      │  时序数据库与采集引擎              │
│  AlertManager           │  告警管理与通知路由               │
│  Grafana                │  可视化仪表盘                     │
│  Node Exporter          │  节点级系统指标采集               │
│  kube-state-metrics     │  K8s 资源状态指标                 │
│  Prometheus Adapter     │  K8s Metrics API 适配器           │
│  Blackbox Exporter      │  黑盒探测与端点监控               │
└─────────────────────────────────────────────────────────────┘

                              ┌─────────────────┐
                              │   User/Admin    │
                              └────────┬────────┘
                                       │
                    ┌──────────────────┼──────────────────┐
                    │                  │                  │
                    ▼                  ▼                  ▼
           ┌─────────────┐    ┌─────────────┐    ┌─────────────┐
           │   Grafana   │    │  Prometheus │    │ AlertManager│
           │   :3000     │    │   :9090     │    │   :9093     │
           └──────┬──────┘    └──────┬──────┘    └──────┬──────┘
                  │                  │                  │
                  │    ┌─────────────┴─────────────┐    │
                  │    │                           │    │
                  └────┤    Prometheus Operator    ├────┘
                       │                           │
                       └─────────────┬─────────────┘
                                     │
         ┌───────────────────────────┼───────────────────────────┐
         │                           │                           │
         ▼                           ▼                           ▼
┌─────────────────┐      ┌─────────────────┐      ┌─────────────────┐
│  ServiceMonitor │      │  PodMonitor     │      │ PrometheusRule  │
│  (服务发现)      │      │  (Pod发现)      │      │  (告警规则)      │
└────────┬────────┘      └────────┬────────┘      └─────────────────┘
         │                        │
         ▼                        ▼
┌─────────────────┐      ┌─────────────────┐
│  Target Apps    │      │  Node Exporter  │
│  (业务应用)      │      │  (节点指标)      │
└─────────────────┘      └─────────────────┘

┌──────────────┐     ┌──────────────┐     ┌──────────────┐     ┌──────────────┐
│  Target      │────▶│  Prometheus  │────▶│ AlertManager │────▶│  Receiver    │
│  (被监控端)   │Pull │  (采集存储)   │Alert│  (告警管理)   │Notify│  (通知渠道)   │
└──────────────┘     └──────┬───────┘     └──────────────┘     └──────────────┘
                            │Query
                            ▼
                     ┌──────────────┐
                     │   Grafana    │
                     │  (可视化)     │
                     └──────────────┘

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
helm pull prometheus-community/kube-prometheus-stack --version 59.1.0
tar xzvf kube-prometheus-stack-59.1.0.tgz
cd kube-prometheus-stack

#修改两个镜像源
vim values.yaml
##将
---
      image:
        registry: registry.k8s.io
        repository: ingress-nginx/kube-webhook-certgen
        tag: v20221220-controller-v1.5.1-58-g787ea74b6
##改成
---
      image:
        registry: registry.cn-hangzhou.aliyuncs.com/google_containers
        repository: kube-webhook-certgen
        tag: v20221220-controller-v1.5.1-58-g787ea74b6

vim charts/kube-state-metrics/values.yaml
#将
---
image:
  registry: registry.k8s.io
  repository: kube-state-metrics/kube-state-metrics
  # If unset use v + .Charts.appVersion
  tag: ""
#改成
---
image:
  registry: docker.io
  repository: bitnami/kube-state-metrics
  # If unset use v + .Charts.appVersion
  tag: "2.10.0"

#将prometheus和alertManager数据持久化
#需要提前部署nfs-provisioner和默认存储类
vim values.yaml
---
    storage:
      volumeClaimTemplate:
        spec:
          storageClassName: sc-nfs
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 1Gi
---
    storageSpec:
      volumeClaimTemplate:
        spec:
          storageClassName: sc-nfs
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 1Gi
---

#grafana数据持久化
vim charts/grafana/values.yaml
persistence:
  type: pvc
  enabled: true
  storageClassName: sc-nfs
  accessModes:
    - ReadWriteOnce
  size: 1Gi

#配置ingress
#需要修改的values.yaml地方如下：
  --set prometheus.ingress.enabled=true \
  --set prometheus.prometheusSpec.retention=7d \
  --set prometheus.ingress.hosts='{prometheus.hanxux.local}' \
  --set prometheus.ingress.paths='{/}' \
  --set prometheus.ingress.pathType=Prefix \
  --set alertmanager.ingress.enabled=true \
  --set alertmanager.ingress.hosts='{alertmanager.hanxux.local}' \
  --set alertmanager.ingress.paths='{/}' \
  --set alertmanager.ingress.pathType=Prefix \
  --set grafana.ingress.enabled=true \
  --set grafana.adminPassword=admin \
  --set grafana.ingress.hosts='{grafana.hanxux.local}' \
  --set grafana.ingress.paths='{/}' \
  --set grafana.ingress.pathType=Prefix . -f values.yaml

additionalDataSources:
    - name: Tempo
      type: tempo
      uid: tempo
      url: http://tempo.monitoring.svc:3100
      access: proxy
      version: 1
      jsonData:
        httpMethod: GET
        serviceMap:
          datasourceUid: 'prometheus'
        tracesToLogsV2: # https://grafana.com/docs/grafana/next/datasources/tempo/#trace-to-logs
          datasourceUid: 'Loki'
          spanStartTimeShift: '-5m'
          spanEndTimeShift: '5m'
          tags: [{ key: 'service.name', value: 'app' }, { key: 'host.name', value: 'pod' }]
    - name: Loki
      type: loki
      url: http://loki.monitoring.svc:3100
      access: proxy

# 这些设置表示提到的selector（rule、service monitor、pod monitor 和 scrape config）将有独立的配置，并且不会基于 Helm Chart默认 values。
ruleSelectorNilUsesHelmValues: false
serviceMonitorSelectorNilUsesHelmValues: false
podMonitorSelectorNilUsesHelmValues: false
probeSelectorNilUsesHelmValues: false
scrapeConfigSelectorNilUsesHelmValues: false

helm install kube-prometheus-stack -n monitoring --create-namespace . -f values.yaml

helm upgrade -i kube-prometheus-stack -n monitoring . -f values.yaml

#通过port forward转发svc端口来本地访问
kubectl port-forward svc/kube-prometheus-stack-prometheus -n monitoring 9000:9090
kubectl port-forward svc/kube-prometheus-stack-grafana -n monitoring 9000:3000
kubectl port-forward svc/kube-prometheus-stack-alertmanager  -n monitoring 9000:9093

helm uninstall kube-prometheus-stack -n monitoring #--dry-run

#CRDs created by this chart are not removed by default and should be manually cleaned up:
kubectl delete crd alertmanagerconfigs.monitoring.coreos.com
kubectl delete crd alertmanagers.monitoring.coreos.com
kubectl delete crd podmonitors.monitoring.coreos.com
kubectl delete crd probes.monitoring.coreos.com
kubectl delete crd prometheusagents.monitoring.coreos.com
kubectl delete crd prometheuses.monitoring.coreos.com
kubectl delete crd prometheusrules.monitoring.coreos.com
kubectl delete crd scrapeconfigs.monitoring.coreos.com
kubectl delete crd servicemonitors.monitoring.coreos.com
kubectl delete crd thanosrulers.monitoring.coreos.com

k get ing -n monitoring

NAME                                 CLASS   HOSTS                       ADDRESS          PORTS   AGE
kube-prometheus-stack-alertmanager   nginx   alertmanager.hanxux.local   172.16.183.101   80      10m
kube-prometheus-stack-grafana        nginx   grafana.hanxux.local        172.16.183.101   80      10m
kube-prometheus-stack-prometheus     nginx   prometheus.hanxux.local     172.16.183.101   80      10m

172.16.183.101 prometheus.hanxux.local alertmanager.hanxux.local grafana.hanxux.local

#查看grafana密码
kubectl get secrets  -n monitoring kube-prometheus-stack-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo

tee certificate-grafana.yaml <<'EOF'
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: cert-grafana
  namespace: monitoring
spec:
  secretName: grafana-tls-cert-secret
  privateKey:
    rotationPolicy: Always
  commonName: grafana.hanxux.local
  dnsNames:
    - grafana.hanxux.local
  usages:
    - digital signature
    - key encipherment
    - server auth
  issuerRef:
    name: selfsigned
    kind: ClusterIssuer
EOF

    ## TLS configuration for grafana Ingress
    ## Secret must be manually created in the namespace
    ##
    tls:
      - secretName: cert-grafana
        hosts:
        - grafana.hanxux.local

annotations:
  nginx.ingress.kubernetes.io/auth-url: "http://oauth2-proxy.oauth2-proxy.svc.cluster.local/oauth2/auth"
  nginx.ingress.kubernetes.io/auth-signin: "https://oauth2proxy.hanxux.local/oauth2/start?rd=https%3A%2F%2Fgrafana.hanxux.local"

./
├── Chart.yaml
├── dashboards #放各种dashboard的json文件
│   ├── aks.json
├── templates
│   └── grafana_dashboard.yaml
├── values
└── values.yaml #主要定义dashboard的名字和json模板路径

apiVersion: v2
name: grafana-dashboards-config
description: A Helm chart for Grafana Dashborads configuration
type: application
version: 1.0.0
appVersion: "1.0.0"

dashboards:
  - file_path: "dashboards/aks.json"
    dashboard_name: aks-platform

{{- range .Values.dashboards }}
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: "grafana-dashboard-{{ .dashboard_name -}}"
  namespace: monitoring
  labels:
    grafana_dashboard: "1"
data:
  grafana_dashboard_{{- .file_path | replace "dashboards/" "" }}: |-
{{ $.Files.Get .file_path | indent 4 }}
{{- end }}

helm upgrade -i grafana-dashboards-config -n monitoring . --values values.yaml

tee pg1-fatasource.yaml <<'EOF'
apiVersion: v1
kind: ConfigMap
metadata:
  name: postgres-event-datasource
  namespace: monitoring
  labels:
    grafana_datasource: "1"
data:
  postgres_event_datasource.yaml: |-
    apiVersion: 1
    datasources:
      - name: PostgresDB
        type: postgres
        access: proxy
        url: {{ .Values.pg1.db.url }}:{{ .Values.pg1.db.port }}
        database: {{ .Values.pg1.db.database }}
        user: {{ .Values.pg1.db.user }}
        secureJsonData:
          password: {{ .Values.pg1.db.password }}
        sslmode: verify-full
        jsonData:
          postgresVersion: 1400 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10
          timescaledb: true
EOF

apiVersion: monitoring.coreos.com/v1 
kind: PodMonitor 
metadata: 
  name: example-app 
  labels: 
    team: frontend 
spec: 
  selector: 
    matchLabels: 
      app: example-app 
  podMetricsEndpoints: 
  - targetPort: 8080 

apiVersion: monitoring.coreos.com/v1
kind: Probe
metadata:
  name: probe-test
  namespace: monitoring
spec:
  jobName: probe-test
  interval: 30s
  module: http_2xx
  prober:
    url: http://blackbox-exporter.monitoring:9115/probe # 专门监控域名的exporter
    scheme: http
    path: /probe
    params:
      module: [http_2xx]
  targets:
    staticConfig:
      static:
      - https://www.kubeasy.com

apiVersion: monitoring.coreos.con/v1alpha1
kind: ScrapeConfig
metadata:
  name: redis-exporter
  namespace: monitoring
spec:
  scrapeInterval: 30s
  jobName: redis-exporter
  metricsPath: /scrape
  scheme: HTTP
  staticConfigs:
  - targets:
    - redis://redis.default:6379 # 写exporter的地址
    labels:
      env: test
    relabelings: # 替换掉prometheus中默认的标签
      - sourceLabels: [__address__]
        targetLabel: __param_target
      - sourceLabels: [__param_target]
        targetLabel: instance
      - targetLabel: __address__
        replacement: redis-exporter.monitoring:9121

apiVersion: v2
name: commoninfra-kube-prometheus-config
description: A Helm chart for kube prometheus configurations
type: application
version: 1.0.0
appVersion: "1.0.0"

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: loki
  namespace: monitoring
spec:
  groups: # 对不同类型的告警规则进行监控。建议对不同种类监控，直接创建新的prometheusRule yaml。
    - name: loki-prometheus # 定义规则组的名称，建议同一个yaml里面就写一个group就行了
      rules: # 规则列表，定义具体的告警规则
        - alert: LokiStorageHigh #告警名称
          expr: kubelet_volume_stats_available_bytes{namespace="monitoring", persistentvolumeclaim=~"storage-loki.*"} / 1024 / 1024 / 1024 < 5 #PromQL表达式，描述触发告警的条件
          labels:
            severity: warning
            namespace: monitoring
          annotations:
            summary: "Loki storage is high"
            description: "Loki storage is high. Please review retention defaults"
            runbook: https://xxx
        - alert: LokiStorageVeryHigh
          expr: kubelet_volume_stats_available_bytes{namespace="monitoring", persistentvolumeclaim=~"storage-loki.*"} / 1024 / 1024 / 1024 < 1
          labels:
            severity: warning
            namespace: monitoring
          annotations:
            summary: "Loki storage is very high"
            description: "Loki storage is very high and is about to auto-expand. Please consider changing retention/storage/alert defaults"
            runbook:  https://xxxx
        - alert: PrometheusStorageHigh
          expr: kubelet_volume_stats_available_bytes{namespace="monitoring", persistentvolumeclaim=~"prometheus-kube-prometheus-stack-prometheus-db.*"} / 1024 / 1024 / 1024 < 5
          labels:
            severity: warning
            namespace: monitoring
          annotations:
            summary: "Prometheus storage is high"
            description: "Prometheus storage is high. Please consider adding more storage"
            runbook: https://xxxxx
        - alert: PrometheusStorageVeryHigh
          expr: kubelet_volume_stats_available_bytes{namespace="monitoring", persistentvolumeclaim=~"prometheus-kube-prometheus-stack-prometheus-db.*"} / 1024 / 1024 / 1024 < 1
          labels:
            severity: critical
            namespace: monitoring
          annotations:
            summary: "Prometheus storage is very high"
            description: "Prometheus storage is very high. Please add more storage"
            runbook: https://xxxxx

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: argocd-rules
  namespace: monitoring
  labels:
    prometheus: k8s
    role: alert-rules
    app.kubernetes.io/name: argocd
    app.kubernetes.io/part-of: argocd
spec:
  groups:
  - name: argocd.rules
    rules:
    # ArgoCD Application Controller 高延迟告警
    - alert: ArgoCDApplicationControllerHighLatency
      expr: histogram_quantile(0.95, rate(argocd_app_controller_reconciliation_duration_seconds_bucket[5m])) > 1
      for: 2m # 持续时长为2min才会触发告警
      labels:
        severity: warning
      annotations:
        summary: "ArgoCD Application Controller Reconciliation Latency High ({{ $labels.namespace }})"
        description: "The reconciliation latency for ArgoCD application controller in namespace {{ $labels.namespace }} is over 1 second for more than 2 minutes."

    # ArgoCD Server 高请求错误率告警
    - alert: ArgoCDServerHighRequestErrorRate
      expr: rate(argocd_server_http_requests_total{status=~"5.*"}[5m]) > 0.05
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: "ArgoCD Server High Error Rate ({{ $labels.namespace }})"
        description: "The ArgoCD server in namespace {{ $labels.namespace }} has an HTTP error rate over 5% for the past 5 minutes."

    # ArgoCD Repo Server 同步失败告警
    - alert: ArgoCDRepoServerSyncFailed
      expr: increase(argocd_repo_server_git_request_failures_total[5m]) > 5
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: "ArgoCD Repo Server Sync Failures ({{ $labels.namespace }})"
        description: "ArgoCD Repo Server in namespace {{ $labels.namespace }} has more than 5 sync failures in the last 5 minutes."

    # ArgoCD Dex Server 不可用告警
    - alert: ArgoCDDexServerDown
      expr: up{job="argocd-dex-server"} == 0
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: "ArgoCD Dex Server Down ({{ $labels.namespace }})"
        description: "The ArgoCD Dex Server in namespace {{ $labels.namespace }} is not running for more than 1 minute."

    # ArgoCD Application 运行失败告警
    - alert: ArgoCDApplicationOutOfSync
      expr: argocd_app_info{health_status!="Healthy"} > 0
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "ArgoCD Applications Out of Sync ({{ $labels.namespace }})"
        description: "There are applications in ArgoCD that are not in a healthy state for more than 5 minutes."

helm upgrade -i commoninfra-kube-prometheus-config -n kube-system . --values ./values/dev.chinanorth3.yaml

cd /etc/prometheus/rules/prometheus-kube-prometheus-stack-prometheus-rulefiles-0

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: mysql-exporter
  namespace: monitoring
  labels:
    app: mysql-exporter
    release: kube-prometheus-stack
spec:
  selector:
    matchLabels: { app: mysql-exporter }
  namespaceSelector:
    matchNames: [database]
  endpoints:
    - port: metrics
      path: /metrics
      interval: 30s
      metricRelabelings:
        - sourceLabels: [__name__]
          regex: '(mysql_global_status_.*|mysql_up|mysql_global_variables_.*)'
          action: keep

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: redis-exporter
  namespace: monitoring
  labels:
    app: redis-exporter
    release: kube-prometheus-stack
spec:
  selector:
    matchLabels: { app: redis-exporter }
  namespaceSelector:
    matchNames: [database]
  endpoints:
    - port: metrics
      path: /metrics
      interval: 30s

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: kafka-exporter
  namespace: monitoring
spec:
  selector:
    matchLabels: { app: kafka-exporter }
  namespaceSelector:
    matchNames: [messaging]
  endpoints:
    - port: metrics
      path: /metrics
      interval: 30s

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: elasticsearch-exporter
  namespace: monitoring
spec:
  selector:
    matchLabels: { app: elasticsearch-exporter }
  namespaceSelector:
    matchNames: [logging]
  endpoints:
    - port: metrics
      path: /metrics
      interval: 30s

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: postgres-exporter
  namespace: monitoring
spec:
  selector:
    matchLabels: { app: postgres-exporter }
  namespaceSelector:
    matchNames: [database]
  endpoints:
    - port: metrics
      path: /metrics
      interval: 30s

apiVersion: monitoring.coreos.con/v1alpha1
kind: AlertmanagerConfig
metadata:
  name: example-alertmanagerconfig
  namespace: monitoring
spec:
  route:
    receiver: team-email # 根路由，默认receiver。如果没有被下面的routes匹配到，就往这里发
    groupBy: [type] # 告警有不同的种类，比如cpu、内存等，他们都是属于某个节点的，就可以打包同一节点的告警（type=node），避免告警轰炸。
    routes:
    - matchers:
      - matchType: =
        name: serverity
        value: critical
      receiver: critical-email
    - continue: true
      matchers:
      - name: slack
        regex: true
        value: ^[@#a-z0-9][a-z0-9._-]*$
      receiver: slack-team-receiver
  receivers: # 上面的receiver都需要在这里有定义
  - name: team-email
    emailConfigs:
    - to: team@example.com
      from: alertmanager@example.com
      smarthost: smtp.example.com:587
      authUsername: user
      authPassword:
        secretKeyRef:
          name: email-secret
          key: password

    groups:
    - name: example
      rules:
      - alert: HighNginxServerRequests
        expr: sum(irate(nginx_server_requests{instance=~"192.168.40.180:9913", code="2xx"}[5m])) by (code)>1000
        for: 2s
        labels:
          severity: critical
        annotations:
          summary: "High Nginx Server Requests"
          description: "在最近2s钟时间,nginx服务请求数达到了1000次"
    - name: 物理节点状态-监控告警
      rules:
      - alert: 物理节点cpu使用率
        expr: 100-avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by(instance)*100 > 20
        for: 2s
        labels:
          severity: ccritical
        annotations:
          summary: "{{ $labels.instance }}cpu使用率过高"
          description: "{{ $labels.instance }}的cpu使用率超过20%,当前使用率[{{ $value }}],需要排查处理" 
      - alert: 物理节点内存使用率
        expr: (node_memory_MemTotal_bytes - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes)) / node_memory_MemTotal_bytes * 100 > 50
        for: 2s
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.instance }}内存使用率过高"
          description: "{{ $labels.instance }}的内存使用率超过60%,当前使用率[{{ $value }}],需要排查处理"
      - alert: InstanceDown
        expr: up == 0
        for: 2s
        labels:
          severity: critical
        annotations:   
          summary: "{{ $labels.instance }}: 服务器宕机"
          description: "{{ $labels.instance }}: 服务器延时超过2分钟"
      - alert: 物理节点磁盘的IO性能
        expr: 100-(avg(irate(node_disk_io_time_seconds_total[1m])) by(instance)* 100) < 60
        for: 2s
        labels:
          severity: critical
        annotations:
          summary: "{{$labels.mountpoint}} 流入磁盘IO使用率过高！"
          description: "{{$labels.mountpoint }} 流入磁盘IO大于60%(目前使用:{{$value}})"
      - alert: 入网流量带宽
        expr: ((sum(rate (node_network_receive_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance)) / 100) > 102400
        for: 2s
        labels:
          severity: critical
        annotations:
          summary: "{{$labels.mountpoint}} 流入网络带宽过高！"
          description: "{{$labels.mountpoint }}流入网络带宽持续5分钟高于100M. RX带宽使用率{{$value}}"
      - alert: 出网流量带宽
        expr: ((sum(rate (node_network_transmit_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance)) / 100) > 102400
        for: 2s
        labels:
          severity: critical
        annotations:
          summary: "{{$labels.mountpoint}} 流出网络带宽过高！"
          description: "{{$labels.mountpoint }}流出网络带宽持续5分钟高于100M. RX带宽使用率{{$value}}"
      - alert: TCP会话
        expr: node_netstat_Tcp_CurrEstab > 1000
        for: 2s
        labels:
          severity: critical
        annotations:
          summary: "{{$labels.mountpoint}} TCP_ESTABLISHED过高！"
          description: "{{$labels.mountpoint }} TCP_ESTABLISHED大于1000%(目前使用:{{$value}}%)"
      - alert: 磁盘容量
        expr: 100-(node_filesystem_free_bytes{fstype=~"ext4|xfs"}/node_filesystem_size_bytes {fstype=~"ext4|xfs"}*100) > 80
        for: 2s
        labels:
          severity: critical
        annotations:
          summary: "{{$labels.mountpoint}} 磁盘分区使用率过高！"
          description: "{{$labels.mountpoint }} 磁盘分区使用大于80%(目前使用:{{$value}}%)"
      - alert: apiserver的cpu使用率大于90%
        expr: rate(process_cpu_seconds_total{job=~"kubernetes-apiserver"}[1m]) * 100 > 0
        for: 2s
        labels:
          severity: warnning
        annotations:
          description: "{{$labels.instance}}的{{$labels.job}}组件的cpu使用率超过80%"
      - alert: 容器内存使用率
        expr: (container_memory_usage_bytes / container_spec_memory_limit_bytes) * 100 > 0
        for: 2s
        labels:
          severity: critical
        annotations:
          summary: "High Container Memory Usage"
          description: "Container memory usage is above 90%."

apiVersion: monitoring.coreos.com/v1 
kind: PrometheusRule 
metadata: 
  name: blackbox  
  namespace: monitoring 
  labels: 
    app.kubernetes.io/component: exporter 
    app.kubernetes.io/name: blackbox-exporter 
    prometheus: k8s 
    role: alert-rules 
spec: 
  groups: 
  - name: blackbox-exporter 
    rules: 
    - alert: DomainCannotAccess 
      annotations: 
        description:  域名：{{ $labels.instance }} 不可达 # 从指标的label中取值
        summary: 域名探测失败
      expr: probe_success == 0 
      for: 1m 
      labels: 
        severity: critical 
        type: blackbox 
    - alert: DomainAccessDelayExceeds1s 
      annotations: 
        description:  域名：{{ $labels.instance }} 探测延迟大于1秒，当前延迟为：{{ $value }} 
        summary: 域名探测，访问延迟超过1秒  
      expr: sum(probe_http_duration_seconds{job=~"blackbox"}) by (instance) > 1  
      for: 1m 
      labels: 
        severity: warning 
        type: blackbox 

apiVersion: monitoring.coreos.com/v1 
kind: PrometheusRule 
metadata: 
  labels: 
    prometheus: k8s 
    role: alert-rules 
  name: basic 
  namespace: monitoring 
spec: 
  groups: 
  - name: mysql # 按照组件类型去分组
    rules: 
    - alert: MySQLStatus 
      annotations: 
        description:  实例：{{ $labels.instance }} 故障 
        summary: MySQL 服务宕机 
      expr: mysql_up == 0 
      for: 1m 
      labels: 
        severity: critical 
        type: basic 
        component: mysql 
  - name: redis 
    rules: 
    - alert: RedisStatus 
      annotations: 
        description:  实例：{{ $labels.instance }} 故障 
        summary: Redis 服务宕机 
      expr: redis_up == 0 
      for: 1m 
      labels: 
        severity: critical 
        type: basic 
        component: redis 

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: prometheus-alerts
  namespace: monitoring
  labels:
    prometheus: k8s
    role: alert-rules
spec:
  groups:
    - name: prometheus.service
      rules:
        - alert: PrometheusConfigurationReloadFailure
          expr: prometheus_config_last_reload_successful != 1
          for: 0m
          labels: { severity: critical }
          annotations:
            summary: "Prometheus 配置重载失败"

        - alert: PrometheusNotConnectedToAlertmanagers
          expr: prometheus_notifications_alertmanagers_discovered < 1
          for: 5m
          labels: { severity: critical }
          annotations:
            summary: "Prometheus 未连接到 AlertManager"

    - name: prometheus.performance
      rules:
        - alert: PrometheusRuleEvaluationSlow
          expr: |
            prometheus_rule_group_last_duration_seconds
            / prometheus_rule_group_interval_seconds > 0.5
          for: 5m
          labels: { severity: warning }
          annotations:
            summary: "规则评估时间超过间隔的 50%"

        - alert: PrometheusTSDBWALCorruptions
          expr: tsdb_wal_corruptions_total > 0
          for: 0m
          labels: { severity: critical }
          annotations:
            summary: "Prometheus TSDB WAL 损坏"

        - alert: PrometheusNotIngestingSamples
          expr: rate(prometheus_tsdb_head_samples_appended_total[5m]) <= 0
          for: 10m
          labels: { severity: critical }
          annotations:
            summary: "过去 10 分钟未写入任何样本"

        - alert: PrometheusQueryErrors
          expr: |
            rate(prometheus_http_requests_total{handler="/api/v1/query",code=~"5.."}[5m])
            / rate(prometheus_http_requests_total{handler="/api/v1/query"}[5m]) > 0.05
          for: 5m
          labels: { severity: warning }
          annotations:
            summary: "查询 API 5xx 错误率超过 5%"

    - name: prometheus.storage
      rules:
        - alert: PrometheusStorageRunningOut
          expr: |
            (prometheus_tsdb_retention_limit_bytes - prometheus_tsdb_storage_blocks_bytes)
            / prometheus_tsdb_retention_limit_bytes < 0.1
          for: 5m
          labels: { severity: warning }
          annotations:
            summary: "Prometheus 存储空间剩余不足 10%"

# Global：全局配置，主要用来配置一些通用的配置，比如邮件通知的账号、密码、SMTP服务器、微信告警等。Global 块配置下的配置选项在本配置文件内的所有配置项下可见，但是文件内其它位置的子配置可以覆盖Global配置
global: 
  resolve_timeout: 5m 
  ... 
# Route：告警路由配置，用于告警信息的分组路由，可以将不同分组的告警发送给不同的收件人。比如将数据库告警发送给DBA，服务器告警发送给OPS
route: 
  receiver: Default 
  group_by: 
  - namespace 
  - job 
  - alertname 
  group_wait: 30s 
  group_interval: 5m 
  repeat_interval: 10m 
  routes: 
    - matchers: 
        - service=~"foo1|foo2|baz" 
      receiver: team-X-mails 
      routes: 
        - matchers: 
            - severity="critical" 
          receiver: team-X-pager 
# Inhibit_rules：告警抑制，主要用于减少告警的次数，防止“告警轰炸”。比如某个宿主机宕机，可能会引起容器重建、漂移、服务不可用等一系列问题，如果每个异常均有告警，会一次性发送很多告警，造成告警轰炸，并且也会干扰定位问题的思路，所以可以使用告警抑制，屏蔽由宿主机宕机引来的其他问题，只发送宿主机宕机的消息即可
inhibit_rules: 
  - source_matchers: [severity="critical"] 
    target_matchers: [severity="warning"] 
    equal: [alertname, cluster, service] 
#eceivers：告警收件人配置，每个receiver都有一个名字，经过route分组并且路由后需要指定一个receiver，就是在此位置配置的
receivers: 
- name: Default 
  email_configs: 
  - send_resolved: true 
    to: kubernetes_guide@163.com 
    from: kubernetes_guide@163.com 
    hello: 163.com 
    smarthost: smtp.163.com:465 
    auth_username: kubernetes_guide@163.com 
    auth_password: <secret> 
    headers: 
      From: kubernetes_guide@163.com 
      Subject: '{{ template "email.default.subject" . }}' 
      To: kubernetes_guide@163.com 
    html: '{{ template "email.default.html" . }}' 
    require_tls: false 
  - name: Watchdog 
  - name: Critical 
# Templates：用于放置自定义模板的位置
 templates: [] 

route: 
  # 告警的通知目标，需要和 receivers 配置中 name 进行匹配。、
  # 需要注意的是route.routes下也可以有receiver配置，优先级高于route.receiver配置的默认接收人。当告警没有匹配到子路由时，会使用route.receiver进行通知
  receiver: Default 
  # 分组配置，值类型为列表。比如配置成['job', 'severity']，代表告警信息包含job和severity标签的会进行分组，且标签的key和value都相同才会被分到一组
  group_by: 
  - namespace 
  - job 
  - alertname 
  routes: # 子路由树，用于详细的告警路由
  - matchers: # matchers：匹配规则
    - owner="team-X" 
    receiver: team-X-pager # 局部收件人 
    continue: false # 决定匹配到该路由后，是否继续后续匹配。默认为false，即匹配到后停止继续匹配
  - matchers: 
    - owner="team-Y" 
    receiver: team-Y-pager  
  group_wait: 30s # 告警通知等待。若一组新的告警产生，则会等group_wait后再发送通知，该功能主要用于当告警在很短时间内接连产生时，在group_wait内合并为单一的告警后再发送，防止告警过多，默认值30s
  group_interval: 5m # 同一组告警通知后，如果有新的告警添加到该组中，不会立即发送。而是等一段时间再发送。默认值为5m 
  repeat_interval: 10m # 如果一条告警通知已成功发送，且在间隔 repeat_interval 后，该告警仍然未被设置为resolved，则会再次发送该告警通知，默认值4h

# cat alertmanager-secret.yaml 
 alertmanager.yaml: |- 
    "global": 
      "resolve_timeout": "5m" 
      smtp_from: "xxx@163.com" 
      smtp_smarthost: "smtp.163.com:465" 
      smtp_hello: "163.com" 
      smtp_auth_username: "xxx@163.com" 
      smtp_auth_password: "xxx" 
      smtp_require_tls: false

"receivers": 
- "name": "Default" 
   "email_configs": # 代表使用邮件通知
   - to: "xxx@163.com" # 收件人，可以配置多个，逗号隔开
     send_resolved: true # 告警如果被解决是否发送解决通知

kubectl replace -f alertmanager-secret.yaml

vim kube-prometheus/manifests/alertmanager-alertmanager.yaml 
replicas: 1 
alertmanagerConfigSelector: 
  matchLabels: 
    alertmanagerConfig: example 

apiVersion: monitoring.coreos.com/v1alpha1 
kind: AlertmanagerConfig 
metadata: 
  name: blackbox 
  namespace: monitoring
  labels: 
    alertmanagerConfig: example
spec: 
  route: 
    groupBy: ['type'] # 因为产生的告警是有type标签的，用他来进行分组
    groupWait: 30s 
    groupInterval: 5m 
    repeatInterval: 12h 
    routes: 
    - matchers: 
      - matchType: = 
        name: type 
        value: blackbox 
      receiver: Default # 只有type=blackbox的告警，才会被Default这个receiver进行通知
  receivers: 
  - name: Default 
    emailConfigs: 
    - to: xxx@163.com 
      sendResolved: true

apiVersion: secrets-store.csi.x-k8s.io/v1
kind: SecretProviderClass
metadata:
  name: sp-alertmanager
  namespace: monitoring
spec:
  provider: azure
  parameters:
    usePodIdentity: "false"
    useVMManagedIdentity: "true" #采用MI的认证方式，secretProviderClass在azure中有默认创建的MI以供使用
    userAssignedIdentityID: <secretProviderIdentityId> # Set the clientID of the user-assigned managed identity to use, e.g. azurekeyvaultsecretsprovider-aks-commoninfra-dev-westeurope
    keyvaultName: <keyvaultName>
    cloudName: <cloudName>
    objects:  |
      array:
        - |
          objectName: <the object name of keyvault>
          objectType: secret
          objectVersion: ""

    tenantId: "<tenant-id>"
  # This creates an actual secret that we can use
  secretObjects:
  - secretName: <actual-secret-name>
    data:
    - key: <key-name-of-the-secret> # add a secret referencable with the same key as was in the keyvault
      objectName: <same-object-name-with-above-objectName>
    type: Opaque

#Opaque类型的secret只接受base64类型的string，先转一下token到base64
echo -n 'xxxxx' | base64

apiVersion: v1
kind: Secret
metadata:
  name: <secret-name>
  namespace: monitoring
type: Opaque
data:
  token:

alertmanager:
...
  ## Alertmanager configuration directives
  ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file
  ##      https://prometheus.io/webtools/alerting/routing-tree-editor/
  ##
  config:
    global:
      resolve_timeout: 5m
      slack_api_url: https://slack.com/api/chat.postMessage
      http_config:
        authorization:
          credentials_file: '/mnt/secrets-store/<key-name-of-secret>' #这里的文件名需要和secret的keyname相一致
    inhibit_rules:
      - source_matchers:
          - 'severity = critical'
        target_matchers:
          - 'severity =~ warning|info'
        equal:
          - 'namespace'
          - 'alertname'
      - source_matchers:
          - 'severity = warning'
        target_matchers:
          - 'severity = info'
        equal:
          - 'namespace'
          - 'alertname'
      - source_matchers:
      # alertmanager自带一条alert: InfoInhibitor，作用是压制Info level的alert。
      # 背景：info的alert正常情况下会noisy，但是每当有warning/critical的alert出现时，info又会提供有价值的信息。
      # 在同一个namespace下，InfoInhibitor会在Info alert触发之后触发，压制Info alert；但又会在有warning/critical出现后停止触发，不压制那时的Info alert。
      # Refer to: https://runbooks.prometheus-operator.dev/runbooks/general/infoinhibitor/
          - 'alertname = InfoInhibitor'
        target_matchers:
          - 'severity = info'
        equal:
          - 'namespace'
    route:
      receiver: 'null'
...
    receivers:
    - name: 'null'
    - name: platform-slack
      slack_configs:
        - channel: '#<channel-name>'
          send_resolved: true
          title: '{{ template "slack.xxx.title" . }}'
          text: '{{ template "slack.xxx.text" . }}'
    - name: platform-slack-info
      slack_configs:
        - channel: '#<channel-name>'
          send_resolved: true
          title: '{{ template "slack.xxx.title" . }}'
          text: '{{ template "slack.xxx.text" . }}'
    templates:
    - '/etc/alertmanager/config/*.tmpl'
...
  ## Alertmanager template files to format alerts
  ## By default, templateFiles are placed in /etc/alertmanager/config/ and if
  ## they have a .tmpl file suffix will be loaded. See config.templates above
  ## to change, add other suffixes. If adding other suffixes, be sure to update
  ## config.templates above to include those suffixes.
  ## ref: https://prometheus.io/docs/alerting/notifications/
  ##      https://prometheus.io/docs/alerting/notification_examples/
  ##
  templateFiles:
    default_title.tmpl: |-
      {{ define "slack.xxx.title" }}
      {{ .CommonLabels.alertname }} - {{ .Alerts.Firing | len }} in {{ .CommonLabels.namespace }}
      {{ end }}
    default_text.tmpl: |-
      {{ define "slack.xxx.text" }}
      {{ range .Alerts }}
      `{{ .Labels.severity }}` - {{ .Annotations.summary }}
      *Description:* {{ .Annotations.description }}
      {{ if match `^http.+` .GeneratorURL }}*Graph:* <{{ .GeneratorURL }}|:chart_with_upwards_trend:> {{ end }}{{ if .Annotations.runbook }}*Runbook:* <{{ .Annotations.runbook }}|:spiral_note_pad:>{{ end }}
      *Details:*
        {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}`
        {{ end }}
      {{ end }}
      {{ end }}

    # Additional volumes on the output StatefulSet definition.
    volumes:
      - name: <volume-name>
        csi:
          driver: secrets-store.csi.k8s.io
          readOnly: true
          volumeAttributes:
            secretProviderClass: "sp-alertmanager"

    # Additional VolumeMounts on the output StatefulSet definition.
    volumeMounts:
      - name: <volume-name>
        mountPath: "/mnt/secrets-store"
        readOnly: true

   "global": 
      "resolve_timeout": "5m" 
     ... 
      wechat_api_url: "https://qyapi.weixin.qq.com/cgi-bin/"  
      wechat_api_corp_id: "wwef86a30130f04f2b"

# kubectl replace -f kube-prometheus/manifests/alertmanager-secret.yaml 

kubectl create secret generic webchat-secret --from-literal=secret=xxx -n monitoring

apiVersion: monitoring.coreos.com/v1alpha1 
kind: AlertmanagerConfig 
metadata: 
  name: basic
  namespace: monitoring
  labels: 
    alertmanagerConfig: example 
spec: 
  route: 
    groupBy: ['type'] 
    groupWait: 30s 
    groupInterval: 5m 
    repeatInterval: 12h 
    routes: 
    - matchers: 
      - matchType: =  
        name: type 
        value: "basic" 
      receiver: Wechat 
  receivers: 
  - name: Wechat 
    wechatConfigs: 
      - sendResolved: true 
        toParty: "4" 
        toUser: '@all' 
        agentID: "1000008" 
        apiSecret:  
          key: "secret" 
          name: "webchat-secret" 

app.kubernetes.io/part-of: kube-prometheus 
app.kubernetes.io/version: 0.21.0 
name: alertmanager-main 
namespace: monitoring 
stringData: 
  wechat.tmpl: |- 
    {{ define "wechat.default.message" }} 
    {{- if gt (len .Alerts.Firing) 0 -}} 
    {{- range $index, $alert := .Alerts -}} 
    {{- if eq $index 0 }} 
    ==========异常告警========== 
    告警类型: {{ $alert.Labels.alertname }} 
    告警级别: {{ $alert.Labels.severity }} 
    告警详情: {{ $alert.Annotations.message }}{{ $alert.Annotations.description}};{{$alert .Annotations.summary}} 
    故障时间: {{ ($alert.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }} 
{{- if gt (len $alert.Labels.instance) 0 }} 
    实例信息: {{ $alert.Labels.instance }} 
    {{- end }} 
    {{- if gt (len $alert.Labels.namespace) 0 }} 
    命名空间: {{ $alert.Labels.namespace }} 
    {{- end }} 
    {{- if gt (len $alert.Labels.node) 0 }} 
    节点信息: {{ $alert.Labels.node }} 
    {{- end }} 
    {{- if gt (len $alert.Labels.pod) 0 }} 
    实例名称: {{ $alert.Labels.pod }} 
    {{- end }} 
    ============END============ 
    {{- end }} 
    {{- end }} 
    {{- end }} 
    {{- if gt (len .Alerts.Resolved) 0 -}} 
    {{- range $index, $alert := .Alerts -}} 
    {{- if eq $index 0 }} 
    ==========异常恢复========== 
    告警类型: {{ $alert.Labels.alertname }} 
    告警级别: {{ $alert.Labels.severity }} 
    告警详情: {{ $alert.Annotations.message }}{{ $alert.Annotations.description}};{{$alert .Annotations.summary}} 
    故障时间: {{ ($alert.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }} 
    恢复时间: {{ ($alert.EndsAt.Add 28800e9).Format "2006-01-02 15:04:05" }} 
    {{- if gt (len $alert.Labels.instance) 0 }} 
    实例信息: {{ $alert.Labels.instance }} 
    {{- end }} 
    {{- if gt (len $alert.Labels.namespace) 0 }} 
    命名空间: {{ $alert.Labels.namespace }} 
    {{- end }} 
    {{- if gt (len $alert.Labels.node) 0 }} 
    节点信息: {{ $alert.Labels.node }} 
    {{- end }} 
    {{- if gt (len $alert.Labels.pod) 0 }} 
    实例名称: {{ $alert.Labels.pod }} 
    {{- end }} 
    ============END============ 
    {{- end }} 
    {{- end }} 
    {{- end }} 
    {{- end }} 
  alertmanager.yaml: |- 
    "global": 
      "resolve_timeout": "5m" 
      ......

    templates: 
    - '/etc/alertmanager/config/*.tmpl' 
    "inhibit_rules": 
    ......

kubectl replace -f alertmanager-secret.yaml -n monitoring 

  receivers: 
  - name: Wechat 
    wechatConfigs: 
      - sendResolved: true 
        toParty: "4" 
        toUser: '@all' 
        agentID: "1000008" 
        apiSecret:  
          key: "secret" 
          name: "webchat-secret" 
        message: '{{ template "wechat.default.message" . }}'

kubectl replace -f basic-alertmanagerconfig.yaml  -n monitoring

git clone https://github.com/timonwong/prometheus-webhook-dingtalk.git
cd prometheus-webhook-dingtalk/contrib/k8s/ 
# 修改配置
vim config/config.yaml
# targets.webhook1.url写上webhook的url
# targets.webhook1.secret写上加签的值 （webhook1就写这俩字段）
# 如果有很多个钉钉群，就写上webhook2、3、4以此类推

kubectl kustomize | kubectl apply -f - -n monitoring

apiVersion: monitoring.coreos.com/v1alpha1 
kind: AlertmanagerConfig 
metadata: 
  name: dingding
  namespace: monitoring
  labels: 
    alertmanagerConfig: example 
spec: 
  route: 
    groupBy: ['alertname'] 
    groupWait: 1m 
    groupInterval: 1m 
    repeatInterval: 1m 
    routes: 
    - matchers: 
      - matchType: "=" 
        name: alertname 
        value: "Watchdog" 
      receiver: dingding-webhook1 
  receivers: 
  - name: dingding-webhook1 
    webhookConfigs: 
      - sendResolved: true 
        # 给webhook的pod的svc发请求即可
        url: http://alertmanager-webhook-dingtalk.monitoring.svc.cluster.local/dingtalk/webhook1/send

# values-production.yaml 中的 prometheusSpec 部分
prometheusSpec:
  retention: 30d
  retentionSize: 450GB              # 保留大小（优先触发）
  tsdb:
    outOfOrderTimeWindow: 0s
  additionalArgs:
    - name: storage.tsdb.min-block-duration
      value: 2h
    - name: storage.tsdb.max-block-duration
      value: 2h
    - name: storage.tsdb.wal-compression
      value: "true"
    - name: storage.tsdb.max-block-chunk-segment-size
      value: "512MB"
    - name: query.max-samples
      value: "50000000"
    - name: query.max-concurrency
      value: "20"
    - name: query.timeout
      value: "2m"
    - name: storage.remote.flush-deadline
      value: "1m"
    - name: rules.alert.for-outage-tolerance
      value: "1h"
    - name: rules.alert.for-grace-period
      value: "10m"
    - name: rules.alert.resend-delay
      value: "1m"

prometheusSpec:
  scrapeInterval: 30s
  evaluationInterval: 30s
  scrapeTimeout: 10s
  externalLabels:
    cluster: production
    replica: '{{.ExternalURL}}'
  enableFeatures:
    - exemplar-storage
    - memory-snapshot-on-shutdown
    - new-service-discovery-manager
    - remote-write-receiver

alertmanager:
  alertmanagerSpec:
    resources:
      limits: { cpu: 1000m, memory: 1Gi }
      requests: { cpu: 100m, memory: 256Mi }
    replicas: 3
    clusterAdvertiseAddress: $(POD_IP)
    additionalArgs:
      - name: cluster.gossip-interval
        value: "200ms"
      - name: cluster.pushpull-interval
        value: "1m0s"
      - name: silences.max-silences
        value: "10000"
      - name: silences.max-silence-size-bytes
        value: "10485760"  # 10MB

grafana:
  resources:
    limits: { cpu: 1000m, memory: 1Gi }
    requests: { cpu: 250m, memory: 512Mi }
  env:
    GF_DATABASE_MAX_OPEN_CONN: "100"
    GF_DATABASE_MAX_IDLE_CONN: "100"
    GF_SESSION_PROVIDER: "memory"
    GF_CACHE_ENABLED: "true"
    GF_LOG_LEVEL: "warn"
    GF_AUTH_ANONYMOUS_ENABLED: "false"
    GF_SECURITY_CSRF_ADDITIONAL_HEADERS: "X-Forwarded-Host"
    GF_SECURITY_STRICT_TRANSPORT_SECURITY: "true"
    GF_SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE_SECONDS: "86400"
    GF_SECURITY_X_CONTENT_TYPE_OPTIONS: "true"
    GF_SECURITY_X_XSS_PROTECTION: "true"

# 1. 使用 rate() 而不是 increase() 计算速率
rate(http_requests_total[5m])           # 推荐
# increase(http_requests_total[5m]) / 300  # 不推荐

# 2. 使用 irate() 用于快速变化的计数器
irate(http_requests_total[5m])

# 3. 避免高基数查询
topk(10, sum by(handler) (rate(http_requests_total[5m])))  # 推荐
# topk(10000, http_requests_total)                         # 不推荐

# 4. 使用 recording rules 预计算复杂查询

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: recording-rules
  namespace: monitoring
spec:
  groups:
    - name: http_requests
      interval: 30s
      rules:
        - record: job:http_requests_total:rate5m
          expr: sum by(job) (rate(http_requests_total[5m]))
        - record: job:http_request_duration_seconds:p95
          expr: |
            histogram_quantile(0.95,
              sum by(job, le) (rate(http_request_duration_seconds_bucket[5m]))
            )

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nfs-optimized
provisioner: nfs.csi.k8s.io
parameters:
  server: <nfs-server-ip>
  share: /data/k8s/nfs
mountOptions:
  - nfsvers=4.1
  - nconnect=16       # 多连接（Linux 5.3+）
  - hard
  - intr
  - rsize=1048576     # 读块大小 1MB
  - wsize=1048576     # 写块大小 1MB
  - noatime
  - nodiratime
reclaimPolicy: Retain
volumeBindingMode: WaitForFirstConsumer

prometheusSpec:
  storageSpec:
    volumeClaimTemplate:
      spec:
        storageClassName: nfs-client
        accessModes: ["ReadWriteOnce"]
        resources:
          requests:
            storage: 500Gi
  # 额外挂载本地 SSD 用于 WAL
  volumes:
    - name: wal-volume
      emptyDir:
        medium: Memory  # 或使用 hostPath 挂载本地 SSD
  volumeMounts:
    - name: wal-volume
      mountPath: /prometheus/wal

prometheusSpec:
  additionalArgs:
    - name: web.enable-http2
      value: "true"
    - name: web.enable-gzip
      value: "true"
    - name: web.max-connections
      value: "512"
    - name: web.read-timeout
      value: "5m"
    - name: web.write-timeout
      value: "5m"

┌─────────────────────────────────────────────────────────────────┐
│                     Load Balancer (Ingress)                     │
└─────────────────────────────┬───────────────────────────────────┘
                              │
        ┌─────────────────────┼─────────────────────┐
        │                     │                     │
        ▼                     ▼                     ▼
┌───────────────┐    ┌───────────────┐    ┌───────────────┐
│ Prometheus-0  │    │ Prometheus-1  │    │ Prometheus-2  │
│ (Shard 0)     │    │ (Shard 1)     │    │ (Shard 2)     │
└───────┬───────┘    └───────┬───────┘    └───────┬───────┘
        │                     │                     │
        └─────────────────────┼─────────────────────┘
                              │
                              ▼
                    ┌─────────────────┐
                    │  Thanos Query   │
                    │  (全局查询视图)  │
                    └────────┬────────┘
                             │
                    ┌────────▼────────┐
                    │   Thanos Store  │
                    │   (长期存储)     │
                    └─────────────────┘

thanosSidecar:
  enabled: true
  image: quay.io/thanos/thanos:v0.35.0
  objectStorageConfig:
    type: s3
    config:
      bucket: "thanos-bucket"
      endpoint: "s3.amazonaws.com"
      access_key: "your-access-key"
      secret_key: "your-secret-key"
      insecure: false
      http_config:
        idle_conn_timeout: 90s
        response_header_timeout: 2m
  resources:
    limits: { cpu: 1000m, memory: 1Gi }
    requests: { cpu: 100m, memory: 256Mi }

apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  name: global
  namespace: monitoring
spec:
  replicas: 2
  additionalScrapeConfigs:
    - job_name: 'federate'
      scrape_interval: 30s
      honor_labels: true
      metrics_path: '/federate'
      params:
        'match[]':
          - '{job="prometheus"}'
          - '{__name__=~"job:.*"}'
          - '{__name__=~"node_.*"}'
      static_configs:
        - targets:
            - 'prometheus-dc1:9090'
            - 'prometheus-dc2:9090'
            - 'prometheus-dc3:9090'

alertmanager:
  alertmanagerSpec:
    replicas: 3
    clusterAdvertiseAddress: $(POD_IP)
    affinity:
      podAntiAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
                - key: app.kubernetes.io/name
                  operator: In
                  values: [alertmanager]
            topologyKey: kubernetes.io/hostname
    tolerations:
      - key: "monitoring"
        operator: "Equal"
        value: "true"
        effect: "NoSchedule"

apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  name: cluster-prometheus
  namespace: monitoring
  labels:
    cluster: cluster-1
spec:
  externalLabels:
    cluster: cluster-1
    region: beijing
    datacenter: dc1
  remoteWrite:
    - url: "http://central-prometheus:9090/api/v1/write"
      queueConfig:
        maxSamplesPerSend: 1000
        maxShards: 200
        capacity: 2500
      writeRelabelConfigs:
        - sourceLabels: [__name__]
          regex: 'up|node_.*|container_.*|kube_.*'
          action: keep

# 查看 Prometheus Pod 状态与日志
kubectl get pods -n monitoring -l app.kubernetes.io/name=prometheus
kubectl logs -f prometheus-k8s-0 -n monitoring -c prometheus

# 查看配置
kubectl get secret prometheus-k8s -n monitoring -o jsonpath='{.data.prometheus\.yaml\.gz}' | base64 -d | gunzip

# 查看 targets（端口转发后访问 http://localhost:9090/targets）
kubectl port-forward svc/prometheus-kube-prometheus-prometheus 9090:9090 -n monitoring

# AlertManager / Grafana 日志
kubectl logs -f alertmanager-main-0 -n monitoring -c alertmanager
kubectl logs -f deployment/kube-prometheus-stack-grafana -n monitoring

# 检查存储
kubectl get pvc -n monitoring
kubectl get pv
kubectl get storageclass

# 1. 检查 ServiceMonitor 标签
kubectl get servicemonitor -n monitoring --show-labels

# 2. 检查 Prometheus 的 serviceMonitorSelector
kubectl get prometheus k8s -n monitoring -o yaml | grep -A 5 serviceMonitorSelector

# 3. 确保 ServiceMonitor 标签与 Prometheus selector 匹配

# 4. 检查 Service 标签和 Endpoints
kubectl get svc -n <namespace> --show-labels
kubectl get endpoints -n <namespace>

# 1. 检查告警规则是否加载（http://localhost:9090/rules）
# 2. 在 Graph 页面测试告警表达式
# 3. 检查告警状态（http://localhost:9090/alerts）
# 4. 检查 AlertManager 连接（http://localhost:9090/status）

# 1. 检查 NFS 服务器
showmount -e <nfs-server-ip>
# 2. 检查 NFS provisioner 日志
kubectl logs -f deployment/nfs-provisioner -n kube-system
# 3. 检查 PVC 事件
kubectl describe pvc <pvc-name> -n monitoring
# 4. 手动测试挂载
kubectl run test-nfs --rm -it --image=busybox -- mount -t nfs <nfs-server-ip>:/data /mnt

# 1. 增加内存限制
kubectl patch prometheus k8s -n monitoring --type merge -p '{"spec":{"resources":{"limits":{"memory":"32Gi"}}}}'
# 2. 减少抓取目标（调整 ServiceMonitor 的 namespaceSelector）
# 3. 增加抓取间隔（scrapeInterval 改为 60s）
# 4. 减少保留时间（retention 改为 15d）

# 修改 values.yaml 中的镜像地址为国内源
# prometheus.prometheusSpec.image.repository: registry.aliyuncs.com/prometheus/prometheus
# 或配置镜像拉取密钥
kubectl create secret docker-registry regcred \
  --docker-server=your-registry.com \
  --docker-username=username \
  --docker-password=password

# 1. 检查 Prometheus 内存使用
kubectl top pod -n monitoring -l app.kubernetes.io/name=prometheus
# 2. 检查 TSDB 状态
kubectl exec -it prometheus-k8s-0 -n monitoring -c prometheus -- wget -qO- http://localhost:9090/api/v1/status/tsdb
# 3. 检查目标数量
kubectl exec -it prometheus-k8s-0 -n monitoring -c prometheus -- wget -qO- http://localhost:9090/api/v1/targets | jq '.data.activeTargets | length'
# 4. 检查序列数量（在 Prometheus UI 执行）
# prometheus_tsdb_head_series

# 方法1：使用 Velero
velero backup create prometheus-backup --include-namespaces monitoring

# 方法2：手动备份
kubectl exec -it prometheus-k8s-0 -n monitoring -c prometheus -- tar czf /tmp/prometheus-backup.tar.gz /prometheus
kubectl cp monitoring/prometheus-k8s-0:/tmp/prometheus-backup.tar.gz ./prometheus-backup.tar.gz

# 恢复
kubectl scale sts prometheus-k8s --replicas=0 -n monitoring
# 恢复数据到 PVC ...
kubectl scale sts prometheus-k8s --replicas=2 -n monitoring

# 节点 CPU 使用率
100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# 节点内存使用率
(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100

# 节点磁盘使用率
(node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_avail_bytes{mountpoint="/"}) / node_filesystem_size_bytes{mountpoint="/"} * 100

# Pod CPU / 内存
rate(container_cpu_usage_seconds_total{container!=""}[5m])
container_memory_usage_bytes{container!=""}

# HTTP 请求速率 / 错误率
rate(http_requests_total[5m])
rate(http_requests_total{status=~"5.."}[5m]) / rate(http_requests_total[5m])

# P95 延迟
histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))

# Pod 重启次数
increase(kube_pod_container_status_restarts_total[1h])

# 预测磁盘 4 小时内写满
predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[1h], 4 * 3600) < 0