containerLogMaxSize: "20Mi"  # 单个文件最大20MiB
containerLogMaxFiles: 10     # 最多保留10个文件

/var/log/pods/<namespace>_<pod-name>_<uid>/<container>/<n>.log
/var/log/containers/<pod-name>_<namespace>_<container>-<container-id>.log  # 软链接

# fluent-bit DaemonSet 核心配置
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluent-bit
  namespace: logging
spec:
  selector:
    matchLabels:
      app: fluent-bit
  template:
    spec:
      serviceAccountName: fluent-bit
      tolerations:
        - key: node-role.kubernetes.io/control-plane
          effect: NoSchedule
      containers:
        - name: fluent-bit
          image: fluent/fluent-bit:3.2
          resources:
            requests:
              cpu: 100m
              memory: 128Mi
            limits:
              cpu: 500m
              memory: 512Mi
          volumeMounts:
            - name: varlog
              mountPath: /var/log
              readOnly: true
            - name: varlibdockercontainers
              mountPath: /var/lib/docker/containers
              readOnly: true
            - name: config
              mountPath: /fluent-bit/etc/
      volumes:
        - name: varlog
          hostPath:
            path: /var/log
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers
        - name: config
          configMap:
            name: fluent-bit-config

[INPUT]
    Name              tail
    Tag               kube.*
    Path              /var/log/containers/*.log
    Parser            cri
    DB                /var/log/flb_kube.db
    Mem_Buf_Limit     50MB
    Skip_Long_Lines   On
    Refresh_Interval  10

[FILTER]
    Name                kubernetes
    Match               kube.*
    Kube_URL            https://kubernetes.default.svc:443
    Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
    Kube_Tag_Prefix     kube.var.log.containers.
    Merge_Log           On
    Keep_Log            Off
    K8S-Logging.Parser  On
    K8S-Logging.Exclude On
    Labels              On
    Annotations         Off

[OUTPUT]
    Name            es
    Match           kube.*
    Host            elasticsearch-master
    Port            9200
    Index           k8s-logs
    Logstash_Format On
    Logstash_Prefix k8s
    Replace_Dots    On
    Retry_Limit     False
    tls             On
    tls.verify      Off

apiVersion: apps/v1
kind: Deployment
metadata:
  name: app-with-sidecar
spec:
  template:
    spec:
      containers:
        - name: app
          image: my-app:v1.0
          volumeMounts:
            - name: app-logs
              mountPath: /app/logs
        - name: log-collector
          image: fluent/fluent-bit:3.2
          resources:
            requests:
              cpu: 50m
              memory: 64Mi
            limits:
              cpu: 200m
              memory: 128Mi
          volumeMounts:
            - name: app-logs
              mountPath: /app/logs
              readOnly: true
            - name: fluent-bit-config
              mountPath: /fluent-bit/etc/
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
      volumes:
        - name: app-logs
          emptyDir: {}
        - name: fluent-bit-config
          configMap:
            name: sidecar-fluent-bit-config

# Python 示例：直接推送到 Loki
import logging
from logging_loki import LokiHandler

handler = LokiHandler(
    url="http://loki:3100/loki/api/v1/push",
    tags={"app": "my-service", "env": "prod"},
    auth=("admin", "password"),
    version="1",
)
logger = logging.getLogger(__name__)
logger.addHandler(handler)
logger.info("User login", extra={"user_id": "12345"})

# Istio 启用访问日志
apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
  name: mesh-logging
  namespace: istio-system
spec:
  accessLogging:
    - providers:
        - name: envoy
      disabled: false

# OpenTelemetry Collector 配置核心片段
receivers:
  filelog:
    include:
      - /var/log/pods/*/*/*.log
    operators:
      - type: regex_parser
        id: parser-containerd
        regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$'
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317

processors:
  batch:
    send_batch_size: 10000
    timeout: 5s
  k8sattributes:
    auth_type: serviceAccount
    extract:
      metadata:
        - k8s.pod.name
        - k8s.deployment.name
        - k8s.namespace.name
        - k8s.node.name

exporters:
  loki:
    endpoint: http://loki:3100/loki/api/v1/push
  elasticsearch:
    endpoint: https://elasticsearch:9200

service:
  pipelines:
    logs:
      receivers: [filelog, otlp]
      processors: [batch, k8sattributes]
      exporters: [loki]

[INPUT]
    Name          tail
    Mem_Buf_Limit 100MB
    Skip_Long_Lines On

[SERVICE]
    storage.type         filesystem
    storage.path         /var/log/flb-storage/
    storage.backlog.mem_limit 100M

# /var/lib/kubelet/config.yaml
containerLogMaxFiles: 20
containerLogMaxSize: 50Mi

[FILTER]
    Name                kubernetes
    Merge_Log           On
    Merge_Log_Key       app_log
    Keep_Log            Off

metadata:
  annotations:
    fluentbit.io/exclude: "true"

应用日志是否全部走 stdout？
├─ 是 → Fluent Bit DaemonSet
│   └─ 是否需要 Traces/Metrics 统一？
│       ├─ 是 → OpenTelemetry Collector
│       └─ 否 → Fluent Bit + Elasticsearch/Loki
└─ 否（部分写文件）→
    ├─ Pod 数量 < 50 → Sidecar 方案
    ├─ Pod 数量 > 50 → 应用改造为 stdout
    └─ 无法改造 → Sidecar + 注解排除 DaemonSet 重复采集

{
  "kind": "Event",
  "apiVersion": "audit.k8s.io/v1",
  "level": "RequestResponse",
  "auditID": "aBcDeFgHi",
  "stage": "ResponseComplete",
  "requestURI": "/api/v1/namespaces/prod/pods/order-service-6d8f9",
  "verb": "delete",
  "user": {
    "username": "zhangsan",
    "groups": ["system:authenticated"]
  },
  "sourceIPs": ["10.0.1.15"],
  "objectRef": {
    "resource": "pods",
    "namespace": "prod",
    "name": "order-service-6d8f9"
  },
  "responseStatus": {
    "code": 200
  },
  "requestObject": {},
  "responseObject": {},
  "timestamp": "2026-05-12T03:15:22Z"
}

# /etc/kubernetes/audit-policy.yaml
apiVersion: audit.k8s.io/v1
kind: Policy
# 默认规则：不记录
omitStages:
  - "RequestReceived"

rules:
  # 1. 在命名空间 default 中，不记录对 Pod 的请求
  - level: None
    namespaces: ["default"]
    resources:
      - group: ""
        resources: ["pods"]

  # 2. 记录对 Secret 和 ConfigMap 的所有请求（安全敏感）
  - level: RequestResponse
    resources:
      - group: ""
        resources: ["secrets", "configmaps"]

  # 3. 记录对 RBAC 相关资源的修改操作
  - level: RequestResponse
    verbs: ["create", "update", "patch", "delete"]
    resources:
      - group: "rbac.authorization.k8s.io"
        resources: ["*"]

  # 4. 记录生产命名空间的所有写操作
  - level: RequestResponse
    verbs: ["create", "update", "patch", "delete"]
    namespaces: ["prod", "prod-chn", "prodfeu-chn"]

  # 5. 健康检查等只读请求，只记录元数据
  - level: Metadata
    verbs: ["get", "list", "watch"]

  # 6. 节点（Node）相关操作，记录请求体
  - level: Request
    resources:
      - group: ""
        resources: ["nodes"]

  # 7. 默认规则：只读操记录元数据，写操作记录完整信息
  - level: Metadata
    verbs: ["get", "list", "watch"]
  - level: RequestResponse

# /etc/kubernetes/manifests/kube-apiserver.yaml
apiVersion: v1
kind: Pod
spec:
  containers:
    - name: kube-apiserver
      command:
        - kube-apiserver
        # 审计策略文件
        - --audit-policy-file=/etc/kubernetes/audit-policy.yaml
        # 审计日志输出文件
        - --audit-log-path=/var/log/kubernetes/audit/audit.log
        # 日志文件最大大小（MB）
        - --audit-log-maxsize=100
        # 保留的旧日志文件数量
        - --audit-log-maxbackup=10
        # 保留的天数
        - --audit-log-maxage=30
        # （可选）输出到webhook
        - --audit-webhook-config-file=/etc/kubernetes/audit-webhook.yaml

# /etc/logrotate.d/k8s-audit
/var/log/kubernetes/audit/audit.log {
    daily
    rotate 7
    compress
    delaycompress
    notifempty
    create 0644 root root
    postrotate
        systemctl reload kubelet
    endscript
}

K8s API Server
    │
    ├── 输出到文件（audit.log）
    │       └── Fluent Bit / Filebeat 采集
    │               └── Elasticsearch / Loki
    │                       └── Kibana / Grafana（可视化查询）
    │
    └── 输出到 Webhook（实时）
            └── 审计专用后端（如 Falco、自研系统）

# /etc/fluent-bit/conf.d/k8s-audit.conf
[INPUT]
    Name tail
    Path /var/log/kubernetes/audit/audit.log
    Parser k8s-audit
    Tag k8s.audit
    Refresh_Interval 5
    Mem_Buf_Limit 50MB

[OUTPUT]
    Name es
    Match k8s.audit
    Host elasticsearch.logging.svc
    Port 9200
    Index k8s-audit-logs

# /etc/fluent-bit/parsers.conf
[PARSER]
    Name k8s-audit
    Format json
    Time_Key timestamp
    Time_Format %Y-%m-%dT%H:%M:%SZ

# 在audit.log中搜索删除操作
grep '"verb":"delete"' /var/log/kubernetes/audit/audit.log | \
  jq 'select(.objectRef.name=="order-service-6d8f9") | 
       {time: .timestamp, user: .user.username, ip: .sourceIPs[0], verb: .verb, resource: .objectRef}'

# 统计每个用户的API调用次数
cat /var/log/kubernetes/audit/audit.log | \
  jq -r '.user.username + " " + .verb + " " + .requestURI' | \
  sort | uniq -c | sort -rn | head -20

# 查找system:anonymous用户的操作
grep '"username":"system:anonymous"' /var/log/kubernetes/audit/audit.log

# Prometheus + Grafana告警规则
groups:
  - name: k8s_audit_alerts
    rules:
      - alert: AnonymousAccessDetected
        expr: |
          sum(rate(k8s_audit_events_total{user="system:anonymous"}[5m])) > 0
        labels:
          severity: critical
        annotations:
          summary: "检测到匿名用户访问K8s API"

// Elasticsearch Watcher示例：告警敏感操作
{
  "trigger": {
    "schedule": { "interval": "1m" }
  },
  "input": {
    "search": {
      "request": {
        "indices": ["k8s-audit-logs"],
        "body": {
          "query": {
            "bool": {
              "must": [
                { "terms": { "verb": ["create", "update", "patch", "delete"] } },
                { "terms": { "objectRef.resource": ["secrets", "configmaps", "roles", "rolebindings"] } },
                { "range": { "@timestamp": { "gte": "now-1m" } } }
              ]
            }
          }
        }
      }
    }
  },
  "condition": { "compare": { "ctx.payload.hits.total": { "gt": 0 } } },
  "actions": { "send_email": { "email": { "to": ["sre@company.com"] } } }
}

# 对核心命名空间使用RequestResponse，其余使用Metadata
rules:
  - level: RequestResponse
    namespaces: ["prod", "prod-chn"]
  - level: Metadata
    namespaces: ["staging", "dev"]
  - level: None
    namespaces: ["kube-system"]  # kube-system内部通信量大，酌情过滤

# 对只读操作采样10%
rules:
  - level: Metadata
    verbs: ["get", "list", "watch"]
    omitStages:
      - "ResponseComplete"
    # 注意：原生K8s审计策略不支持采样
    # 如需采样，可通过webhook后端实现

# kube-apiserver配置：输出到webhook（非阻塞）
- --audit-webhook-config-file=/etc/kubernetes/audit-webhook.yaml
- --audit-webhook-mode=batch  # 批量发送，减少API Server阻塞

# OPA Gatekeeper策略示例：禁止在prod命名空间创建LoadBalancer类型Service
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sNoLoadBalancerInProd
metadata:
  name: no-lb-in-prod
spec:
  match:
    kinds:
      - apiGroups: [""]
        kinds: ["Service"]
    namespaces: ["prod", "prod-chn"]
  parameters:
    prohibitedType: "LoadBalancer"

# 1. 创建审计策略文件
cat > /etc/kubernetes/audit-policy.yaml <<EOF
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
  - level: Metadata
EOF

# 2. 修改kube-apiserver配置
# 添加 --audit-policy-file 和 --audit-log-path 参数

# 3. 重启kubelet使配置生效
systemctl restart kubelet

# 4. 验证审计日志
tail -f /var/log/kubernetes/audit/audit.log | jq .