output.elasticsearch:
  hosts: ["192.168.40.180:9200"]

output.logstash:
  hosts: ["192.168.40.180:5044"]

output.kafka:
  enabled: true
  hosts: ["192.168.40.180:9092"]
  topic: elfk8stest

#安装nfs服务,nfs server端是master1节点
#所有node上yum安装nfs
yum install nfs-utils -y

#所有node上启动nfs服务
systemctl start nfs
systemctl enable nfs

#在master-1上创建nfs共享目录
mkdir /data/v1 -p

#编辑/etc/exports文件
vim /etc/exports
/data/v1 192.168.40.0/24(rw,no_root_squash)

#加载配置,使配置生效
exportfs -arv
systemctl restart nfs

#创建sa
apiVersion: v1
kind: ServiceAccount
metadata:
 name: nfs-provisioner
---
#创建nfs用的clusterrole
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
  - apiGroups: [""]
    resources: ["services", "endpoints"]
    verbs: ["get"]
  - apiGroups: ["extensions"]
    resources: ["podsecuritypolicies"]
    resourceNames: ["nfs-provisioner"]
    verbs: ["use"]
---
#给sa赋权
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-provisioner
    namespace: default
roleRef:
  kind: ClusterRole
  name: nfs-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-provisioner
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-provisioner
    namespace: default
roleRef:
  kind: Role
  name: leader-locking-nfs-provisioner
  apiGroup: rbac.authorization.k8s.io

spec:
  containers:
  - command:
  - kube-apiserver
  #添加这一行:
  - --feature-gates=RemoveSelfLink=false

#更新apiserver
kubectl apply -f /etc/kubernetes/manifests/kube-apiserver.yaml

kubectl delete pods kube-apiserver -n kube-system

kind: Deployment
apiVersion: apps/v1
metadata:
  name: nfs-provisioner
spec:
  selector:
    matchLabels:
      app: nfs-provisioner
  replicas: 1
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: nfs-provisioner
    spec:
      serviceAccount: nfs-provisioner
      containers:
        - name: nfs-provisioner
          #这个image需要先上传
          image: registry.cn-hangzhou.aliyuncs.com/open-ali/xianchao/nfs-client-provisioner:v1
          imagePullPolicy: IfNotPresent
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: example.com/nfs
            - name: NFS_SERVER
              value: 192.168.40.180 #nfs服务端ip
            - name: NFS_PATH
              value: /data/v1 #nfs服务端共享的目录
      volumes:
        - name: nfs-client-root
          nfs:
            server: 192.168.40.180
            path: /data/v1

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: do-block-storage
provisioner: example.com/nfs #该值需要和nfs provisioner deploy的env里配置的PROVISIONER_NAME值保持一致

#创建namespace
kubectl create ns kube-logging

#创建headless svc代理sts pod
kind: Service
apiVersion: v1
metadata:
  name: elasticsearch
  namespace: kube-logging
  labels:
    app: elasticsearch
spec:
  selector:
    app: elasticsearch
  clusterIP: None
  ports:
    - port: 9200
      name: rest
    - port: 9300
      name: inter-node
---
#创建sts
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: es-cluster #pod名称为es-cluster,可以通过es-cluster-[0,1,2].elasticsearch.kube-logging.svc.cluster.local,由headless svc解析到pod ip地址
  namespace: kube-logging
spec:
  serviceName: elasticsearch
  replicas: 3
  selector:
    matchLabels:
      app: elasticsearch
  template:
    metadata:
      labels:
        app: elasticsearch
    spec:
      containers:
      - name: elasticsearch
        image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0
        imagePullPolicy: IfNotPresent
        resources:
            limits:
              cpu: 1000m
            requests:
              cpu: 100m
        ports:
        - containerPort: 9200
          name: rest
          protocol: TCP
        - containerPort: 9300
          name: inter-node
          protocol: TCP
        volumeMounts:
        - name: data
          mountPath: /usr/share/elasticsearch/data
        env:
          - name: cluster.name
            value: k8s-logs
          - name: node.name
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
          - name: discovery.seed_hosts
            value: "es-cluster-0.elasticsearch.kube-logging.svc.cluster.local,es-cluster-1.elasticsearch.kube-logging.svc.cluster.local,es-cluster-2.elasticsearch.kube-logging.svc.cluster.local"
          #此字段用于设置在es集群中节点相互连接的发现方法,它为集群指定了一个静态主机列表。sts Pod具有唯一的 DNS 地址es-cluster-[0,1,2].elasticsearch.kube-logging.svc.cluster.local,因此我们相应地设置此地址变量即可。es发现的更多信息:https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html
          - name: cluster.initial_master_nodes
            value: "es-cluster-0,es-cluster-1,es-cluster-2"
          - name: ES_JAVA_OPTS
            value: "-Xms512m -Xmx512m"
            #这里我们设置为-Xms512m -Xmx512m,告诉JVM使用512 MB的最小和最大堆。
      initContainers:
      - name: fix-permissions
        image: busybox
        imagePullPolicy: IfNotPresent
        command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
        #将 Elasticsearch 数据目录的用户和组更改为1000:1000(es服务的 UID),pod里面运行的服务通常是uid 1000
        #默认k8s用root挂载数据目录,这会使得es无法访问数据目录,参考es中的默认注意事项:https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#_notes_for_production_use_and_defaults。
        securityContext:
          privileged: true
        volumeMounts:
        - name: data
          mountPath: /usr/share/elasticsearch/data
      - name: increase-vm-max-map
        image: busybox
        imagePullPolicy: IfNotPresent
        command: ["sysctl", "-w", "vm.max_map_count=262144"] #增加操作系统对mmap计数的限制,默认情况下该值可能太低,导致内存不足的错误:https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html
        securityContext:
          privileged: true
      - name: increase-fd-ulimit
        image: busybox
        imagePullPolicy: IfNotPresent
        command: ["sh", "-c", "ulimit -n 65536"] #增加打开文件描述符的最大数量
        securityContext:
          privileged: true
  volumeClaimTemplates:
  - metadata:
      name: data
      labels:
        app: elasticsearch
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: do-block-storage
      resources:
        requests:
          storage: 5Gi

kubectl port-forward es-cluster-0 9200:9200 --namespace=kube-logging

#本地主机的 9200 端口转发到es-cluster-0 的 9200 端口。可以通过访问本地主机的 9200 端口来访问 es-cluster-0 Pod 的 9200 端口提供的服务了。
#命令是临时的,ctrl+c一停止,端口转发就停止了。

curl http://localhost:9200/_cluster/state?pretty
#返回json文件,说明了es集群的状态

apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluentd
  namespace: kube-logging
  labels:
    app: fluentd
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluentd
  labels:
    app: fluentd
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - namespaces
  verbs:
  - get
  - list
  - watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: fluentd
roleRef:
  kind: ClusterRole
  name: fluentd
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
  name: fluentd
  namespace: kube-logging
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluentd
  namespace: kube-logging
  labels:
    app: fluentd
spec:
  selector:
    matchLabels:
      app: fluentd
  template:
    metadata:
      labels:
        app: fluentd
    spec:
      serviceAccount: fluentd
      serviceAccountName: fluentd
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
      containers:
      - name: fluentd
        image: fluent/fluentd-kubernetes-daemonset:v1.4.2-debian-elasticsearch-1.1
        #适用于containerd容器运行时的镜像:docker.io/fluent/fluentd-kubernetes-daemonset:v1.16-debian-elasticsearch7-1
        imagePullPolicy: IfNotPresent
        env:
          - name:  FLUENT_ELASTICSEARCH_HOST
            value: "elasticsearch.kube-logging.svc.cluster.local"
          - name:  FLUENT_ELASTICSEARCH_PORT
            value: "9200"
          - name: FLUENT_ELASTICSEARCH_SCHEME
            value: "http"
          - name: FLUENTD_SYSTEMD_CONF
            value: disable
          #禁用systemd日志收集,通常用于优化Fluentd的性能,因为收集和处理 systemd 的日志可能会消耗大量的 CPU 和内存资源。如果你的应用程序不依赖systemd 的日志,或者你有其他方式来收集这些日志,那么可以安全地禁用这个功能。
          #systemd 的日志是由 systemd 产生的,记录了系统服务的启动、运行和停止等信息。这些日志通常存储在 /var/log/journal/ 目录下
          #如果用containerd作为容器运行时,需要加上下面两行。
          #- name: FLUENT_CONTAINER_TAIL_PARSER_TYPE
          #  value: "cri"
          #- name: FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT
          #  value: "%Y-%m-%dT%H:%M:%S.%L%z"
        resources:
          limits:
            memory: 512Mi
          requests:
            cpu: 100m
            memory: 200Mi
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
      terminationGracePeriodSeconds: 30
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers

apiVersion: v1
kind: Service
metadata:
  name: kibana
  namespace: kube-logging
  labels:
    app: kibana
spec:
  ports:
  - port: 5601
  selector:
    app: kibana
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kibana
  namespace: kube-logging
  labels:
    app: kibana
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kibana
  template:
    metadata:
      labels:
        app: kibana
    spec:
      containers:
      - name: kibana
        image: docker.elastic.co/kibana/kibana:7.2.0
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            cpu: 1000m
          requests:
            cpu: 100m
        env:
          - name: ELASTICSEARCH_URL
            value: http://elasticsearch.kube-logging.svc.cluster.loccal:9200 #环境变量设置es暴露的rest接口的端口
        ports:
        - containerPort: 5601

#部署示例业务pod
apiVersion: apps/v1
kind: Deployment
metadata:
  name: dep-busybox
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: busybox
  template:
    metadata:
      labels:
        app: busybox
    spec:
      containers:
      - name: busybox
        image: busybox:latest
        imagePullPolicy: IfNotPresent
        args: ["/bin/sh", "-c", 'i=0; while true; do echo "$i: $(date)"; i=$((i+1)); sleep 1; done']
        volumeMounts:
        - name: localtime
          mountPath: /etc/localtime
      volumes:
      - name: localtime
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai

        volumeMounts:
        - name: localtime
          mountPath: /etc/localtime
      volumes:
      - name: localtime
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai

kubernetes.pod_name : dep-busybox-577ccf67c4-nqbcn