k8s企业级DevOps实践-Prometheus监控k8s集群、事件收集告警

apiVersion: v1
kind: Service
metadata:
  namespace: kube-system
  name: kube-controller-manager
  labels:
    k8s-app: kube-controller-manager
spec:
  type: ClusterIP
  clusterIP: None
  ports:
  - name: http-metrics
    port: 10252
    targetPort: 10252
    protocol: TCP

---
apiVersion: v1
kind: Endpoints
metadata:
  labels:
    k8s-app: kube-controller-manager
  name: kube-controller-manager
  namespace: kube-system
subsets:
- addresses:
  - ip: 172.19.244.101
  - ip: 172.19.244.102
  ports:
  - name: http-metrics
    port: 10252
    protocol: TCP

---

apiVersion: v1
kind: Service
metadata:
  namespace: kube-system
  name: kube-scheduler
  labels:
    k8s-app: kube-scheduler
spec:
  type: ClusterIP
  clusterIP: None
  ports:
  - name: http-metrics
    port: 10251
    targetPort: 10251
    protocol: TCP

---
apiVersion: v1
kind: Endpoints
metadata:
  labels:
    k8s-app: kube-scheduler
  name: kube-scheduler
  namespace: kube-system
subsets:
- addresses:
  - ip: 172.19.244.101
  - ip: 172.19.244.102
  ports:
  - name: http-metrics
    port: 10251
    protocol: TCP

# vim servicemonitor-service.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    app: ingress
  name: nginx-ingress-scraping
  namespace: ingress-nginx
spec:
  endpoints:
  - interval: 30s # 每30s获取一次信息
    path: /metrics
    port: http-metrics # 对应service的端口名
  jobLabel: app
  namespaceSelector: # 表示去匹配某一命名空间中的service，如果想从所有的namespace中匹配用any: true
    matchNames:
    - ingress-nginx
  selector: # 匹配的 Service 的labels，如果使用mathLabels，则下面的所有标签都匹配时才会匹配该service，如果使用matchExpressions，则至少匹配一个标签的service都会被选择
    matchLabels:
      k8s-app: ingress-nginx-metrics #匹配对应的Service
---
apiVersion: v1
kind: Service
metadata:
  namespace: ingress-nginx
  name: ingress-nginx-metrics
  labels:
    k8s-app: ingress-nginx-metrics
spec:
  type: ClusterIP
  clusterIP: None
  ports:
  - name: http-metrics
    port: 10254
    targetPort: 10254
    protocol: TCP
  selector:
    app.kubernetes.io/name: ingress-nginx

# kubectl edit clusterrole prometheus-k8s
---原始
rules:
- apiGroups:
  - ""
  resources:
  - nodes/metrics
  verbs:
  - get
- nonResourceURLs:
  - /metrics
  verbs:
  - get

---

---修改后
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  - services
  - endpoints
  - pods
  - nodes/proxy
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - configmaps
  - nodes/metrics
  verbs:
  - get
- nonResourceURLs:
  - /metrics
  verbs:
  - get

# vim prometheus-etcd.yaml 
apiVersion: v1
kind: Service
metadata:
  name: etcd-k8s
  namespace: monitoring
  labels:
    k8s-app: etcd
spec:
  type: ClusterIP
  clusterIP: None
  ports:
  - name: api
    port: 2379
    protocol: TCP
---
apiVersion: v1
kind: Endpoints
metadata:
  name: etcd-k8s
  namespace: monitoring
  labels:
    k8s-app: etcd
subsets:
- addresses:
  - ip: 172.19.244.101
  - ip: 172.19.244.102
  - ip: 172.19.244.103
  ports:
  - name: api
    port: 2379
    protocol: TCP
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: etcd-k8s
  namespace: monitoring
  labels:
    k8s-app: etcd-k8s
spec:
  jobLabel: k8s-app
  endpoints:
  - port: api
    interval: 30s
    scheme: https
    tlsConfig:
      caFile: /etc/prometheus/secrets/etcd-certs/ca.pem
      certFile: /etc/prometheus/secrets/etcd-certs/server.pem
      keyFile: /etc/prometheus/secrets/etcd-certs/server-key.pem
      #use insecureSkipVerify only if you cannot use a Subject Alternative Name
      insecureSkipVerify: true 
  selector:
    matchLabels:
      k8s-app: etcd
  namespaceSelector:
    matchNames:
    - monitoring

# 如上所说，我们就需要为两个副本创建两个pvc，命名分别为
prometheus-k8s-db-prometheus-k8s-0
prometheus-k8s-db-prometheus-k8s-1

mkdir -p prometheus/{prometheus-k8s-db-prometheus-k8s-0,prometheus-k8s-db-prometheus-k8s-1} && chmod -R 777 /data/prometheus/
# 创建两个pv-pvc存储卷
apiVersion: v1
kind: PersistentVolume
metadata:
  name: prometheus-k8s-db-prometheus-k8s-{1,0}
  labels:
    type: prometheus-k8s-db-prometheus-k8s-{1,0}
spec:
  capacity:
    storage: 1Pi
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  storageClassName: nfs
  nfs:
    path: /prometheus/prometheus-k8s-db-prometheus-k8s-{1,0}/
    server: 3xxxxxxxxnghai.nas.aliyuncs.com

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: prometheus-k8s-db-prometheus-k8s-{1,0}
  namespace: monitoring
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Pi
  storageClassName: nfs
  selector:
    matchLabels:
      type: prometheus-k8s-db-prometheus-k8s-{1,0}

[root@node001 ~]# kubectl -n monitoring get pvc
NAME                                 STATUS   VOLUME                               CAPACITY   ACCESS MODES   STORAGECLASS   AGE
prometheus-k8s-db-prometheus-k8s-0   Bound    prometheus-k8s-db-prometheus-k8s-0   1Pi        RWX            nfs            10m
prometheus-k8s-db-prometheus-k8s-1   Bound    prometheus-k8s-db-prometheus-k8s-1   1Pi        RWX            nfs            9m25s

# 准备prometheus持久化的pvc配置
# kubectl -n monitoring edit prometheus k8s

spec:
......
  storage:
    volumeClaimTemplate:
      spec:
        accessModes: [ "ReadWriteOnce" ]
        storageClassName: "nfs"
        resources:
          requests:
            storage: 1Pi

[root@node001 ~]# kubectl -n monitoring exec -it prometheus-k8s-0 -c prometheus -- sh
/prometheus $ df -Th
Filesystem           Type            Size      Used Available Use% Mounted on
...
39c39494bb-ctc52.cn-shanghai.nas.aliyuncs.com:/prometheus/prometheus-k8s-db-prometheus-k8s-0/prometheus-db
                     nfs4            1.0P    990.0M   1024.0T   0% /prometheus
...

mkdir /data/prometheus/grafana && chmod -R 777 /data/prometheus/grafana/
# 保存pvc为grafanapv-pvc.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: grafanapv
  labels:
    type: grafanapv
spec:
  capacity:
    storage: 1Pi
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  storageClassName: nfs
  nfs:
    path: /prometheus/grafana/
    server: 3xxxxxxxxnghai.nas.aliyuncs.com

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: grafana
  namespace: monitoring
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Pi
  storageClassName: nfs
  selector:
    matchLabels:
      type: grafanapv

# global块配置下的配置选项在本配置文件内的所有配置项下可见
global:
  # 在Alertmanager内管理的每一条告警均有两种状态: "resolved"或者"firing". 在altermanager首次发送告警通知后, 该告警会一直处于firing状态,设置resolve_timeout可以指定处于firing状态的告警间隔多长时间会被设置为resolved状态, 在设置为resolved状态的告警后,altermanager不会再发送firing的告警通知.
#  resolve_timeout: 1h
  resolve_timeout: 10m

  # 告警通知模板
templates:
- '/etc/altermanager/config/*.tmpl'

# route: 根路由,该模块用于该根路由下的节点及子路由routes的定义. 子树节点如果不对相关配置进行配置，则默认会从父路由树继承该配置选项。每一条告警都要进入route，即要求配置选项group_by的值能够匹配到每一条告警的至少一个labelkey(即通过POST请求向altermanager服务接口所发送告警的labels项所携带的<labelname>)，告警进入到route后，将会根据子路由routes节点中的配置项match_re或者match来确定能进入该子路由节点的告警(由在match_re或者match下配置的labelkey: labelvalue是否为告警labels的子集决定，是的话则会进入该子路由节点，否则不能接收进入该子路由节点).
route:
  # 例如所有labelkey:labelvalue含cluster=A及altertname=LatencyHigh labelkey的告警都会被归入单一组中
  group_by: ['job', 'altername', 'cluster', 'service','severity']
  # 若一组新的告警产生，则会等group_wait后再发送通知，该功能主要用于当告警在很短时间内接连产生时，在group_wait内合并为单一的告警后再发送
#  group_wait: 30s
  group_wait: 10s
  # 再次告警时间间隔
#  group_interval: 5m
  group_interval: 20s
  # 如果一条告警通知已成功发送，且在间隔repeat_interval后，该告警仍然未被设置为resolved，则会再次发送该告警通知
#  repeat_interval: 12h
  repeat_interval: 1m
  # 默认告警通知接收者，凡未被匹配进入各子路由节点的告警均被发送到此接收者
  receiver: 'webhook'
  # 上述route的配置会被传递给子路由节点，子路由节点进行重新配置才会被覆盖

  # 子路由树
  routes:
  # 该配置选项使用正则表达式来匹配告警的labels，以确定能否进入该子路由树
  # match_re和match均用于匹配labelkey为service,labelvalue分别为指定值的告警，被匹配到的告警会将通知发送到对应的receiver
  - match_re:
      service: ^(foo1|foo2|baz)$
    receiver: 'webhook'
    # 在带有service标签的告警同时有severity标签时，他可以有自己的子路由，同时具有severity != critical的告警则被发送给接收者team-ops-wechat,对severity == critical的告警则被发送到对应的接收者即team-ops-pager
    routes:
    - match:
        severity: critical
      receiver: 'webhook'
  # 比如关于数据库服务的告警，如果子路由没有匹配到相应的owner标签，则都默认由team-DB-pager接收
  - match:
      service: database
    receiver: 'webhook'
  # 我们也可以先根据标签service:database将数据库服务告警过滤出来，然后进一步将所有同时带labelkey为database
  - match:
      severity: critical
    receiver: 'webhook'
# 抑制规则，当出现critical告警时 忽略warning
inhibit_rules:
- source_match:
    severity: 'critical'
  target_match:
    severity: 'warning'
  # Apply inhibition if the alertname is the same.
  #   equal: ['alertname', 'cluster', 'service']
  #
# 收件人配置
receivers:
- name: 'webhook'
  webhook_configs:
  - url: 'http://alertmanaer-dingtalk-svc.kube-system/1bdc0637/prometheus/feishu'
    send_resolved: true

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    name: kube-eventer
  name: kube-eventer
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kube-eventer
  template:
    metadata:
      labels:
        app: kube-eventer
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      dnsPolicy: ClusterFirstWithHostNet
      serviceAccount: kube-eventer
      containers:
        - image: registry.aliyuncs.com/acs/kube-eventer-amd64:v1.2.0-484d9cd-aliyun
          name: kube-eventer
          command:
            - "/kube-eventer"
            - "--source=kubernetes:https://kubernetes.default"
            ## .e.g,dingtalk sink demo
            #- --sink=dingtalk:[your_webhook_url]&label=[your_cluster_id]&level=[Normal or Warning(default)]
            #- --sink=webhook:https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=07055f32-a04e-4ad7-9cb1-d22352769e1c&level=Warning&label=oa-k8s
            - --sink=webhook:https://open.feishu.cn/open-apis/bot/v2/hook/beb78afe-0658-47ef-a2d9-2xxxxxxx8?level=Normal&header=Convtent-Type=application/json&custom_body_configmap=custom-webhook-body&custom_body_configmap_namespace=kube-system&method=POST
            - --sink=webhook:https://open.feishu.cn/open-apis/bot/v2/hook/beb78afe-0658-47ef-a2d9-2xxxxxxx8?level=Warning&header=Convtent-Type=application/json&custom_body_configmap=custom-webhook-body&custom_body_configmap_namespace=kube-system&method=POST
          env:
          # If TZ is assigned, set the TZ value as the time zone
          - name: TZ
            value: "Asia/Shanghai"
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
              readOnly: true
            - name: zoneinfo
              mountPath: /usr/share/zoneinfo
              readOnly: true
          resources:
            requests:
              cpu: 100m
              memory: 100Mi
            limits:
              cpu: 500m
              memory: 250Mi
      volumes:
        - name: localtime
          hostPath:
            path: /etc/localtime
        - name: zoneinfo
          hostPath:
            path: /usr/share/zoneinfo
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: kube-eventer
rules:
  - apiGroups:
      - ""
    resources:
      - events
      - configmaps
    verbs:
      - get
      - list
      - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kube-eventer
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kube-eventer
subjects:
  - kind: ServiceAccount
    name: kube-eventer
    namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kube-eventer
  namespace: kube-system
---
apiVersion: v1
data:
  content: >-
    {"msg_type":"interactive","card":{"config":{"wide_screen_mode":true},"header":{"title":{"tag":"plain_text","content":"EventType:{{ .Type }} In Shanghai k8s"},"template":"blue"},"elements":[{"tag":"markdown","content":"**EventNamespace**:{{ .InvolvedObject.Namespace }} \n**EventKind**:{{ .InvolvedObject.Kind }} \n**EventObject**:{{ .InvolvedObject.Name }} \n**EventReason**:{{ .Reason }} \n**EventTime**:{{ .LastTimestamp }} \n**EventMessage**:{{ .Message }} \n[k8s面板](https://ops.xxxx.com:4433/)|[Grafana集群监控](https://ops.xxxx.com:4434/login) \n<at id=6833974120049278977></at><at id=6795355516563357697></at>\n ---"}]}}
kind: ConfigMap
metadata:
  name: custom-webhook-body
  namespace: kube-system
# es提交
# --sink=elasticsearch:http://es-cn-0pp16xvo90009zf49.public.elasticsearch.aliyuncs.com:9200?sniff=false&ver=6&index=kubesystem-eventmonitoring&healthCheck=false&cluster_name=sh&esUserName=felix&esUserSecret=felix123