关于kubernetes:k8sprometheus

promethus

基于 k8s

收集数据

node-exporter

vi node-exporter-ds.yml

apiVersion: extensions/v1beta1

kind: DaemonSet

metadata:

  name: node-exporter

  labels:

    app: node-exporter

spec:

  template:

    metadata:

      labels:

        app: node-exporter

    spec:

      hostNetwork: true

      containers:

      – image: prom/node-exporter

        name: node-exporter

        ports:

        – containerPort: 9100

        volumeMounts:

        – mountPath: “/etc/localtime”

          name: timezone

      volumes:

      – name: timezone

          hostPath:

            path: /etc/localtime

存储，长久卷，创立一个 10G 的 pv, 基于 nfs

vi prometheus-pv.yaml

apiVersion: v1

kind: PersistentVolume

metadata:

  name: gwj-pv-prometheus

  labels:

    app: gwj-pv

spec:

  capacity:

    storage: 10Gi

  volumeMode: Filesystem

  accessModes:

  – ReadWriteMany

  persistentVolumeReclaimPolicy: Recycle

  storageClassName: slow

  mountOptions:

  – hard

  – nfsvers=4.1

  nfs:

    path: /storage/gwj-prometheus

    server: 10.1.99.1

长久卷申领，基于刚刚创立的 pv，申领一个 5G 的 pvc

vi prometheus-pvc.yaml

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

  name: gwj-prometheus-pvc

  namespace: gwj

spec:

  accessModes:

  – ReadWriteMany

  volumeMode: Filesystem

  resources:

    requests:

      storage: 5Gi

  selector:

    matchLabels:

      app: gwj-pv

  storageClassName: slow

设置 prometheus rbac 权限

clusterrole.rbac.authorization.k8s.io/gwj-prometheus-clusterrole created

serviceaccount/gwj-prometheus created

clusterrolebinding.rbac.authorization.k8s.io/gwj-prometheus-rolebinding created

vi prometheus-rbac.yml

apiVersion: rbac.authorization.k8s.io/v1beta1

kind: ClusterRole

metadata:

  name: gwj-prometheus-clusterrole

rules:

• apiGroups: [“”]

  resources:

  – nodes

  – nodes/proxy

  – services

  – endpoints

  – pods

  verbs: [“get”, “list”, “watch”]

• apiGroups:

  – extensions

  resources:

  – ingresses

  verbs: [“get”, “list”, “watch”]

• nonResourceURLs: [“/metrics”]

  verbs: [“get”]

apiVersion: v1

kind: ServiceAccount

metadata:

  namespace: gwj

  name: gwj-prometheus

apiVersion: rbac.authorization.k8s.io/v1beta1

kind: ClusterRoleBinding

metadata:

  name: gwj-prometheus-rolebinding

roleRef:

  apiGroup: rbac.authorization.k8s.io

  kind: ClusterRole

  name: gwj-prometheus-clusterrole

subjects:

• kind: ServiceAccount

  name: gwj-prometheus

  namespace: gwj

创立 prometheus 配置文件，应用 configmap

vi prometheus-cm.yml

apiVersion: v1

kind: ConfigMap

metadata:

  name: gwj-prometheus-cm

  namespace: gwj

data:

  prometheus.yml: |

    rule_files:

    – /etc/prometheus/rules.yml

    alerting:

      alertmanagers:

      – static_configs:

        – targets: [“gwj-alertmanger-svc:80”]

    global:

      scrape_interval: 10s

      scrape_timeout: 10s

      evaluation_interval: 10s

    scrape_configs:

    – job_name: ‘kubernetes-nodes’

      scheme: https

      tls_config:

        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      kubernetes_sd_configs:

      – role: node

      relabel_configs:

      – action: labelmap

        regex: __meta_kubernetes_node_label_(.+)

      – source_labels: [__meta_kubernetes_node_name]

        regex: (.+)

        target_label: metrics_path

        replacement: /api/v1/nodes/${1}/proxy/metrics

      – target_label: address

        replacement: kubernetes.default.svc:443

    – job_name: ‘kubernetes-node-exporter’

      tls_config:

        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      kubernetes_sd_configs:

      – role: node

      relabel_configs:

      – action: labelmap

        regex: __meta_kubernetes_node_label_(.+)

      – source_labels: [__meta_kubernetes_role]

        action: replace

        target_label: kubernetes_role

      – source_labels: [__address__]

        regex: ‘(.*):10250’

        replacement: ‘${1}:9100’

        target_label: address

    – job_name: ‘kubernetes-pods’

      kubernetes_sd_configs:

      – role: pod

      relabel_configs:

      – source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]

        action: keep

        regex: true

      – source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]

        action: replace

        target_label: address

        regex: (¹+)(?::d+)?;(d+)

        replacement: $1:$2

      – action: labelmap

        regex: __meta_kubernetes_pod_label_(.+)

      – source_labels: [__meta_kubernetes_namespace]

        action: replace

        target_label: kubernetes_namespace

      – source_labels: [__meta_kubernetes_pod_name]

        action: replace

        target_label: kubernetes_pod_name

    – job_name: ‘kubernetes-cadvisor’

      scheme: https

      tls_config:

        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      kubernetes_sd_configs:

      – role: node

      relabel_configs:

      – action: labelmap

        regex: __meta_kubernetes_node_label_(.+)

      – target_label: address

        replacement: kubernetes.default.svc:443

      – source_labels: [__meta_kubernetes_node_name]

        regex: (.+)

        target_label: metrics_path

        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor

    – job_name: ‘kubernetes-service-endpoints’

      kubernetes_sd_configs:

      – role: endpoints

      relabel_configs:

      – source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]

        action: keep

        regex: true

      – source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]

        action: replace

        target_label: scheme

        regex: (https?)

      – source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]

        action: replace

        target_label: metrics_path

        regex: (.+)

      – source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]

        action: replace

        target_label: address

        regex: (¹+)(?::d+)?;(d+)

        replacement: $1:$2

      – action: labelmap

        regex: __meta_kubernetes_service_label_(.+)

      – source_labels: [__meta_kubernetes_namespace]

        action: replace

        target_label: kubernetes_namespace

      – source_labels: [__meta_kubernetes_service_name]

        action: replace

        target_label: kubernetes_name

  rules.yml: |

    groups:

    – name: kebernetes_rules

      rules:

      – alert: InstanceDown

        expr: up{job=”kubernetes-node-exporter”} == 0

        for: 5m

        labels:

          severity: page

        annotations:

          summary: “Instance {{$labels.instance}} down”

          description: “{{$labels.instance}} of job {{$labels.job}} has been down for more than 5 minutes.”

      – alert: APIHighRequestLatency

        expr: api_http_request_latencies_second{quantile=”0.5″} > 1

        for: 10m

        annotations:

          summary: “High request latency on {{$labels.instance}}”

          description: “{{$labels.instance}} has a median request latency above 1s (current value: {{ $value}}s)”

      – alert: StatefulSetReplicasMismatch

        annotations:

          summary: “Replicas miss match”

          description: StatefulSet {{$labels.namespace}}/{{$labels.statefulset}} has not matched the expected number of replicas for longer than 3 minutes.

        expr: label_join(kube_statefulset_status_replicas_ready != kube_statefulset_replicas, “instance”, “/”, “namespace”, “statefulset”)

        for: 3m

        labels:

          severity: critical

      – alert: PodFrequentlyRestarting

        expr: increase(kube_pod_container_status_restarts_total[1h]) > 5

        for: 5m

        labels:

          severity: warning

        annotations:

          description: Pod {{$labels.namespaces}}/{{$labels.pod}} is was restarted {{$value}} times within the last hour

          summary: Pod is restarting frequently

      – alert: DeploymentReplicasNotUpdated

        expr: ((kube_deployment_status_replicas_updated != kube_deployment_spec_replicas)

          or (kube_deployment_status_replicas_available != kube_deployment_spec_replicas))

          unless (kube_deployment_spec_paused == 1)

        for: 5m

        labels:

          severity: critical

        annotations:

          description: Replicas are not updated and available for deployment {{$labels.namespace}}/{{$labels.deployment}}

          summary: Deployment replicas are outdated

      – alert: DaemonSetRolloutStuck

        expr: kube_daemonset_status_number_ready / kube_daemonset_status_desired_number_scheduled * 100 < 100

        for: 5m

        labels:

          severity: critical

        annotations:

          description: Only {{$value}}% of desired pods scheduled and ready for daemonset {{$labels.namespace}}/{{$labels.daemonset}}

          summary: DaemonSet is missing pods

      – alert: DaemonSetsNotScheduled

        expr: kube_daemonset_status_desired_number_scheduled – kube_daemonset_status_current_number_scheduled > 0

        for: 10m

        labels:

          severity: warning

        annotations:

          description: ‘{{{{$value}}}} Pods of DaemonSet {{{{$labels.namespace}}}}/{{{{$labels.daemonset}}}} are not scheduled.’

          summary: Daemonsets are not scheduled correctly

      – alert: DaemonSetsMissScheduled

        expr: kube_daemonset_status_number_misscheduled > 0

        for: 10m

        labels:

          severity: warning

        annotations:

          description: ‘{{{{$value}}}} Pods of DaemonSet {{{{$labels.namespace}}}}/{{{{$labels.daemonset}}}} are running where they are not supposed to run.’

          summary: Daemonsets are not scheduled correctly

      – alert: Node_Boot_Time

        expr: (node_time_seconds – node_boot_time_seconds) <= 150

        for: 15s

        annotations:

          summary: “ 机器 {{$labels.instacnce}} 刚刚重启，工夫少于 150s”

      – alert: Available_Percent

        expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes <= 0.2

        for: 15s

        annotations:

          summary: “ 机器 {{$labels.instacnce}} available less than 20%”

      – alert: FD_Used_Percent

        expr: (node_filefd_allocated / node_filefd_maximum) >= 0.8

        for: 15s

        annotations:

          summary: “ 机器 {{$labels.instacnce}} FD used more than 80%”

依据刚刚创立的 cm 的要求，创立 alertmanger 用于告警

vi alertmanger.yml

kind: Service

apiVersion: v1

metadata:

  name: gwj-alertmanger-svc

  namespace: gwj

spec:

  selector:

    app: gwj-alert-pod

  ports:

    – protocol: TCP

      port: 80

      targetPort: 9093

apiVersion: apps/v1

kind: StatefulSet

metadata:

  name: gwj-alert-sts

  namespace: gwj

  labels:

    app: gwj-alert-sts

spec:

  replicas: 1

  serviceName: gwj-alertmanger-svc

  selector:

    matchLabels:

      app: gwj-alert-pod

  template:

    metadata:

      labels:

        app: gwj-alert-pod

    spec:

      containers:

      – image: prom/alertmanager:v0.14.0

        name: gwj-alert-pod

        ports:

        – containerPort: 9093

          protocol: TCP

        volumeMounts:

        – mountPath: “/etc/localtime”

          name: timezone

      volumes:

      – name: timezone

        hostPath:

          path: /etc/localtime

kubectl apply -f alertmanger.yml

  service/gwj-alertmanger-svc created

  statefulset.apps/gwj-alert-sts created

创立 prometheus statefulset 来创立 prometheus

service/gwj-prometheus-svc created

statefulset.apps/gwj-prometheus-sts created

/prometheus

pvc: gwj-prometheus-pvc

/etc/prometheus/

configMap:

  name: gwj-prometheus-cm

vi prometheus-sts.yml

kind: Service

apiVersion: v1

metadata:

  name: gwj-prometheus-svc

  namespace: gwj

  labels:

    app: gwj-prometheus-svc

spec:

  ports:

  – port: 80

    targetPort: 9090

  selector:

    app: gwj-prometheus-pod

apiVersion: apps/v1

kind: StatefulSet

metadata:

  name: gwj-prometheus-sts

  namespace: gwj

  labels:

    app: gwj-prometheus-sts

spec:

  replicas: 1

  serviceName: gwj-prometheus-svc

  selector:

    matchLabels:

      app: gwj-prometheus-pod

  template:

    metadata:

      labels:

        app: gwj-prometheus-pod

    spec:

      containers:

      – image: prom/prometheus:v2.9.2

        name: gwj-prometheus-pod

        ports:

        – containerPort: 9090

          protocol: TCP

        volumeMounts:

        – mountPath: “/prometheus”

          name: data

        – mountPath: “/etc/prometheus/”

          name: config-volume

        – mountPath: “/etc/localtime”

          name: timezone

        resources:

          requests:

            cpu: 100m

            memory: 100Mi

          limits:

            cpu: 500m

            memory: 2000Mi

      serviceAccountName: gwj-prometheus

      volumes:

      – name: data

        persistentVolumeClaim:

          claimName: gwj-prometheus-pvc

      – name: config-volume

        configMap:

          name: gwj-prometheus-cm

      – name: gwj-prometheus-rule-cm

        configMap:

          name: gwj-prometheus-rule-cm

      – name: timezone

        hostPath:

          path: /etc/localtime

kubectl apply -f prometheus-sts.yml

  service/gwj-prometheus-svc created

  statefulset.apps/gwj-prometheus-sts created

创立 ingress，依据域名散发到不同的 service

vi prometheus-ingress.yml

apiVersion: extensions/v1beta1

kind: Ingress

metadata:

  namespace: gwj

  annotations:

  name: gwj-ingress-prometheus

spec:

  rules:

  – host: gwj.syncbug.com

    http:

      paths:

        – path: /

          backend:

            serviceName: gwj-prometheus-svc

            servicePort: 80

  – host: gwj-alert.syncbug.com

    http:

      paths:

        – path: /

          backend:

            serviceName: gwj-alertmanger-svc

            servicePort: 80

kubectl apply -f prometheus-ingress.yml

  ingress.extensions/gwj-ingress-prometheus created

拜访对应的域名

gwj.syncbug.com

查看指标对象是否正确

http://gwj.syncbug.com/targets

查看配置文件是否正确

http://gwj.syncbug.com/config

gwj-alert.syncbug.com

===grafana

vi grafana-pv.yaml

apiVersion: v1

kind: PersistentVolume

metadata:

  name: gwj-pv-grafana

  labels:

    app: gwj-pv-gra

spec:

  capacity:

    storage: 2Gi

  volumeMode: Filesystem

  accessModes:

  – ReadWriteMany

  persistentVolumeReclaimPolicy: Recycle

  storageClassName: slow

  mountOptions:

  – hard

  – nfsvers=4.1

  nfs:

    path: /storage/gwj-grafana

    server: 10.1.99.1

vi grafana-pvc.yaml

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

  name: gwj-grafana-pvc

  namespace: gwj

spec:

  accessModes:

  – ReadWriteMany

  volumeMode: Filesystem

  resources:

    requests:

      storage: 1Gi

  selector:

    matchLabels:

      app: gwj-pv-gra

  storageClassName: slow

vi grafana-deployment.yaml

apiVersion: extensions/v1beta1

kind: Deployment

metadata:

  labels:

    name: grafana

  name: grafana

  namespace: gwj

spec:

  replicas: 1

  revisionHistoryLimit: 10

  selector:

    matchLabels:

      app: grafana

  template:

    metadata:

      labels:

        app: grafana

      name: grafana

    spec:

      containers:

      – env:

        – name: GF_PATHS_DATA

          value: /var/lib/grafana/

        – name: GF_PATHS_PLUGINS

          value: /var/lib/grafana/plugins

        image: grafana/grafana:6.2.4

        imagePullPolicy: IfNotPresent

        name: grafana

        ports:

        – containerPort: 3000

          name: grafana

          protocol: TCP

        volumeMounts:

        – mountPath: /var/lib/grafana/

          name: data

        – mountPath: /etc/localtime

          name: localtime

      dnsPolicy: ClusterFirst

      restartPolicy: Always

      volumes:

      – name: data

        persistentVolumeClaim:

          claimName: gwj-grafana-pvc

      – name: localtime

        hostPath:

          path: /etc/localtime

vi grafana-ingress.yaml

apiVersion: extensions/v1beta1

kind: Ingress

metadata:

  namespace: gwj

  annotations:

  name: gwj-ingress-grafana

spec:

  rules:

  – host: gwj-grafana.syncbug.com

    http:

      paths:

        – path: /

          backend:

            serviceName: gwj-grafana-svc

            servicePort: 80

kind: Service

apiVersion: v1

metadata:

  name: gwj-grafana-svc

  namespace: gwj

spec:

  selector:

    app: grafana

  ports:

    – protocol: TCP

      port: 80

      targetPort: 3000

进入 grafana，gwj-grafana.syncbug.com

默认：admin admin

输出 datasource: http://gwj-prometheus-svc:80

import 模版

1. : ↩