关于kubernetes:k8sprometheus

<article class=“article fmt article-content”>promethus基于k8s收集数据node-exportervi node-exporter-ds.yml<code></code>apiVersion: extensions/v1beta1kind: DaemonSetmetadata: name: node-exporter labels: app: node-exporterspec: template: metadata: labels: app: node-exporter spec: hostNetwork: true containers: - image: prom/node-exporter name: node-exporter ports: - containerPort: 9100 volumeMounts: - mountPath: “/etc/localtime” name: timezone volumes: - name: timezone hostPath: path: /etc/localtime存储，长久卷，创立一个10G的pv,基于nfsvi prometheus-pv.yaml<code></code>apiVersion: v1kind: PersistentVolumemetadata: name: gwj-pv-prometheus labels: app: gwj-pvspec: capacity: storage: 10Gi volumeMode: Filesystem accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Recycle storageClassName: slow mountOptions: - hard - nfsvers=4.1 nfs: path: /storage/gwj-prometheus server: 10.1.99.1长久卷申领，基于刚刚创立的pv，申领一个5G的pvcvi prometheus-pvc.yaml<code></code>apiVersion: v1kind: PersistentVolumeClaimmetadata: name: gwj-prometheus-pvc namespace: gwjspec: accessModes: - ReadWriteMany volumeMode: Filesystem resources: requests: storage: 5Gi selector: matchLabels: app: gwj-pv storageClassName: slow设置prometheus rbac权限clusterrole.rbac.authorization.k8s.io/gwj-prometheus-clusterrole createdserviceaccount/gwj-prometheus createdclusterrolebinding.rbac.authorization.k8s.io/gwj-prometheus-rolebinding createdvi prometheus-rbac.yml<code></code>apiVersion: rbac.authorization.k8s.io/v1beta1kind: ClusterRolemetadata: name: gwj-prometheus-clusterrolerules:<ul><li>apiGroups: [""]</li></ul> resources: - nodes - nodes/proxy - services - endpoints - pods verbs: [“get”, “list”, “watch”]<ul><li>apiGroups:</li></ul> - extensions resources: - ingresses verbs: [“get”, “list”, “watch”]<ul><li>nonResourceURLs: ["/metrics"]</li></ul> verbs: [“get”]<hr/>apiVersion: v1kind: ServiceAccountmetadata: namespace: gwj name: gwj-prometheus<hr/>apiVersion: rbac.authorization.k8s.io/v1beta1kind: ClusterRoleBindingmetadata: name: gwj-prometheus-rolebindingroleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: gwj-prometheus-clusterrolesubjects:<ul><li>kind: ServiceAccount</li></ul> name: gwj-prometheus namespace: gwj创立prometheus 配置文件，应用configmapvi prometheus-cm.yml<code></code>apiVersion: v1kind: ConfigMapmetadata: name: gwj-prometheus-cm namespace: gwjdata: prometheus.yml: | rule_files: - /etc/prometheus/rules.yml alerting: alertmanagers: - static_configs: - targets: [“gwj-alertmanger-svc:80”] global: scrape_interval: 10s scrape_timeout: 10s evaluation_interval: 10s scrape_configs: - job_name: ‘kubernetes-nodes’ scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: - role: node relabel_configs: - action: labelmap regex: _meta_kubernetes_node_label(.+) - source_labels: [__meta_kubernetes_node_name] regex: (.+) target_label: metrics_path replacement: /api/v1/nodes/${1}/proxy/metrics - target_label: address replacement: kubernetes.default.svc:443 - job_name: ‘kubernetes-node-exporter’ tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: - role: node relabel_configs: - action: labelmap regex: _meta_kubernetes_node_label(.+) - source_labels: [__meta_kubernetes_role] action: replace target_label: kubernetes_role - source_labels: [address] regex: ‘(.*):10250’ replacement: ‘${1}:9100’ target_label: address - job_name: ‘kubernetes-pods’ kubernetes_sd_configs: - role: pod relabel_configs: - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] action: keep regex: true - source_labels: [address, __meta_kubernetes_pod_annotation_prometheus_io_port] action: replace target_label: address regex: (1+)(?::d+)?;(d+) replacement: $1:$2 - action: labelmap regex: _meta_kubernetes_pod_label(.+) - source_labels: [__meta_kubernetes_namespace] action: replace target_label: kubernetes_namespace - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: kubernetes_pod_name - job_name: ‘kubernetes-cadvisor’ scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: - role: node relabel_configs: - action: labelmap regex: _meta_kubernetes_node_label(.+) - target_label: address replacement: kubernetes.default.svc:443 - source_labels: [__meta_kubernetes_node_name] regex: (.+) target_label: metrics_path replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor - job_name: ‘kubernetes-service-endpoints’ kubernetes_sd_configs: - role: endpoints relabel_configs: - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] action: keep regex: true - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] action: replace target_label: scheme regex: (https?) - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] action: replace target_label: metrics_path regex: (.+) - source_labels: [address, __meta_kubernetes_service_annotation_prometheus_io_port] action: replace target_label: address regex: (1+)(?::d+)?;(d+) replacement: $1:$2 - action: labelmap regex: _meta_kubernetes_service_label(.+) - source_labels: [__meta_kubernetes_namespace] action: replace target_label: kubernetes_namespace - source_labels: [__meta_kubernetes_service_name] action: replace target_label: kubernetes_name rules.yml: | groups: - name: kebernetes_rules rules: - alert: InstanceDown expr: up{job=“kubernetes-node-exporter”} == 0 for: 5m labels: severity: page annotations: summary: “Instance {{ $labels.instance }} down” description: “{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes." - alert: APIHighRequestLatency expr: api_http_request_latencies_second{quantile=“0.5”} > 1 for: 10m annotations: summary: “High request latency on {{ $labels.instance }}" description: “{{ $labels.instance }} has a median request latency above 1s (current value: {{ $value }}s)" - alert: StatefulSetReplicasMismatch annotations: summary: “Replicas miss match” description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 3 minutes. expr: label_join(kube_statefulset_status_replicas_ready != kube_statefulset_replicas, “instance”, “/”, “namespace”, “statefulset”) for: 3m labels: severity: critical - alert: PodFrequentlyRestarting expr: increase(kube_pod_container_status_restarts_total[1h]) > 5 for: 5m labels: severity: warning annotations: description: Pod {{ $labels.namespaces }}/{{ $labels.pod }} is was restarted {{ $value }} times within the last hour summary: Pod is restarting frequently - alert: DeploymentReplicasNotUpdated expr: ((kube_deployment_status_replicas_updated != kube_deployment_spec_replicas) or (kube_deployment_status_replicas_available != kube_deployment_spec_replicas)) unless (kube_deployment_spec_paused == 1) for: 5m labels: severity: critical annotations: description: Replicas are not updated and available for deployment {{ $labels.namespace }}/{{ $labels.deployment }} summary: Deployment replicas are outdated - alert: DaemonSetRolloutStuck expr: kube_daemonset_status_number_ready / kube_daemonset_status_desired_number_scheduled * 100 < 100 for: 5m labels: severity: critical annotations: description: Only {{ $value }}% of desired pods scheduled and ready for daemonset {{ $labels.namespace }}/{{ $labels.daemonset }} summary: DaemonSet is missing pods - alert: DaemonSetsNotScheduled expr: kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_current_number_scheduled > 0 for: 10m labels: severity: warning annotations: description: ‘{{<code>{{ $value }}</code>}} Pods of DaemonSet {{<code>{{ $labels.namespace }}</code>}}/{{<code>{{ $labels.daemonset }}</code>}} are not scheduled.’ summary: Daemonsets are not scheduled correctly - alert: DaemonSetsMissScheduled expr: kube_daemonset_status_number_misscheduled > 0 for: 10m labels: severity: warning annotations: description: ‘{{<code>{{ $value }}</code>}} Pods of DaemonSet {{<code>{{ $labels.namespace }}</code>}}/{{<code>{{ $labels.daemonset }}</code>}} are running where they are not supposed to run.’ summary: Daemonsets are not scheduled correctly - alert: Node_Boot_Time expr: (node_time_seconds - node_boot_time_seconds) <= 150 for: 15s annotations: summary: “机器{{ $labels.instacnce }} 刚刚重启，工夫少于 150s” - alert: Available_Percent expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes <= 0.2 for: 15s annotations: summary: “机器{{ $labels.instacnce }} available less than 20%" - alert: FD_Used_Percent expr: (node_filefd_allocated / node_filefd_maximum) >= 0.8 for: 15s annotations: summary: “机器{{ $labels.instacnce }} FD used more than 80%"依据刚刚创立的cm的要求，创立alertmanger 用于告警vi alertmanger.yml<code></code><hr/>kind: ServiceapiVersion: v1metadata: name: gwj-alertmanger-svc namespace: gwjspec: selector: app: gwj-alert-pod ports: - protocol: TCP port: 80 targetPort: 9093<hr/>apiVersion: apps/v1kind: StatefulSetmetadata: name: gwj-alert-sts namespace: gwj labels: app: gwj-alert-stsspec: replicas: 1 serviceName: gwj-alertmanger-svc selector: matchLabels: app: gwj-alert-pod template: metadata: labels: app: gwj-alert-pod spec: containers: - image: prom/alertmanager:v0.14.0 name: gwj-alert-pod ports: - containerPort: 9093 protocol: TCP volumeMounts: - mountPath: “/etc/localtime” name: timezone volumes: - name: timezone hostPath: path: /etc/localtimekubectl apply -f alertmanger.yml service/gwj-alertmanger-svc created statefulset.apps/gwj-alert-sts created创立prometheus statefulset来创立prometheusservice/gwj-prometheus-svc createdstatefulset.apps/gwj-prometheus-sts created/prometheuspvc: gwj-prometheus-pvc/etc/prometheus/configMap: name: gwj-prometheus-cmvi prometheus-sts.yml<code></code><hr/>kind: ServiceapiVersion: v1metadata: name: gwj-prometheus-svc namespace: gwj labels: app: gwj-prometheus-svcspec: ports: - port: 80 targetPort: 9090 selector: app: gwj-prometheus-pod<hr/>apiVersion: apps/v1kind: StatefulSetmetadata: name: gwj-prometheus-sts namespace: gwj labels: app: gwj-prometheus-stsspec: replicas: 1 serviceName: gwj-prometheus-svc selector: matchLabels: app: gwj-prometheus-pod template: metadata: labels: app: gwj-prometheus-pod spec: containers: - image: prom/prometheus:v2.9.2 name: gwj-prometheus-pod ports: - containerPort: 9090 protocol: TCP volumeMounts: - mountPath: “/prometheus” name: data - mountPath: “/etc/prometheus/" name: config-volume - mountPath: “/etc/localtime” name: timezone resources: requests: cpu: 100m memory: 100Mi limits: cpu: 500m memory: 2000Mi serviceAccountName: gwj-prometheus volumes: - name: data persistentVolumeClaim: claimName: gwj-prometheus-pvc - name: config-volume configMap: name: gwj-prometheus-cm - name: gwj-prometheus-rule-cm configMap: name: gwj-prometheus-rule-cm - name: timezone hostPath: path: /etc/localtimekubectl apply -f prometheus-sts.yml service/gwj-prometheus-svc created statefulset.apps/gwj-prometheus-sts created创立ingress，依据域名散发到不同的servicevi prometheus-ingress.yml<code></code><hr/>apiVersion: extensions/v1beta1kind: Ingressmetadata: namespace: gwj annotations: name: gwj-ingress-prometheusspec: rules: - host: gwj.syncbug.com http: paths: - path: / backend: serviceName: gwj-prometheus-svc servicePort: 80 - host: gwj-alert.syncbug.com http: paths: - path: / backend: serviceName: gwj-alertmanger-svc servicePort: 80kubectl apply -f prometheus-ingress.yml ingress.extensions/gwj-ingress-prometheus created拜访对应的域名gwj.syncbug.com查看指标对象是否正确http://gwj.syncbug.com/targets查看配置文件是否正确http://gwj.syncbug.com/configgwj-alert.syncbug.com===grafanavi grafana-pv.yaml<code></code>apiVersion: v1kind: PersistentVolumemetadata: name: gwj-pv-grafana labels: app: gwj-pv-graspec: capacity: storage: 2Gi volumeMode: Filesystem accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Recycle storageClassName: slow mountOptions: - hard - nfsvers=4.1 nfs: path: /storage/gwj-grafana server: 10.1.99.1vi grafana-pvc.yaml<code></code>apiVersion: v1kind: PersistentVolumeClaimmetadata: name: gwj-grafana-pvc namespace: gwjspec: accessModes: - ReadWriteMany volumeMode: Filesystem resources: requests: storage: 1Gi selector: matchLabels: app: gwj-pv-gra storageClassName: slowvi grafana-deployment.yaml<code></code>apiVersion: extensions/v1beta1kind: Deploymentmetadata: labels: name: grafana name: grafana namespace: gwjspec: replicas: 1 revisionHistoryLimit: 10 selector: matchLabels: app: grafana template: metadata: labels: app: grafana name: grafana spec: containers: - env: - name: GF_PATHS_DATA value: /var/lib/grafana/ - name: GF_PATHS_PLUGINS value: /var/lib/grafana/plugins image: grafana/grafana:6.2.4 imagePullPolicy: IfNotPresent name: grafana ports: - containerPort: 3000 name: grafana protocol: TCP volumeMounts: - mountPath: /var/lib/grafana/ name: data - mountPath: /etc/localtime name: localtime dnsPolicy: ClusterFirst restartPolicy: Always volumes: - name: data persistentVolumeClaim: claimName: gwj-grafana-pvc - name: localtime hostPath: path: /etc/localtimevi grafana-ingress.yaml<code></code><hr/>apiVersion: extensions/v1beta1kind: Ingressmetadata: namespace: gwj annotations: name: gwj-ingress-grafanaspec: rules: - host: gwj-grafana.syncbug.com http: paths: - path: / backend: serviceName: gwj-grafana-svc servicePort: 80<hr/>kind: ServiceapiVersion: v1metadata: name: gwj-grafana-svc namespace: gwjspec: selector: app: grafana ports: - protocol: TCP port: 80 targetPort: 3000进入grafana，gwj-grafana.syncbug.com默认： admin admin输出datasource: http://gwj-prometheus-svc:80import模版<hr/><ol><li id=“fn-1”>: ↩</li></ol></article>