本文借鉴于监控–Prometheus部署篇
1. prometheus权限设置
prometheus-rbac.yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:name: prometheus
rules:
- apiGroups: [""]resources:- nodes- nodes/proxy- services- endpoints- podsverbs: ["get", "list", "watch"]
- apiGroups:- extensionsresources:- ingressesverbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics"]verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:name: prometheusnamespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:name: prometheus
roleRef:apiGroup: rbac.authorization.k8s.iokind: ClusterRolename: prometheus
subjects:
- kind: ServiceAccountname: prometheusnamespace: kube-system

2. 配置文件
prometheus-comfigMap.yaml

apiVersion: v1
kind: ConfigMap
metadata:name: prometheus-configlabels:monitor: prometheusnamespace: kube-system
data:hoststats.rules: |groups:- name: nodehostrules:# Alert for 85% usage of Cluster CPU for > 1m- alert: hostCpuUsageAlertexpr: sum(avg (irate(node_cpu{mode!="idle",job="kubernetes_node"}[5m])) without (cpu)) by (instance) > 0.85for: 3mlabels:severity: pageannotations:summary: "Instance {{ $labels.instance }} CPU usgae high"description: "{{ $labels.instance }} CPU usage above 85% (current value: {{ $value }})"- alert: hostMemUsageAlertexpr: (node_memory_MemTotal-node_memory_MemAvailable)/node_memory_MemTotal{job="kubernetes_node"} * 100 > 85for: 5mlabels:severity: pageannotations:summary: "Instance {{ $labels.instance }}MEM usage high"description: "Instance {{ $labels.instance }} Memory usage above 85% (current value: {{ $value }})"- alert: hostDiskUsageAlertexpr: container_fs_usage_bytes{device=~"^/dev/[sv]d[a-z][1-9]$",id="/"}/container_fs_limit_bytes{device=~"^/dev/[sv]d[a-z][1-9]$",id="/"} > 0.8for: 5mlabels:severity: pageannotations:summary: "Instance {{ $labels.instance }}Disk usage high"description: "Instance {{ $labels.instance }} Disk usage above 80% (current value: {{ $value }})"prometheus.yml: |global:scrape_interval:     15sevaluation_interval: 15srule_files:- /etc/prometheus/rules/*.rulesalerting:alertmanagers:- static_configs:- targets: ["alertmanager:9093"]scrape_configs:- job_name: 'kubernetes-apiservers'kubernetes_sd_configs:- role: endpointsscheme: httpstls_config:ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crtbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/tokenrelabel_configs:- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]action: keepregex: default;kubernetes;https- job_name: 'kubernetes-cadvisor'kubernetes_sd_configs:- role: nodescheme: httpstls_config:ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crtbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/tokenrelabel_configs:- action: labelmapregex: __meta_kubernetes_node_label_(.+)- target_label: __address__replacement: kubernetes.default.svc:443- source_labels: [__meta_kubernetes_node_name]regex: (.+)target_label: __metrics_path__replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor- job_name: 'kubernetes-service-endpoints'kubernetes_sd_configs:- role: endpointsrelabel_configs:- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]action: keepregex: true- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]action: replacetarget_label: __scheme__regex: (https?)- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]action: replacetarget_label: __metrics_path__regex: (.+)- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]action: replacetarget_label: __address__regex: ([^:]+)(?::\d+)?;(\d+)replacement: $1:$2- action: labelmapregex: __meta_kubernetes_service_label_(.+)- source_labels: [__meta_kubernetes_namespace]action: replacetarget_label: kubernetes_namespace- source_labels: [__meta_kubernetes_service_name]action: replacetarget_label: kubernetes_name- job_name: 'kubernetes-services'kubernetes_sd_configs:- role: servicemetrics_path: /probeparams:module: [http_2xx]relabel_configs:- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]action: keepregex: true- source_labels: [__address__]target_label: __param_target- target_label: __address__replacement: blackbox-exporter.example.com:9115- source_labels: [__param_target]target_label: instance- action: labelmapregex: __meta_kubernetes_service_label_(.+)- source_labels: [__meta_kubernetes_namespace]target_label: kubernetes_namespace- source_labels: [__meta_kubernetes_service_name]target_label: kubernetes_name- job_name: 'kubernetes-ingresses'kubernetes_sd_configs:- role: ingressrelabel_configs:- source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe]action: keepregex: trueregex: (.+)- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]action: replaceregex: ([^:]+)(?::\d+)?;(\d+)replacement: $1:$2target_label: __address__- action: labelmapregex: __meta_kubernetes_pod_label_(.+)- source_labels: [__meta_kubernetes_namespace]action: replacetarget_label: kubernetes_namespace- source_labels: [__meta_kubernetes_pod_name]action: replacetarget_label: kubernetes_pod_name- job_name: 'kubernetes_node'tls_config:ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crtbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/tokenkubernetes_sd_configs:# 基于endpoint的服务发现，不再经过service代理层面- role: endpointsrelabel_configs:- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape, __meta_kubernetes_endpoint_port_name]regex: true;prometheus-node-exporteraction: keep- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]action: replacetarget_label: __scheme__regex: (https?)- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]action: replacetarget_label: __metrics_path__regex: (.+)- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]action: replacetarget_label: __address__regex: (.+)(?::\d+);(\d+)replacement: $1:$2# 去掉label name中的前缀__meta_kubernetes_service_label_- action: labelmapregex: __meta_kubernetes_service_label_(.+)# 为了区分所属node,把instance 从node-exporter ep的实例，替换成ep所在node的ip- source_labels: [__meta_kubernetes_pod_host_ip]regex: '(.*)'replacement: '${1}'target_label: instance

3. app部署
prometheus.rc.yaml

apiVersion: v1
kind: ReplicationController
metadata:labels:monitor: prometheusname: prometheusnamespace: kube-system
spec:replicas: 1template:metadata:name: prometheuslabels:monitor: prometheusspec:containers:- image: prom/prometheus:v2.0.0name: prometheuscommand:- "/bin/prometheus"args:- "--config.file=/etc/prometheus/prometheus.yml"- "--storage.tsdb.path=/prometheus"- "--storage.tsdb.retention=24h"ports:- containerPort: 9090protocol: TCPvolumeMounts:- name: datamountPath: "/prometheus"- name: config-volumemountPath: "/etc/prometheus/prometheus.yml"subPath: prometheus.yml- name: config-volumemountPath: "/etc/prometheus/rules/hoststats.rules"subPath: hoststats.rulesresources:requests:cpu: 100mmemory: 100Milimits:cpu: 500mmemory: 2500MiserviceAccountName: prometheus    volumes:- name: dataemptyDir: {}- name: config-volumeconfigMap:name: prometheus-config
---
kind: Service
apiVersion: v1
metadata:labels:monitor: prometheusname: prometheusnamespace: kube-system
spec:type: NodePortports:- port: 80              # 访问服务名即可跳转到对于目标容器端口protocol: TCPtargetPort: 9090      # 容器开发端口nodePort: 39090       # 绑定集群节点端口selector:monitor: prometheus

4. 节点采集器部署

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:name: prometheus-node-exporternamespace: kube-systemlabels:k8s-app: prometheus-node-exporter
spec:template:metadata:labels:k8s-app: prometheus-node-exporterspec:tolerations:- key: node-role.kubernetes.io/masteroperation: Equaleffect: NoSchedulecontainers:- image: prom/node-exporter:v0.14.0name: node-exporterports:- containerPort: 9100protocol: TCPname: httphostPort: 9101volumeMounts:- mountPath: /host/procname: proc- mountPath: /host/sysname: sysargs:- -collector.procfs=/host/proc - -collector.sysfs=/host/sys - -collector.filesystem.ignored-mount-points- "^/(dev|proc|sys|host|etc|rootfs|docker)($|/)"volumes:- hostPath:path: /procname: proc- hostPath:path: /sysname: sys
---
apiVersion: v1
kind: Service
metadata:annotations:prometheus.io/scrape: 'true'prometheus.io/app-metrics: 'true'prometheus.io/app-metrics-path: '/metrics'name: prometheus-node-exporternamespace: kube-systemlabels:app: prometheus-node-exporter
spec:clusterIP: Noneports:- name: prometheus-node-exporterport: 9100protocol: TCPselector:k8s-app: prometheus-node-exportertype: ClusterIP