14 篇博文 含有标签「Prometheus」

global:
  resolve_timeout: 5m
  smtp_from: 'xxxx@qq.com'
  smtp_smarthost: 'smtp.qq.com:465'
  smtp_auth_username: 'xxxx@qq.com'
  smtp_auth_password: 'uukxxxxdvnxzbiaf'
  smtp_require_tls: false
  smtp_hello: '@qq.com'
route:
  group_by: ['alertname']
  group_wait: 10s
  group_interval: 2m
  repeat_interval: 5m
  receiver: 'web.hook'
#  #receiver: 'default-receiver' #其他的告警发送给default-receiver
#  routes: #将critical的报警发送给myalertname
#  - reciver: myalertname
#  group_wait: 10s
receivers:
  - name: 'web.hook'
#    webhook_configs:
#      - url: 'http://127.0.0.1:5001/'
    email_configs:
    - to: 'xxxx@qq.com'
inhibit_rules:
  - source_match: #源匹配级别，当匹配成功发出通知，但是其他'alertname'，'dev'，'instance'产生的warning级别的告警通知将被抑制
      severity: 'critical' #报警的事件级别
    target_match:
      severity: 'warning' #匹配目标为新产生的目标告警为'warning' 将被抑制
    equal: ['alertname', 'dev', 'instance']

# 创建角色目录
mkdir /apps/prometheus/rules && cd /apps/prometheus/rules

# 编写配置文件
vim server_rules.yaml
#---------------------------------

groups:
  - name: alertmanager_pod.rules
    rules:
    - alert: Pod_all_cpu_usage
      expr: (sum by(name)(rate(container_cpu_usage_seconds_total{image!=""}[5m]))*100) > 10
      for: 2m
      labels:
        severity: critical
        service: pods
      annotations:
        description: 容器 {{ $labels.name }} CPU 资源利用率大于 10% , (current value is {{ $value }})
        summary: Dev CPU 负载告警

    - alert: Pod_all_memory_usage  
      #expr: sort_desc(avg by(name)(irate(container_memory_usage_bytes{name!=""}[5m]))*100) > 10  #内存大于10%
      expr: sort_desc(avg by(name)(irate(node_memory_MemFree_bytes {name!=""}[5m]))) > 2147483648   #内存大于2G
      for: 2m
      labels:
        severity: critical
      annotations:
        description: 容器 {{ $labels.name }} Memory 资源利用率大于 2G , (current value is {{ $value }})
        summary: Dev Memory 负载告警

    - alert: Pod_all_network_receive_usage
      expr: sum by (name)(irate(container_network_receive_bytes_total{container_name="POD"}[1m])) > 50*1024*1024
      for: 2m
      labels:
        severity: critical
      annotations:
        description: 容器 {{ $labels.name }} network_receive 资源利用率大于 50M , (current value is {{ $value }})

    - alert: node内存可用大小 
      expr: node_memory_MemFree_bytes > 512*1024*1024 #故意写错的
      #expr: node_memory_MemFree_bytes > 1 #故意写错的(容器可用内存小于100k)
      for: 15s
      labels:
        severity: critical
      annotations:
        description: node可用内存小于4G

  - name: alertmanager_node.rules
    rules:
    - alert: 磁盘容量
      expr: 100-(node_filesystem_free_bytes{fstype=~"ext4|xfs"}/node_filesystem_size_bytes {fstype=~"ext4|xfs"}*100) > 80  #磁盘容量利用率大于80%
      for: 2s
      labels:
        severity: critical
      annotations:
        summary: "{{$labels.mountpoint}} 磁盘分区使用率过高！"
        description: "{{$labels.mountpoint }} 磁盘分区使用大于80%(目前使用:{{$value}}%)"

    - alert: 磁盘容量
      expr: 100-(node_filesystem_free_bytes{fstype=~"ext4|xfs"}/node_filesystem_size_bytes {fstype=~"ext4|xfs"}*100) > 60 #磁盘容量利用率大于60%
      for: 2s
      labels:
        severity: warning
      annotations:
        summary: "{{$labels.mountpoint}} 磁盘分区使用率过高！"
        description: "{{$labels.mountpoint }} 磁盘分区使用大于80%(目前使用:{{$value}}%)"

alerting:
  alertmanagers:
  - static_configs:
    - targets:
      - 192.168.15.100:9093

# 文件路径:
rule_files:
  - /apps/prometheus/rules/server_rules.yaml

global:
  resolve_timeout: 5m
  smtp_from: 'xxxx@qq.com'
  smtp_smarthost: 'smtp.qq.com:465'
  smtp_auth_username: 'xxxx@qq.com'
  smtp_auth_password: 'gtiuxxxxngxybhdi'
  smtp_require_tls: false
  smtp_hello: '@qq.com'
route:
  group_by: ['alertname']
  group_wait: 10s
  group_interval: 1m
  repeat_interval: 5m
  receiver: 'dingding'
  #receiver: 'web.hook'
#  #receiver: 'default-receiver' #其他的告警发送给default-receiver
#  routes: #将critical的报警发送给myalertname
#  - reciver: myalertname
#  group_wait: 10s
receivers:
  - name: dingding
    webhook_configs:
    - url: 'http://192.168.15.100:8060/dingding/alertname/send'
      send_resolved: true
  - name: 'web.hook'
#    webhook_configs:
#      - url: 'http://127.0.0.1:5001/'
    email_configs:
    - to: 'xxxx@qq.com'
      send_resolved: true
inhibit_rules:
  - source_match: #源匹配级别，当匹配成功发出通知，但是其他'alertname'，'dev'，'instance'产生的warning级别的告警通知将被抑制
      severity: 'critical' #报警的事件级别
    target_match:
      severity: 'warning' #匹配目标为新产生的目标告警为'warning' 将被抑制
    equal: ['alertname', 'dev', 'instance']

  - job_name: 'prometheus'
    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'prometheus-federate-2.101'
    scrape_interval: 10s
    honor_labels: true
    metrics_path: '/federate'
    params:
      'match[]':
       - '{job="prometheus"}'
       - '{__name__=~"job:.*"}'
       - '{__name__=~"node.*"}'
    static_configs:
    - targets:
      - '192.168.15.101:9090'

  - job_name: 'prometheus-federate-2.102'
    scrape_interval: 10s
    honor_labels: true
    metrics_path: '/federate'
    params:
      'match[]':
       - '{job="prometheus"}'
       - '{__name__=~"job:.*"}'
       - '{__name__=~"node.*"}'
    static_configs:
    - targets:
      - '192.168.15.102:9090'

  - job_name: 'pushgateway-monitor'
    scrape_interval: 5s
    static_configs:
    - targets: ['192.168.15.100:9091']
    honor_labels: true