|
https://blog.csdn.net/qq_41475058/article/details/125150567
https://github.com/topics/blackbox-exporter
quay.io/prometheus/blackbox-exporter:latest
https://github.com/oguzhan-yilmaz/auto-blackbox-exporter/releases/download/v0.7.3/auto-blackbox-exporter-0.7.3.tgz
https://github.com/prometheus-community/helm-charts.git
一、blackbox_exporter应用场景
HTTP 测试： 定义 Request Header 信息、判断 Http status / Http Respones Header / Http Body 内容
TCP 测试： 业务组件端口状态监听、应用层协议定义与监听
ICMP 测试： 主机探活机制
POST 测试： 接口联通性
SSL证书过期时间
二、blackbox_exporter安装
# wget https://github.com/prometheus/blackbox_exporter/releases/download/v0.16.0/blackbox_exporter-0.16.0.linux-amd64.tar.gz
# tar xf blackbox_exporter-0.16.0.linux-amd64.tar.gz -C /usr/local/
# ln -s /usr/local/blackbox_exporter-0.16.0.linux-amd64/ /usr/local/blackbox_exporter
# 使用systemd进行管理blackbox_exporter服务
# vim /usr/lib/systemd/system/blackbox_exporter.service
[Unit]
Description=blackbox_exporter
After=network.target
[Service]
User=root
Type=simple
ExecStart=/usr/local/blackbox_exporter/blackbox_exporter --config.file=/usr/local/blackbox_exporter/blackbox.yml
Restart=on-failure
[Install]
WantedBy=multi-user.target
# systemctl daemon-reload
# systemctl start blackbox_exporter.service
# systemctl enable blackbox_exporter.service
配置文件参考：
modules:
http_2xx:
prober: http
http_post_2xx:
prober: http
http:
method: POST
http_4xx: #4xx状态监控
prober: http
http:
valid_http_versions: ["HTTP/1.1", "HTTP/2"] #协议版本
valid_status_codes: [401,403,404] #自定义正常响应的状态码
tcp_connect:
prober: tcp
pop3s_banner:
prober: tcp
tcp:
query_response:
- expect: "^+OK"
tls: true
tls_config:
insecure_skip_verify: false
ssh_banner:
prober: tcp
tcp:
query_response:
- expect: "^SSH-2.0-"
irc_banner:
prober: tcp
tcp:
query_response:
- send: "NICK prober"
- send: "USER prober prober prober :prober"
- expect: "PING :([^ ]+)"
send: "PONG ${1}"
- expect: "^:[^ ]+ 001"
icmp:
prober: icmp
三、配置prometheus采集数据
# cd /usr/local/prometheus
# vim prometheus.yml
# my global config
global:
scrape_interval: 15s
evaluation_interval: 15s
alerting:
alertmanagers:
- static_configs:
- targets:
rule_files:
# - "first_rules.yml"
# - "second_rules.yml"
scrape_configs:
- job_name: ‘prometheus‘
static_configs:
- targets: [‘192.168.5.237:9090‘]
- job_name: ‘node‘
file_sd_configs:
- refresh_interval: 30s
files:
- "/usr/local/prometheus/sd_config/node*.yml"
- job_name: ‘docker‘
static_configs:
- targets: [‘192.168.5.85:8080‘]
- job_name: ‘mysql‘
static_configs:
- targets: [‘192.168.5.237:9104‘]
labels:
app: zabbix-server-mysql
# 网站监控
- job_name: ‘http_status‘
metrics_path: /probe
params:
module: [http_2xx]
static_configs:
- targets: [‘http://www.lzfn.com‘, ‘wiki.lzfn.com‘]
labels:
instance: http_status
group: web
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- target_label: __address__
replacement: 192.168.5.85:9115
# ping 检测
- job_name: ‘ping_status‘
metrics_path: /probe
params:
module: [icmp]
static_configs:
- targets: [‘192.168.5.85‘]
labels:
instance: ‘ping_status‘
group: ‘icmp‘
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- target_label: __address__
replacement: 192.168.5.85:9115
# 端口监控
- job_name: ‘port_status‘
metrics_path: /probe
params:
module: [tcp_connect]
static_configs:
- targets: [‘192.168.5.85:80‘, ‘192.168.5.85:9000‘, ‘192.168.5.85:8080‘]
labels:
instance: ‘port_status‘
group: ‘port‘
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- target_label: __address__
replacement: 192.168.5.85:9115
# 检查配置文件并重启服务
# ./promtool check config prometheus.yml
Checking prometheus.yml
SUCCESS: 0 rule files found
# systemctl restart prometheus.service
四、配置Grafana展示数据
使用9965模板，需要安装饼状态图插件，安装后重启grafana
grafana-cli plugins install grafana-piechart-panel
systemctl restart grafana-server.service
五、k8s集群部署auto-blackbox-exporter
(一)在线部署
# Add the Helm repository
helm repo add auto-blackbox-exporter https://oguzhan-yilmaz.github.io/auto-blackbox-exporter/
# Update the repository
helm repo update auto-blackbox-exporter
# print Helm Manifests that'd apply to K8s
helm template auto-blackbox-exporter/auto-blackbox-exporter --dry-run=server
# Install the chart
helm install -n monitoring \
auto-blackbox-exporter auto-blackbox-exporter/auto-blackbox-exporter
(二)本地chart离线部署：
1、下载仓库
git clone https://github.com/oguzhan-yilmaz/auto-blackbox-exporter.git
2、更新依赖仓库配置
cd auto-blackbox-exporter/
Update dependencies:
helm dependency update
注意：依赖仓库在线下下载可能失败，可以离线下载指定为离线helm包auto-blackbox-exporter
下载地址：https://github.com/prometheus-community/helm-charts.git
root@dev-shared:/tmp/auto-blackbox-exporter# ls
Chart.lock Chart.yaml LICENSE README.md add.txt argocd-app.yaml auto-blackbox-exporter charts dev.README.md helm-charts prometheus-blackbox-exporter templates values.yaml
修改Chart.yaml：
apiVersion: v2
name: auto-blackbox-exporter
description: Auto generate a prometheus scrape config for existing Ingress objects. Includes Uptime and SSL Certificate Expiry Alerts.
type: application
version: 0.1.0
appVersion: "1.16.0"
dependencies:
- name: prometheus-blackbox-exporter
version: ">= 9.3.0"
repository: "file://./prometheus-blackbox-exporter"
condition: blackboxExporter.install
再更新依赖
helm dependency update
root@dev-shared:/tmp/auto-blackbox-exporter# ls charts/
prometheus-blackbox-exporter-11.4.1.tgz
values参数：
blackboxExporter:
install: false #是否开启blackboxExporter部署，如果已部署获取有其它blackboxExporter，可以关闭
#blackboxExporter的模块采集参数，自定义一些状态
prometheus-blackbox-exporter:
config:
modules:
http_2xx:
prober: http
timeout: 15s
http:
valid_status_codes: [200, 201, 202, 203, 204, 205, 206, 207, 208, 226, 401, 405, 403, 404]
valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
follow_redirects: true
preferred_ip_protocol: "ip4"
3、查看部署的配置
Generate manifests:
helm template auto-blackbox-exporter/ --dry-run=server
4、更改模板（非必须，主要时增加ingress的一些标签，便于管理）
templates/_blackbox.tpl：可以指定外部的lackbox-exporter地址
ingress的相关信息获取模板：
root@dev-shared:/tmp/auto-blackbox-exporter# cat templates/_dynamic_ingress.tpl
{{- define "blackbox.fetch-ingresses" -}}
hosts:
{{- range $ingressobj := (lookup "networking.k8s.io/v1" "Ingress" "" "").items }}
{{- $ingressName := $ingressobj.metadata.name }}
{{- $namespace := $ingressobj.metadata.namespace }}
{{- range $rule := $ingressobj.spec.rules }}
{{- $baseHost := (or (hasPrefix "https://" $rule.host) (hasPrefix "http://" $rule.host)) | ternary $rule.host (printf "http://%s" $rule.host) }}
{{- if $rule.http }}
{{- range $path := $rule.http.paths }}
- {{ $baseHost }}{{ $path.path | default "/" }}
{{- end }}
{{- else }}
- {{ $baseHost }}/
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- define "blackbox.ingress" -}}
{{- $ingressList := (include "blackbox.fetch-ingresses" . | fromYaml) -}}
{{- $hosts := list }}
{{- if $ingressList.hosts }}
{{- $hosts = concat $hosts $ingressList.hosts }}
{{- end }}
{{- if .Values.additionalHosts }}
{{- $hosts = concat $hosts .Values.additionalHosts }}
{{- end }}
hosts:
{{ toYaml $hosts | indent 2 }}
{{- end }}
Prometheus采集的配置文件：
root@dev-shared:/tmp/auto-blackbox-exporter# cat templates/additional-scape-config-secret.yaml
{{ $ingressList := (include "blackbox.ingress" . | fromYaml) -}}
{{- if $ingressList}}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "blackbox.name" .}}-scrape-config
labels:
{{- include "blackbox.prometheus-additional-scrape-config-labels" . | nindent 4 }}
{{- include "blackbox.selectorLabels" . | nindent 4 }}
stringData:
{{ .Values.configObjKeyName }}: |-
- job_name: {{ .Values.prometheusJobName }}
metrics_path: /probe
scrape_timeout: {{ .Values.prometheus.scrape_timeout }}
scrape_interval: {{ .Values.prometheus.scrape_interval }}
params:
module: [http_2xx] # Send a "module" HTTP parameter to the exporter to select the right probe module.
static_configs:
{{- if $ingressList.hosts }}
{{- range $host := $ingressList.hosts }}
- targets:
- {{ $host }}
labels:
env: 'prod'
{{- /* 为每个host查找对应的ingress名称和namespace */}}
{{- $ingressName := "" }}
{{- $ingressNamespace := "" }}
{{- $isAdditionalHost := false }}
{{- /* 检查是否是additionalHosts中的host */}}
{{- if $.Values.additionalHosts }}
{{- range $additionalHost := $.Values.additionalHosts }}
{{- if eq $additionalHost $host }}
{{- $isAdditionalHost = true }}
{{- end }}
{{- end }}
{{- end }}
{{- if not $isAdditionalHost }}
{{- /* 从Ingress对象中查找对应的name和namespace */}}
{{- range $ingressobj := (lookup "networking.k8s.io/v1" "Ingress" "" "").items }}
{{- range $rule := $ingressobj.spec.rules }}
{{- $baseHost := (or (hasPrefix "https://" $rule.host) (hasPrefix "http://" $rule.host)) | ternary $rule.host (printf "http://%s" $rule.host) }}
{{- if $rule.http }}
{{- range $path := $rule.http.paths }}
{{- $fullPath := printf "%s%s" $baseHost ($path.path | default "/") }}
{{- if eq $fullPath $host }}
{{- $ingressName = $ingressobj.metadata.name }}
{{- $ingressNamespace = $ingressobj.metadata.namespace }}
{{- end }}
{{- end }}
{{- else }}
{{- $fullPath := printf "%s/" $baseHost }}
{{- if eq $fullPath $host }}
{{- $ingressName = $ingressobj.metadata.name }}
{{- $ingressNamespace = $ingressobj.metadata.namespace }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- if and $ingressName $ingressNamespace }}
ingress_name: '{{ $ingressName }}'
ingress_namespace: '{{ $ingressNamespace }}'
{{- end }}
{{- end }}
{{- else }}
- targets:
- error--you-have-no-ingress-objects.auto-blackbox-exporter.io
labels:
env: 'prod'
{{- end }}
relabel_configs:
# Set the "target" HTTP parameter to the target URL that we want to probe.
- source_labels: [__address__]
target_label: __param_target
# Set the "instance" label to the target URL that we want to probe.
- source_labels: [__param_target]
target_label: instance
# Don't actually scrape the target itself, but the Blackbox Exporter.
- target_label: __address__
replacement: {{ (include "blackbox.blackbox-service-endpoint" . | fromYaml).uri }}
{{- end }}
5、配置kube-Prometheus自动发现的相关配置
查看kubectl edit prometheus monitoring prometheus -n monitoring
是否有如下配置：
spec:
additionalScrapeConfigs:
key: auto-blackbox-exporter-config.yaml
name: auto-blackbox-exporter-scrape-config
scrapeConfigSelector:
matchLabels:
release: monitoring
6、部署
helm upgrade --install \
-n monitoring \
auto-blackbox-exporter auto-blackbox-exporter/
|