标签：none false 展示 color auto Nginx mode 日志 type
一、 效果展示
二、流程
三、 Nginx 配置
nginx.conf，修改日志格式，指定需要收集的字段及字段分割
http{
........
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent"';
........
}
四、 Filebeat 配置
修改filebeat.yml
egrep -v "^#|^ #|^$" filebeat.yml
filebeat.inputs: - type: log enabled: true paths: - /var/log/nginx/xxx.log fields: file_type: nginx-log fields_under_root: true encoding: utf-8 filebeat.config.modules: path: ${path.config}/modules.d/*.yml reload.enabled: false setup.template.settings: index.number_of_shards: 1 setup.kibana: output.logstash: hosts: ["xxx.xxx.xxx.xxx:xxxxx"] processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - add_docker_metadata: ~ - add_kubernetes_metadata: ~
五、 Logstash 配置
安装geoip插件
./bin/logstash-plugin install logstash-filter-geoip
修改logstash.conf
input {
beats {
host => '0.0.0.0'
port => xxxxx
codec => plain{ charset => "GBK" }
}
}
filter {
if [file_type] == "nginx-log"{
grok {
match => [
'message', '%{IPORHOST:remote_addr} - %{NOTSPACE:remote_user} \[%{HTTPDATE:time_local}\] \"%{WORD:method} %{NOTSPACE:request_uri} %{URIPROTO:proto}/%{NUMBER:httpversion}\" %{NUMBER:status} (?:%{NUMBER:size}|-) %{QS:referrer} %{QS:user_agent}'
]
}
geoip {
source => "remote_addr"
fields => ["location","country_name","city_name","region_name"]
}
mutate {
remove_field => ["host", "agent"]
}
}
}
output {
if [file_type] == "nginx-log"{
elasticsearch {
hosts => ["xxx.xxx.xxx.xxx:xxxxx"]
index => "%{file_type}-%{+YYYY.MM.dd}"
user => 'xxxx'
password => 'xxxxxx'
}
}
}
六、 Elasticsearch 配置
没有需要特别配置的，保证运行正常就行
七、 Grafana 添加数据源
八、 Json导出（只展示panels）
"panels": [
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 0,
"y": 0
},
"id": 12,
"options": {
"colorMode": "none",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"last"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_uptime{ident=\"$host\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "运行时间",
"type": "stat"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 2
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 4,
"y": 0
},
"id": 20,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_variables_innodb_flush_log_at_trx_commit{ident=\"$host\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Variables_innodb_flush_log_at_trx_commit",
"type": "stat"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 8388608
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 7,
"y": 0
},
"id": 18,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_variables_max_allowed_packet{ident=\"$host\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Variables_max_allowed_packet",
"type": "stat"
},
{
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 2097152
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 10,
"y": 0
},
"id": 14,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_variables_sort_buffer_size{ident=\"$host\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Variables_sort_buffer_size",
"type": "stat"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 2097152
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 13,
"y": 0
},
"id": 16,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_variables_join_buffer_size{ident=\"$host\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Variables_join_buffer_size",
"type": "stat"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 536870912
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 16,
"y": 0
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_variables_innodb_buffer_pool_size{ident=\"$host\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Variables_innodb_buffer_pool_size",
"type": "stat"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 0.95
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 20,
"y": 0
},
"id": 22,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "(mysql_innodb_buffer_pool_read_requests{ident=\"$host\"} - mysql_innodb_buffer_pool_reads{ident=\"$host\"}) / mysql_innodb_buffer_pool_read_requests{ident=\"$host\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "缓存命中率",
"type": "stat"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 37,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-blue",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 4
},
"id": 32,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "procstat_memory_data{ident=\"$host\",user=\"mysql\"}",
"interval": "",
"legendFormat": "memory_data",
"refId": "A"
}
],
"title": "内存占用",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-blue",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 4
},
"id": 34,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "procstat_cpu_usage{ident=\"$host\",user=\"mysql\"}",
"interval": "",
"legendFormat": "cpu_usage",
"refId": "A"
}
],
"title": "cpu使用",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 59,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-green",
"value": null
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 4
},
"id": 8,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_process_list_threads_idle{ident=\"$host\"}",
"interval": "",
"legendFormat": "process_list",
"refId": "A"
}
],
"title": "进程数",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 34,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-blue",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 12
},
"id": 26,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_threads_created{ident=\"$host\"}",
"interval": "",
"legendFormat": "threads_created",
"refId": "A"
}
],
"title": "已创建线程",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 40,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-blue",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 12
},
"id": 28,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_threads_running{ident=\"$host\"}",
"interval": "",
"legendFormat": "threads_running",
"refId": "A"
}
],
"title": "在运行线程",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 42,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-blue",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 12
},
"id": 30,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_threads_connected{ident=\"$host\"}",
"interval": "",
"legendFormat": "threads_connected",
"refId": "A"
}
],
"title": "已连接线程",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds",
"seriesBy": "last"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 1,
"drawStyle": "line",
"fillOpacity": 52,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-blue",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 20
},
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.4.5",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_innodb_buffer_pool_bytes_data{ident=\"$host\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "innodb_buffer_pool_bytes_data",
"refId": "A"
}
],
"title": "缓存池数据大小",
"type": "timeseries"
},
{
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 50,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "light-green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 20
},
"id": 24,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "CHsKN4Unz"
},
"exemplar": true,
"expr": "mysql_slow_queries{ident=\"$host\"}",
"interval": "",
"legendFormat": "slow_queries",
"refId": "A"
}
],
"title": "慢查询",
"type": "timeseries"
}
]
标签：none,false,展示,color,auto,Nginx,mode,日志,type 来源： https://www.cnblogs.com/Jupiter-blog/p/16225138.html
本站声明： 1. iCode9 技术分享网（下文简称本站）提供的所有内容，仅供技术学习、探讨和分享； 2. 关于本站的所有留言、评论、转载及引用，纯属内容发起人的个人观点，与本站观点和立场无关； 3. 关于本站的所有言论和文字，纯属内容发起人的个人观点，与本站观点和立场无关； 4. 本站文章均是网友提供，不完全保证技术分享内容的完整性、准确性、时效性、风险性和版权归属；如您发现该文章侵犯了您的权益，可联系我们第一时间进行删除； 5. 本站为非盈利性的个人网站，所有内容不会用来进行牟利，也不会利用任何形式的广告来间接获益，纯粹是为了广大技术爱好者提供技术内容和技术思想的分享性交流网站。