标签：fluentd compose log -- EFK elasticsearch docker yml

1 整体目录结构

root@shutang:/home/shutang/docker-scripts# tree log-scripts/
log-scripts/
|-- README.MD
|-- curator
|   |-- Dockerfile
|   |-- curator_prod.yml
|   |-- curator_qa.yml
|   |-- delete_indices.yml
|   `-- docker-compose.yml
|-- docker-compose.yml
|-- es
|   `-- elasticsearch.yml
|-- fluentd
|   |-- Dockerfile
|   `-- conf
|       `-- fluent.conf
`-- kibana
    |-- Dockerfile
    `-- kibana.yml

2 部署efk服务的docker-compose文件

version: '3'

services:
  fluentd:
    build: ./fluentd
    restart: always
    volumes:
      - /mnt/disks/append-disk/fluentd/log:/fluentd/log
      - ./fluentd/conf/fluent.conf:/fluentd/etc/fluent.conf
    ports:
      - "24224:24224"
      - "24224:24224/udp"
    environment:
      - "discovery.zen.ping.unicast.hosts=elasticsearch"
    networks:
      - esnet

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:6.8.22
    container_name: elasticsearch
    restart: always
    environment:
      - node.name=master1
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms6g -Xmx6g"
      - network.publish_host=10.175.100.100
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esdata1:/usr/share/elasticsearch/data
      - ./es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    expose:
      - 9200
      - 9300
    ports:
      - 9200:9200
      - 9300:9300
    networks:
      - esnet

  kibana:
    build: ./kibana
    restart: always
    environment:
      - "discovery.zen.ping.unicast.hosts=elasticsearch"
    ports:
      - "5601:5601"
    networks:
      - esnet
    volumes:
      - ./kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
volumes:
  esdata1:
    driver: local
networks:
  esnet:

2 es/elasticsearch.yml 文件 [两个es实例同时作为master]

cluster.name: "docker-cluster"
network.host: 0.0.0.0

# minimum_master_nodes need to be explicitly set when bound on a public IP
# set to 1 to allow single node clusters
# Details: https://github.com/elastic/elasticsearch/pull/17288
discovery.zen.minimum_master_nodes: 1
discovery.zen.ping.unicast.hosts: ["10.175.100.100", "10.175.100.101"]
xpack.security.enabled: false

3 fluentd/Dockerfile 文件

FROM fluent/fluentd:v1.5

USER root

RUN echo "source 'https://mirrors.tuna.tsinghua.edu.cn/rubygems/'" > Gemfile && gem install bundler

RUN gem install fluent-plugin-elasticsearch -v 4.0.3   --no-document
#RUN gem install fluent-plugin-elasticsearch  --no-document
RUN gem install fluent-plugin-concat

USER fluent

CMD ["fluentd", "-o", "/fluentd/log/fluentd.log", "--log-rotate-age", "daily", "--log-rotate-size", "102400"]

4 /fluentd/conf/fluent.conf

<source>
  @type forward
  port 24224
  bind 0.0.0.0
</source>

<filter *.log>
  @type concat
  @log_level trace
  key log
  multiline_start_regexp /^\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}|^- GET|^\d{4}-\d{1,2}-\d{1,2}.\d{1,2}:\d{1,2}:\d{1,2}/
  multiline_end_regexp /\n$/
</filter>

<match *.log>
  @type copy
  <store>
    @type elasticsearch
    host 10.175.100.100
    port 9200
    logstash_format true
    logstash_prefix mxlog
    logstash_dateformat %Y%m%d
    include_tag_key true
    tag_key @log_name
    flush_interval 1s
  </store>
  <store>
    @type stdout
  </store>
</match>

<match *.**>
  @type copy
  <store>
    @type elasticsearch
    host 10.175.100.100
    port 9200
    logstash_format true
    logstash_prefix log
    logstash_dateformat %Y%m%d
    include_tag_key true
    tag_key @log_name
    flush_interval 1s
  </store>
  <store>
    @type stdout
  </store>
</match>

5 10.175.100.101 部署master2

root@shutang:/home/shutang/docker-scripts/log-scripts# cat es2-dc.yml
version: '3'

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:6.8.22
    container_name: elasticsearch-slave
    restart: always
    environment:
      - node.name=master2
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - network.publish_host=10.175.100.101
    expose:
      - 9200
      - 9300
    ports:
      - 9200:9200
      - 9300:9300
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - ./es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /mnt/disks/append/es-data:/usr/share/elasticsearch/data

6 es/elasticsearch.yml

cluster.name: "docker-cluster"
network.host: 0.0.0.0

# minimum_master_nodes need to be explicitly set when bound on a public IP
# set to 1 to allow single node clusters
# Details: https://github.com/elastic/elasticsearch/pull/17288
discovery.zen.minimum_master_nodes: 1
discovery.zen.ping.unicast.hosts: ["10.175.100.100", "10.176.100.101"]

标签：fluentd,compose,log,--,EFK,elasticsearch,docker,yml
来源： https://blog.csdn.net/weixin_48505120/article/details/122876699

本站声明： 1. iCode9 技术分享网（下文简称本站）提供的所有内容，仅供技术学习、探讨和分享；
2. 关于本站的所有留言、评论、转载及引用，纯属内容发起人的个人观点，与本站观点和立场无关；
3. 关于本站的所有言论和文字，纯属内容发起人的个人观点，与本站观点和立场无关；
4. 本站文章均是网友提供，不完全保证技术分享内容的完整性、准确性、时效性、风险性和版权归属；如您发现该文章侵犯了您的权益，可联系我们第一时间进行删除；
5. 本站为非盈利性的个人网站，所有内容不会用来进行牟利，也不会利用任何形式的广告来间接获益，纯粹是为了广大技术爱好者提供技术内容和技术思想的分享性交流网站。