关于ELK:ELK79-及Kafka-222-安装与配置

一、Kafka 2.2.2

1、server.properties 配置文件

broker.id=128
listeners=PLAINTEXT://cwbg001:9092
num.network.threads=3
num.io.threads=4
socket.send.buffer.bytes=1024000
socket.receive.buffer.bytes=1024000
socket.request.max.bytes=104857600
log.dirs=/home/kafka/kafka_2.12-2.2.2/data
num.partitions=3
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.32.128:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=3000

二、Filebeat7.9.3

1、filebeat.yml

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /home/elastic/elasticsearch-7.9.3/logs/elasticsearch_server.json
  fields:
    log_topic: 'prod-app-service-name-app-prod'
  exclude_files: [".tar$",".tgz$",".gz$",".bz2$",".zip$"]
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
# output kafka
output.kafka:
  hosts: ["192.168.32.128:9092"]
  topic: '%{[fields.log_topic]}'
  partition.round_robin:
    reachable_only: true
  required_acks: 1
  compression: gzip
  max_message_bytes: 1000000

三、logstash 7.9.1

1、logstash.conf