共计 1148 个字符，预计需要花费 3 分钟才能阅读完成。
一、Kafka 2.2.2
1、server.properties 配置文件
broker.id=128
listeners=PLAINTEXT://cwbg001:9092
num.network.threads=3
num.io.threads=4
socket.send.buffer.bytes=1024000
socket.receive.buffer.bytes=1024000
socket.request.max.bytes=104857600
log.dirs=/home/kafka/kafka_2.12-2.2.2/data
num.partitions=3
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.32.128:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=3000二、Filebeat7.9.3
1、filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /home/elastic/elasticsearch-7.9.3/logs/elasticsearch_server.json
fields:
log_topic: 'prod-app-service-name-app-prod'
exclude_files: [".tar$",".tgz$",".gz$",".bz2$",".zip$"]
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
# output kafka
output.kafka:
hosts: ["192.168.32.128:9092"]
topic: '%{[fields.log_topic]}'
partition.round_robin:
reachable_only: true
required_acks: 1
compression: gzip
max_message_bytes: 1000000三、logstash 7.9.1
1、logstash.conf
正文完