Confluent Kafka Monitoring with Zabbix

Zabbix Server

Enable Zabbix JMX Gateway

Install zabbix-java-gataway
```
yum install -y zabbix-java-gataway
```
Configuring zabbix-java-gataway
```
vi /etc/zabbix/zabbix_java_gateway.conf
```
set START_POLLERS=10
Configuring zabbix-server
```
vi /etc/zabbix/zabbix_server.conf
```
set to StartJavaPollers=5 Change IP for JavaGateway=IP_address_java_gateway

restart zabbix-server and zabbix-java-gataway

systemctl restart zabbix-server
systemctl start zabbix-java-gataway
systemctl enable zabbix-java-gataway

Install scripts for discovery JMX at Zabbix Server

# cd /tmp/
# git clone https://github.com/peepeepopapapeepeepo/zabbix-kafka-monitoring.git
# cd zabbix-kafka-monitoring/zabbix-server/
# cp jmx_discovery /etc/zabbix/externalscripts
# cp JMXDiscovery-0.0.1.jar /etc/zabbix/externalscripts
# chmod +x /etc/zabbix/externalscripts/jmx_discovery
# chmod +x /etc/zabbix/externalscripts/JMXDiscovery-0.0.1.jar

Kafka Each Servers

Enable JMX Kafka Server

Create authentication files

# cd /etc/kafka/
# _Username='kafka'
# _Password='k@fKaZaB'
# echo -e "${_Username}\t${_Password}" >> jmxremote.password
# echo -e "${_Username}\treadonly" >> jmxremote.access
# chown cp-kafka.confluent jmxremote.*
# chmod 600 jmxremote.*

Edit systemd startup file

# systemctl edit confluent-kafka
     | [Service]
     |                  :                       :
    +| Environment="KAFKA_JMX_OPTS=-Dcom.sun.management.jmxremote=true \
    +| -Dcom.sun.management.jmxremote.ssl=false \
    +| -Djava.net.preferIPv4Stack=true \
    +| -Dcom.sun.management.jmxremote.local.only=false \
    +| -Dcom.sun.management.jmxremote.port=12345 \
    +| -Dcom.sun.management.jmxremote.rmi.port=12345 \
    +| -Djava.rmi.server.hostname=<HostIP> \
    +| -Dcom.sun.management.jmxremote.password.file=/etc/kafka/jmxremote.password \
    +| -Dcom.sun.management.jmxremote.access.file=/etc/kafka/jmxremote.access"
# systemctl restart confluent-kafka

Test JMX

# mkdir /root/jmxterm && cd /root/jmxterm
# curl -sLO https://github.com/jiaqi/jmxterm/releases/download/v1.0.1/jmxterm-1.0.1-uber.jar
# java -jar jmxterm-1.0.1-uber.jar -l 127.0.0.1:12345 -u kafka -p k@fKaZaB
$> beans
$> info -b java.lang:type=Threading
$> get -b kafka.server:name=PartitionCount,type=ReplicaManager Value
$> get -b java.lang:type=Threading TotalStartedThreadCount
$> exit
# mkdir /root/jmxdump && cd /root/jmxdump
# curl -sLO https://github.com/r4um/jmx-dump/releases/download/0.9.3/jmx-dump-0.9.3-standalone.jar
# java -jar jmx-dump-0.9.3-standalone.jar -h 127.0.0.1 -p 12345 -c kafka:k@fKaZaB --dump-all

Allow firewalld

# firewall-cmd --permanent --new-ipset=jmx-zabbix --type=hash:ip
# firewall-cmd --reload
# firewall-cmd --permanent --ipset=jmx-zabbix --add-entry=<IP_address_java_gateway>
# firewall-cmd --reload
# firewall-cmd --permanent --new-service=jmx-zabbix
# firewall-cmd --permanent --service=jmx-zabbix --set-description="JMX for Zabbix JMX Gateway"
# firewall-cmd --permanent --service=jmx-zabbix --set-short="JMX for Zabbix"
# firewall-cmd --permanent --service=jmx-zabbix --add-port=12345/tcp
# firewall-cmd --reload
# firewall-cmd --permanent --new-zone jmx-zabbix
# firewall-cmd --reload
# firewall-cmd --permanent --zone=jmx-zabbix --add-source=ipset:jmx-zabbix
# firewall-cmd --permanent --zone=jmx-zabbix --add-service=jmx-zabbix
# firewall-cmd --reload
# firewall-cmd --info-ipset=jmx-zabbix
# firewall-cmd --zone=jmx-zabbix --list-all
# firewall-cmd --info-service=jmx-zabbix

Install Burrow

Install Golang

# cd /tmp
# curl -sLO https://storage.googleapis.com/golang/go1.13.linux-amd64.tar.gz
# tar -C /usr/local -xzf /tmp/go1.13.linux-amd64.tar.gz
# ln -s /usr/local/go/bin/go /usr/local/bin/go
# ln -s /usr/local/go/bin/godoc /usr/local/bin/godoc
# ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt
# export GOROOT=/usr/local/go
# export PATH=$PATH:$GOROOT/bin
# vi ~/.bash_profile
  +| export GOROOT=/usr/local/go
  +| export PATH=$PATH:$GOROOT/bin
# go version

Compile and install Burrow

# mkdir -p /workspace/go/src/github.com/linkedin
# cd /tmp
# curl -sLO https://raw.githubusercontent.com/pote/gpm/v1.4.0/bin/gpm
# chmod +x gpm
# mv gpm /usr/local/bin/
# yum install gcc
# export GOPATH=/workspace/go
# cd /workspace/go/src/github.com/linkedin/
# git clone https://github.com/linkedin/Burrow.git
# cd Burrow
# go mod tidy
# go install
# ls -l $GOPATH/bin/Burrow

Configure Burrow

# mkdir -p /opt/burrow/bin/
# cp $GOPATH/bin/Burrow /opt/burrow/bin/
# cp -r config /opt/burrow/
# cd /opt/burrow/config
# cp burrow.toml burrow.toml.org

edit burrow.toml to match your environment

# vi burrow.toml
    +| [general]
    +| pidfile="burrow.pid"
    +| stdout-logfile="burrow.out"
    +| access-control-allow-origin="*"
    +|  
    +| [logging]
    +| filename="logs/burrow.log"
    +| level="info"
    +| maxsize=100
    +| maxbackups=30
    +| maxage=10
    +| use-localtime=false
    +| use-compression=true
    +|  
    +| [zookeeper]
    +| servers=[ "kk1.kafka.local:2181", "kk2.kafka.local:2181", "kk3.kafka.local:2181" ]
    +| timeout=6
    +| root-path="/burrow"
    +|  
    +| [client-profile.test]
    +| client-id="burrow-test"
    +| kafka-version="0.10.0"
    +|  
    +| [cluster.local]
    +| class-name="kafka"
    +| servers=[ "kk1.kafka.local:9092", "kk2.kafka.local:9092", "kk3.kafka.local:9092" ]
    +| client-profile="test"
    +| topic-refresh=120
    +| offset-refresh=30
    +|  
    +| [consumer.local]
    +| class-name="kafka"
    +| cluster="local"
    +| servers=[ "kk1.kafka.local:9092", "kk2.kafka.local:9092", "kk3.kafka.local:9092" ]
    +| client-profile="test"
    +| group-blacklist="^(console-consumer-|python-kafka-consumer-|quick-).*$"
    +| group-whitelist=""
    +|  
    +| [consumer.local_zk]
    +| class-name="kafka_zk"
    +| cluster="local"
    +| servers=[ "kk1.kafka.local:2181", "kk2.kafka.local:2181", "kk3.kafka.local:2181" ]
    +| zookeeper-path="/kafka-cluster"
    +| zookeeper-timeout=30
    +| group-blacklist="^(console-consumer-|python-kafka-consumer-|quick-).*$"
    +| group-whitelist=""
    +|  
    +| [httpserver.default]
    +| address="127.0.0.1:8000"
    +|  
    +| [storage.default]
    +| class-name="inmemory"
    +| workers=20
    +| intervals=15
    +| expire-group=604800
    +| min-distance=1

Install startup script and start Burrow as linux service

# cd /tmp/
# git clone https://github.com/peepeepopapapeepeepo/zabbix-kafka-monitoring.git
# cd zabbix-kafka-monitoring/kafka-consumer/
# cp burrow.service /etc/systemd/service/
# systemctl daemon-reload
# systemctl start burrow.service
# systemctl enable burrow.service

Functional Test

# curl --noproxy "*" -s http://127.0.0.1:8000/v3/kafka/local/consumer | jq

Configure Zabbix Agent

Install kafka_consumers.sh

# mkdir /var/lib/zabbix/
# cd /tmp/zabbix-kafka-monitoring/kafka-consumer/
# cp kafka_consumers.sh /var/lib/zabbix/
# chmod +x /var/lib/zabbix/kafka_consumers.sh
# chown -R zabbix.zabbix /var/lib/zabbix/

Install zabbix-agent configuration

# cp userparameter_kafkaconsumer.conf /etc/zabbix/zabbix_agentd.d
# systemctl restart zabbix-agent

Zabbix Web GUI

Import template and value mapping file

Import template (zbx_kafka_templates.xml, zbx_templates_kafkaconsumers.xml)

Configuration > Templates > Import > Choose Files > Import
Import value mapping (zbx_valuemaps_kafkaconsumers.xml)

Administration > General > Drop Down List: Value Mapping > Import > Choose Files > Import

Note: Don't forget to set macro {$JMX_PASS} and {$JMX_USER}