Java.rmi.ConnectException: Connection refused to host: 127.0.0.1
sureshoao opened this issue · 2 comments
kafka-prometheus-jmx-exporter is unable to connect to Kafka broker.
Jul 14, 2019 9:35:42 AM io.prometheus.jmx.JmxCollector collect
SEVERE: JMX scrape failed: java.io.IOException: Failed to retrieve RMIServer stub: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: 127.0.0.1; nested exception is:
java.net.ConnectException: Connection refused (Connection refused)]
at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:369)
at javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
at io.prometheus.jmx.JmxScraper.doScrape(JmxScraper.java:106)
at io.prometheus.jmx.JmxCollector.collect(JmxCollector.java:415)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.findNextElement(CollectorRegistry.java:180)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:213)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:134)
at io.prometheus.client.exporter.common.TextFormat.write004(TextFormat.java:22)
at io.prometheus.client.exporter.HTTPServer$HTTPMetricHandler.handle(HTTPServer.java:59)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)
at sun.net.httpserver.AuthFilter.doFilter(AuthFilter.java:83)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:82)
at sun.net.httpserver.ServerImpl$Exchange$LinkHandler.handle(ServerImpl.java:675)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)
at sun.net.httpserver.ServerImpl$Exchange.run(ServerImpl.java:647)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Please provide background such as what you applied, and please quote terminal output with ```.
Yaml file:
apiVersion: v1
kind: ConfigMap
metadata:
name: kafka-cp-jmx-configmap
namespace: kafka
labels:
app: cp-kafka
data:
jmx-kafka-prometheus.yaml: |+
jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
lowercaseOutputName: true
lowercaseOutputLabelNames: true
ssl: false
rules:
- pattern : kafka.server<type=ReplicaManager, name=(.+)><>(Value|OneMinuteRate)
name: "cp_kafka_server_replicamanager_$1"
- pattern : kafka.controller<type=KafkaController, name=(.+)><>Value
name: "cp_kafka_controller_kafkacontroller_$1"
- pattern : kafka.server<type=BrokerTopicMetrics, name=(.+)><>OneMinuteRate
name: "cp_kafka_server_brokertopicmetrics_$1"
- pattern : kafka.network<type=RequestMetrics, name=RequestsPerSec, request=(.+)><>OneMinuteRate
name: "cp_kafka_network_requestmetrics_requestspersec_$1"
- pattern : kafka.network<type=SocketServer, name=NetworkProcessorAvgIdlePercent><>Value
name: "cp_kafka_network_socketserver_networkprocessoravgidlepercent"
- pattern : kafka.server<type=ReplicaFetcherManager, name=MaxLag, clientId=(.+)><>Value
name: "cp_kafka_server_replicafetchermanager_maxlag_$1"
- pattern : kafka.server<type=KafkaRequestHandlerPool, name=RequestHandlerAvgIdlePercent><>OneMinuteRate
name: "cp_kafka_kafkarequesthandlerpool_requesthandleravgidlepercent"
- pattern : kafka.controller<type=ControllerStats, name=(.+)><>OneMinuteRate
name: "cp_kafka_controller_controllerstats_$1"
- pattern : kafka.server<type=SessionExpireListener, name=(.+)><>OneMinuteRate
name: "cp_kafka_server_sessionexpirelistener_$1"
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: kafka-cp
namespace: kafka
labels:
app: cp-kafka
spec:
serviceName: kafka-cp-headless
podManagementPolicy: OrderedReady
replicas: 5
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
app: cp-kafka
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "5556"
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
podAffinityTerm:
labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- cp-kafka
- key: "release"
operator: In
values:
- kafka
topologyKey: "kubernetes.io/hostname"
containers:
- name: prometheus-jmx-exporter
image: "solsson/kafka-prometheus-jmx-exporter:latest"
command:
- java
- -XX:+UnlockExperimentalVMOptions
- -XX:+UseCGroupMemoryLimitForHeap
- -XX:MaxRAMFraction=1
- -XshowSettings:vm
- -jar
- jmx_prometheus_httpserver.jar
- "5556"
- /etc/jmx-kafka/jmx-kafka-prometheus.yaml
ports:
- containerPort: 5556
resources:
null
volumeMounts:
- name: jmx-config
mountPath: /etc/jmx-kafka
- name: cp-kafka-broker
image: "solsson/kafka:2.2.1"
imagePullPolicy: "IfNotPresent"
readinessProbe:
tcpSocket:
port: kafka
ports:
- containerPort: 9092
name: kafka
- containerPort: 5555
name: jmx
resources:
requests:
memory: 12000Mi
cpu: 4000m
limits:
memory: 12000Mi
cpu: 4000m
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: KAFKA_HEAP_OPTS
value: -Xms12G -Xmx12G
- name: KAFKA_ZOOKEEPER_CONNECT
value: "zookeeper-headless.kafka.svc.cluster.local:2181"
- name: "KAFKA_ADVERTISED_LISTENERS"
value: "EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID}))"
- name: "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP"
value: "PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT"
- name: "KAFKA_LOG_DIRS"
value: "/opt/kafka/data/logs"
- name: "KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE"
value: "false"
- name: "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR"
value: "2"
- name: KAFKA_JMX_PORT
value: "5555"
command:
- sh
- -exc
- |
export KAFKA_BROKER_ID=${HOSTNAME##*-} && \
export KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://${POD_IP}:9092,EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID})) && \
exec /etc/confluent/docker/run
command:
- sh
- -c
- "exec kafka-server-start /etc/kafka/server.properties --override advertised.listeners=PLAINTEXT://clustername-kafka-${HOSTNAME##*-}.aws:443 \
--override zookeeper.connect=zookeeper-headless.kafka.svc.cluster.local:2181 \
--override log.dirs=/opt/kafka/data/logs \
--override broker.id=${HOSTNAME##*-} \
--override auto.create.topics.enable=true \
--override num.network.threads=3 \
--override num.io.threads=8 \
--override socket.send.buffer.bytes=102400 \
--override socket.receive.buffer.bytes=102400 \
--override socket.request.max.bytes=104857600 \
--override num.partitions=30 \
--override num.recovery.threads.per.data.dir=1 \
--override offsets.topic.replication.factor=2 \
--override transaction.state.log.replication.factor=2 \
--override transaction.state.log.min.isr=1 \
--override log.retention.hours=168 \
--override log.segment.bytes=1073741824 \
--override log.retention.check.interval.ms=300000 \
--override zookeeper.connection.timeout.ms=6000 \
--override group.initial.rebalance.delay.ms=0 \ "
volumeMounts:
- name: datadir
mountPath: /opt/kafka/data
volumes:
- name: jmx-config
configMap:
name: kafka-cp-jmx-configmap
volumeClaimTemplates:
- metadata:
name: datadir
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: "50Gi"
storageClassName: infrastructure-storage
logs of prometheus-jmx-exporter:
'''
kubectl logs -f kafka-cp-0 -n kafka -c prometheus-jmx-exporter
VM settings:
Max. Heap Size (Estimated): 13.93G
Ergonomics Machine Class: server
Using VM: OpenJDK 64-Bit Server VM
Jul 15, 2019 6:10:32 AM io.prometheus.jmx.JmxCollector collect
SEVERE: JMX scrape failed: java.io.IOException: Failed to retrieve RMIServer stub: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: localhost; nested exception is:
java.net.ConnectException: Connection refused (Connection refused)]
at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:369)
at javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
at io.prometheus.jmx.JmxScraper.doScrape(JmxScraper.java:106)
at io.prometheus.jmx.JmxCollector.collect(JmxCollector.java:415)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.findNextElement(CollectorRegistry.java:180)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:213)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:134)
at io.prometheus.client.exporter.common.TextFormat.write004(TextFormat.java:22)
at io.prometheus.client.exporter.HTTPServer$HTTPMetricHandler.handle(HTTPServer.java:59)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)
at sun.net.httpserver.AuthFilter.doFilter(AuthFilter.java:83)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:82)
at sun.net.httpserver.ServerImpl$Exchange$LinkHandler.handle(ServerImpl.java:675)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)
at sun.net.httpserver.ServerImpl$Exchange.run(ServerImpl.java:647)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: localhost; nested exception is:
java.net.ConnectException: Connection refused (Connection refused)]
at com.sun.jndi.rmi.registry.RegistryContext.lookup(RegistryContext.java:136)
at com.sun.jndi.toolkit.url.GenericURLContext.lookup(GenericURLContext.java:205)
at javax.naming.InitialContext.lookup(InitialContext.java:417)
at javax.management.remote.rmi.RMIConnector.findRMIServerJNDI(RMIConnector.java:1955)
at javax.management.remote.rmi.RMIConnector.findRMIServer(RMIConnector.java:1922)
at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:287)
... 17 more
Caused by: java.rmi.ConnectException: Connection refused to host: localhost; nested exception is:
java.net.ConnectException: Connection refused (Connection refused)
at sun.rmi.transport.tcp.TCPEndpoint.newSocket(TCPEndpoint.java:619)
at sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:216)
at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:202)
at sun.rmi.server.UnicastRef.newCall(UnicastRef.java:338)
at sun.rmi.registry.RegistryImpl_Stub.lookup(RegistryImpl_Stub.java:112)
at com.sun.jndi.rmi.registry.RegistryContext.lookup(RegistryContext.java:132)
... 22 more
Caused by: java.net.ConnectException: Connection refused (Connection refused)
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at java.net.Socket.connect(Socket.java:538)
at java.net.Socket.(Socket.java:434)
at java.net.Socket.(Socket.java:211)
at sun.rmi.transport.proxy.RMIDirectSocketFactory.createSocket(RMIDirectSocketFactory.java:40)
at sun.rmi.transport.proxy.RMIMasterSocketFactory.createSocket(RMIMasterSocketFactory.java:148)
at sun.rmi.transport.tcp.TCPEndpoint.newSocket(TCPEndpoint.java:613)
... 27 more
Jul 15, 2019 6:10:35 AM io.prometheus.jmx.JmxCollector collect
SEVERE: JMX scrape failed: java.io.IOException: Failed to retrieve RMIServer stub: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: localhost; nested exception is:
java.net.ConnectException: Connection refused (Connection refused)]
at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:369)
at javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
at io.prometheus.jmx.JmxScraper.doScrape(JmxScraper.java:106)
at io.prometheus.jmx.JmxCollector.collect(JmxCollector.java:415)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.findNextElement(CollectorRegistry.java:180)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:213)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:134)
at io.prometheus.client.exporter.common.TextFormat.write004(TextFormat.java:22)
at io.prometheus.client.exporter.HTTPServer$HTTPMetricHandler.handle(HTTPServer.java:59)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)
at sun.net.httpserver.AuthFilter.doFilter(AuthFilter.java:83)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:82)
at sun.net.httpserver.ServerImpl$Exchange$LinkHandler.handle(ServerImpl.java:675)
'''