Random timeouts connecting to clickhouse cloud
abhishekgahlot2 opened this issue · 5 comments
abhishekgahlot2 commented
Hi
I tried adding to aws msk connectors with following configuration
connector.class=com.clickhouse.kafka.connect.ClickHouseSinkConnector
tasks.max=1
topics=momo
ssl=true
security.protocol=SSL
hostname=n123456789.asia-something.clickhouse.cloud
database=momo
password=.n123456789
ssl.truststore.location=/tmp/kafka.client.truststore.jks
port=8443
value.converter.schemas.enable=false
value.converter=org.apache.kafka.connect.json.JsonConverter
exactlyOnce=true
username=default
schemas.enable=false
Security groups of msk has all inbound/outbound connections allowed.
I am also able to telnet from msk vpc and local host to clickhouse cloud.
Here are the connector logs
[Worker-0e0766492fd1ececf] [2024-01-31 06:43:56,822] WARN [clk-cloud|task-0] Ping retry 2 out of 3 (com.clickhouse.kafka.connect.sink.db.helper.ClickHouseHelperClient:103)
[Worker-0e0766492fd1ececf] [2024-01-31 06:44:26,852] WARN [clk-cloud|task-0] Ping retry 3 out of 3 (com.clickhouse.kafka.connect.sink.db.helper.ClickHouseHelperClient:103)
[Worker-0e0766492fd1ececf] [2024-01-31 06:44:26,852] ERROR [clk-cloud|task-0] Unable to ping ClickHouse instance. (com.clickhouse.kafka.connect.sink.db.helper.ClickHouseHelperClient:105)
[Worker-0e0766492fd1ececf] [2024-01-31 06:44:26,852] ERROR [clk-cloud|task-0] Unable to ping Clickhouse server. (com.clickhouse.kafka.connect.sink.state.provider.KeeperStateProvider:54)
[Worker-0e0766492fd1ececf] [2024-01-31 06:44:26,852] INFO [clk-cloud|task-0] Ping is successful. (com.clickhouse.kafka.connect.sink.state.provider.KeeperStateProvider:57)
[Worker-0e0766492fd1ececf] [2024-01-31 06:44:31,858] WARN [clk-cloud|task-0] Query retry 1 out of 3 (com.clickhouse.kafka.connect.sink.db.helper.ClickHouseHelperClient:133)
[Worker-0e0766492fd1ececf] com.clickhouse.client.ClickHouseException: Connect to https://neojwczqre.asia-southeast1.gcp.clickhouse.cloud:8443 [neojwczqre.asia-southeast1.gcp.clickhouse.cloud/34.160.80.214] failed: connect timed out, server ClickHouseNode [uri=https://neojwczqre.asia-southeast1.gcp.clickhouse.cloud:8443/momo, options={sslmode=STRICT}]@1336548521
[Worker-0e0766492fd1ececf] at com.clickhouse.client.ClickHouseException.of(ClickHouseException.java:164)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.AbstractClient.lambda$execute$0(AbstractClient.java:275)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1700)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
[Worker-0e0766492fd1ececf] at java.base/java.lang.Thread.run(Thread.java:829)
[Worker-0e0766492fd1ececf] Caused by: org.apache.hc.client5.http.ConnectTimeoutException: Connect to https://neojwczqre.asia-southeast1.gcp.clickhouse.cloud:8443 [neojwczqre.asia-southeast1.gcp.clickhouse.cloud/34.160.80.214] failed: connect timed out
[Worker-0e0766492fd1ececf] at java.base/java.net.PlainSocketImpl.socketConnect(Native Method)
[Worker-0e0766492fd1ececf] at java.base/java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:412)
[Worker-0e0766492fd1ececf] at java.base/java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:255)
[Worker-0e0766492fd1ececf] at java.base/java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:237)
[Worker-0e0766492fd1ececf] at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[Worker-0e0766492fd1ececf] at java.base/java.net.Socket.connect(Socket.java:609)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory.lambda$connectSocket$0(SSLConnectionSocketFactory.java:246)
[Worker-0e0766492fd1ececf] at java.base/java.security.AccessController.doPrivileged(Native Method)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:245)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.io.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:184)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:447)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.InternalExecRuntime.connectEndpoint(InternalExecRuntime.java:162)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.InternalExecRuntime.connectEndpoint(InternalExecRuntime.java:172)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ConnectExec.execute(ConnectExec.java:142)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ProtocolExec.execute(ProtocolExec.java:192)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.HttpRequestRetryExec.execute(HttpRequestRetryExec.java:96)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.RedirectExec.execute(RedirectExec.java:115)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.InternalHttpClient.doExecute(InternalHttpClient.java:170)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.CloseableHttpClient.execute(CloseableHttpClient.java:123)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.http.ApacheHttpConnectionImpl.post(ApacheHttpConnectionImpl.java:241)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.http.ClickHouseHttpClient.send(ClickHouseHttpClient.java:118)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.AbstractClient.sendAsync(AbstractClient.java:161)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.AbstractClient.lambda$execute$0(AbstractClient.java:273)
[Worker-0e0766492fd1ececf] ... 4 more
[Worker-0e0766492fd1ececf] [2024-01-31 06:44:36,866] WARN [clk-cloud|task-0] Query retry 2 out of 3 (com.clickhouse.kafka.connect.sink.db.helper.ClickHouseHelperClient:133)
[Worker-0e0766492fd1ececf] com.clickhouse.client.ClickHouseException: Connect to https://neojwczqre.asia-southeast1.gcp.clickhouse.cloud:8443 [neojwczqre.asia-southeast1.gcp.clickhouse.cloud/34.160.80.214] failed: connect timed out, server ClickHouseNode [uri=https://neojwczqre.asia-southeast1.gcp.clickhouse.cloud:8443/momo, options={sslmode=STRICT}]@1336548521
[Worker-0e0766492fd1ececf] at com.clickhouse.client.ClickHouseException.of(ClickHouseException.java:164)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.AbstractClient.lambda$execute$0(AbstractClient.java:275)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1700)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
[Worker-0e0766492fd1ececf] at java.base/java.lang.Thread.run(Thread.java:829)
[Worker-0e0766492fd1ececf] Caused by: org.apache.hc.client5.http.ConnectTimeoutException: Connect to https://neojwczqre.asia-southeast1.gcp.clickhouse.cloud:8443 [neojwczqre.asia-southeast1.gcp.clickhouse.cloud/34.160.80.214] failed: connect timed out
[Worker-0e0766492fd1ececf] at java.base/java.net.PlainSocketImpl.socketConnect(Native Method)
[Worker-0e0766492fd1ececf] at java.base/java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:412)
[Worker-0e0766492fd1ececf] at java.base/java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:255)
[Worker-0e0766492fd1ececf] at java.base/java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:237)
[Worker-0e0766492fd1ececf] at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[Worker-0e0766492fd1ececf] at java.base/java.net.Socket.connect(Socket.java:609)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory.lambda$connectSocket$0(SSLConnectionSocketFactory.java:246)
[Worker-0e0766492fd1ececf] at java.base/java.security.AccessController.doPrivileged(Native Method)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:245)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.io.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:184)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:447)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.InternalExecRuntime.connectEndpoint(InternalExecRuntime.java:162)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.InternalExecRuntime.connectEndpoint(InternalExecRuntime.java:172)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ConnectExec.execute(ConnectExec.java:142)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ProtocolExec.execute(ProtocolExec.java:192)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.HttpRequestRetryExec.execute(HttpRequestRetryExec.java:96)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.RedirectExec.execute(RedirectExec.java:115)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.InternalHttpClient.doExecute(InternalHttpClient.java:170)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.CloseableHttpClient.execute(CloseableHttpClient.java:123)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.http.ApacheHttpConnectionImpl.post(ApacheHttpConnectionImpl.java:241)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.http.ClickHouseHttpClient.send(ClickHouseHttpClient.java:118)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.AbstractClient.sendAsync(AbstractClient.java:161)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.AbstractClient.lambda$execute$0(AbstractClient.java:273)
[Worker-0e0766492fd1ececf] ... 4 more
[Worker-0e0766492fd1ececf] [2024-01-31 06:44:41,874] WARN [clk-cloud|task-0] Query retry 3 out of 3 (com.clickhouse.kafka.connect.sink.db.helper.ClickHouseHelperClient:133)
[Worker-0e0766492fd1ececf] com.clickhouse.client.ClickHouseException: Connect to https://neojwczqre.asia-southeast1.gcp.clickhouse.cloud:8443 [neojwczqre.asia-southeast1.gcp.clickhouse.cloud/34.160.80.214] failed: connect timed out, server ClickHouseNode [uri=https://neojwczqre.asia-southeast1.gcp.clickhouse.cloud:8443/momo, options={sslmode=STRICT}]@1336548521
[Worker-0e0766492fd1ececf] at com.clickhouse.client.ClickHouseException.of(ClickHouseException.java:164)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.AbstractClient.lambda$execute$0(AbstractClient.java:275)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1700)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
[Worker-0e0766492fd1ececf] at java.base/java.lang.Thread.run(Thread.java:829)
[Worker-0e0766492fd1ececf] Caused by: org.apache.hc.client5.http.ConnectTimeoutException: Connect to https://neojwczqre.asia-southeast1.gcp.clickhouse.cloud:8443 [neojwczqre.asia-southeast1.gcp.clickhouse.cloud/34.160.80.214] failed: connect timed out
[Worker-0e0766492fd1ececf] at java.base/java.net.PlainSocketImpl.socketConnect(Native Method)
[Worker-0e0766492fd1ececf] at java.base/java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:412)
[Worker-0e0766492fd1ececf] at java.base/java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:255)
[Worker-0e0766492fd1ececf] at java.base/java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:237)
[Worker-0e0766492fd1ececf] at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[Worker-0e0766492fd1ececf] at java.base/java.net.Socket.connect(Socket.java:609)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory.lambda$connectSocket$0(SSLConnectionSocketFactory.java:246)
[Worker-0e0766492fd1ececf] at java.base/java.security.AccessController.doPrivileged(Native Method)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:245)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.io.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:184)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:447)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.InternalExecRuntime.connectEndpoint(InternalExecRuntime.java:162)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.InternalExecRuntime.connectEndpoint(InternalExecRuntime.java:172)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ConnectExec.execute(ConnectExec.java:142)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ProtocolExec.execute(ProtocolExec.java:192)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.HttpRequestRetryExec.execute(HttpRequestRetryExec.java:96)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.RedirectExec.execute(RedirectExec.java:115)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.InternalHttpClient.doExecute(InternalHttpClient.java:170)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.CloseableHttpClient.execute(CloseableHttpClient.java:123)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.http.ApacheHttpConnectionImpl.post(ApacheHttpConnectionImpl.java:241)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.http.ClickHouseHttpClient.send(ClickHouseHttpClient.java:118)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.AbstractClient.sendAsync(AbstractClient.java:161)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.AbstractClient.lambda$execute$0(AbstractClient.java:273)
[Worker-0e0766492fd1ececf] ... 4 more
[Worker-0e0766492fd1ececf] [2024-01-31 06:44:41,875] ERROR [clk-cloud|task-0] WorkerSinkTask{id=clk-cloud-0} Task threw an uncaught and unrecoverable exception. Task is being killed and will not recover until manually restarted (org.apache.kafka.connect.runtime.WorkerTask:191)
[Worker-0e0766492fd1ececf] java.lang.RuntimeException: com.clickhouse.client.ClickHouseException: Connect to https://neojwczqre.asia-southeast1.gcp.clickhouse.cloud:8443 [neojwczqre.asia-southeast1.gcp.clickhouse.cloud/34.160.80.214] failed: connect timed out, server ClickHouseNode [uri=https://neojwczqre.asia-southeast1.gcp.clickhouse.cloud:8443/momo, options={sslmode=STRICT}]@1336548521
[Worker-0e0766492fd1ececf] at com.clickhouse.kafka.connect.sink.db.helper.ClickHouseHelperClient.query(ClickHouseHelperClient.java:137)
[Worker-0e0766492fd1ececf] at com.clickhouse.kafka.connect.sink.db.helper.ClickHouseHelperClient.query(ClickHouseHelperClient.java:115)
[Worker-0e0766492fd1ececf] at com.clickhouse.kafka.connect.sink.state.provider.KeeperStateProvider.init(KeeperStateProvider.java:74)
[Worker-0e0766492fd1ececf] at com.clickhouse.kafka.connect.sink.state.provider.KeeperStateProvider.<init>(KeeperStateProvider.java:58)
[Worker-0e0766492fd1ececf] at com.clickhouse.kafka.connect.sink.ProxySinkTask.<init>(ProxySinkTask.java:43)
[Worker-0e0766492fd1ececf] at com.clickhouse.kafka.connect.sink.ClickHouseSinkTask.start(ClickHouseSinkTask.java:57)
[Worker-0e0766492fd1ececf] at org.apache.kafka.connect.runtime.WorkerSinkTask.initializeAndStart(WorkerSinkTask.java:308)
[Worker-0e0766492fd1ececf] at org.apache.kafka.connect.runtime.WorkerSinkTask.execute(WorkerSinkTask.java:196)
[Worker-0e0766492fd1ececf] at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:189)
[Worker-0e0766492fd1ececf] at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:238)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
[Worker-0e0766492fd1ececf] at java.base/java.lang.Thread.run(Thread.java:829)
[Worker-0e0766492fd1ececf] Caused by: com.clickhouse.client.ClickHouseException: Connect to https://neojwczqre.asia-southeast1.gcp.clickhouse.cloud:8443 [neojwczqre.asia-southeast1.gcp.clickhouse.cloud/34.160.80.214] failed: connect timed out, server ClickHouseNode [uri=https://neojwczqre.asia-southeast1.gcp.clickhouse.cloud:8443/momo, options={sslmode=STRICT}]@1336548521
[Worker-0e0766492fd1ececf] at com.clickhouse.client.ClickHouseException.of(ClickHouseException.java:164)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.AbstractClient.lambda$execute$0(AbstractClient.java:275)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1700)
[Worker-0e0766492fd1ececf] ... 3 more
[Worker-0e0766492fd1ececf] Caused by: org.apache.hc.client5.http.ConnectTimeoutException: Connect to https://neojwczqre.asia-southeast1.gcp.clickhouse.cloud:8443 [neojwczqre.asia-southeast1.gcp.clickhouse.cloud/34.160.80.214] failed: connect timed out
[Worker-0e0766492fd1ececf] at java.base/java.net.PlainSocketImpl.socketConnect(Native Method)
[Worker-0e0766492fd1ececf] at java.base/java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:412)
[Worker-0e0766492fd1ececf] at java.base/java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:255)
[Worker-0e0766492fd1ececf] at java.base/java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:237)
[Worker-0e0766492fd1ececf] at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[Worker-0e0766492fd1ececf] at java.base/java.net.Socket.connect(Socket.java:609)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory.lambda$connectSocket$0(SSLConnectionSocketFactory.java:246)
[Worker-0e0766492fd1ececf] at java.base/java.security.AccessController.doPrivileged(Native Method)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:245)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.io.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:184)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:447)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.InternalExecRuntime.connectEndpoint(InternalExecRuntime.java:162)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.InternalExecRuntime.connectEndpoint(InternalExecRuntime.java:172)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ConnectExec.execute(ConnectExec.java:142)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ProtocolExec.execute(ProtocolExec.java:192)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.HttpRequestRetryExec.execute(HttpRequestRetryExec.java:96)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.RedirectExec.execute(RedirectExec.java:115)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.InternalHttpClient.doExecute(InternalHttpClient.java:170)
[Worker-0e0766492fd1ececf] at org.apache.hc.client5.http.impl.classic.CloseableHttpClient.execute(CloseableHttpClient.java:123)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.http.ApacheHttpConnectionImpl.post(ApacheHttpConnectionImpl.java:241)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.http.ClickHouseHttpClient.send(ClickHouseHttpClient.java:118)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.AbstractClient.sendAsync(AbstractClient.java:161)
[Worker-0e0766492fd1ececf] at com.clickhouse.client.AbstractClient.lambda$execute$0(AbstractClient.java:273)
[Worker-0e0766492fd1ececf] ... 4 more
[Worker-0e0766492fd1ececf] [2024-01-31 06:44:41,875] WARN [clk-cloud|task-0] Could not stop task (org.apache.kafka.connect.runtime.WorkerSinkTask:171)
[Worker-0e0766492fd1ececf] java.lang.NullPointerException
[Worker-0e0766492fd1ececf] at com.clickhouse.kafka.connect.sink.ClickHouseSinkTask.stop(ClickHouseSinkTask.java:85)
[Worker-0e0766492fd1ececf] at org.apache.kafka.connect.runtime.WorkerSinkTask.close(WorkerSinkTask.java:169)
[Worker-0e0766492fd1ececf] at org.apache.kafka.connect.runtime.WorkerTask.doClose(WorkerTask.java:168)
[Worker-0e0766492fd1ececf] at org.apache.kafka.connect.runtime.WorkerTask.doRun(WorkerTask.java:194)
[Worker-0e0766492fd1ececf] at org.apache.kafka.connect.runtime.WorkerTask.run(WorkerTask.java:238)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
[Worker-0e0766492fd1ececf] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
[Worker-0e0766492fd1ececf] at java.base/java.lang.Thread.run(Thread.java:829)
[Worker-0e0766492fd1ececf] [2024-01-31 06:44:41,876] INFO [clk-cloud|task-0] Metrics scheduler closed (org.apache.kafka.common.metrics.Metrics:668)
[Worker-0e0766492fd1ececf] [2024-01-31 06:44:41,876] INFO [clk-cloud|task-0] Closing reporter org.apache.kafka.common.metrics.JmxReporter (org.apache.kafka.common.metrics.Metrics:672)
[Worker-0e0766492fd1ececf] [2024-01-31 06:44:41,876] INFO [clk-cloud|task-0] Metrics reporters closed (org.apache.kafka.common.metrics.Metrics:678)
[Worker-0e0766492fd1ececf] [2024-01-31 06:44:41,877] INFO [clk-cloud|task-0] App info kafka.consumer for connector-consumer-clk-cloud-0 unregistered (org.apache.kafka.common.utils.AppInfoParser:83)
from kafka import KafkaProducer
from kafka.errors import KafkaError
import socket
import time
from aws_msk_iam_sasl_signer import MSKAuthTokenProvider
class MSKTokenProvider():
def token(self):
token, _ = MSKAuthTokenProvider.generate_auth_token('zone')
return token
tp = MSKTokenProvider()
producer = KafkaProducer(
bootstrap_servers='serverstring',
security_protocol='SASL_SSL',
sasl_mechanism='OAUTHBEARER',
sasl_oauth_token_provider=tp,
client_id=socket.gethostname(),
)
topic = "momo"
import time, random, json
while True:
try:
time.sleep(0.2)
d = {'name': str(random.random())}
producer.send(topic, json.dumps(d).encode('utf-8'))
producer.flush()
print("Produced!")
except Exception:
print("Failed to send message:", e)
producer.close()
abhishekgahlot2 commented
I am wondering if it could be due to sasl connection. my iam connection to msk is fine but sasl connection seems to flaky possibly due to acl.
Paultagoras commented
SASL shouldn't affect your connection to Clickhouse - that's not the protocol it uses. One option to check - could you set exactlyOnce=false for now?
Also, could you reconfirm the hostname and port for the database?
abhishekgahlot2 commented
Yup, i am gonna retry with exactlyOnce=false. however, i was able to do push data from the confluent cloud from the same settings to click house cloud without much issue. but msk kept on giving timeout
abhishekgahlot2 commented
I am closing the issue now. looks like its the internet connectivity issue from aws side.
abhishekgahlot2 commented
Reopening this again because it comes up at least once while deploying couple of times.
[Worker-04c93c16f7d15e23e] Caused by: com.clickhouse.client.ClickHouseException: Connect to https://a88q4ot1bn.us-west-2.aws.clickhouse.cloud:8443 [a88q4ot1bn.us-west-2.aws.clickhouse.cloud/35.82.252.60, a88q4ot1bn.us-west-2.aws.clickhouse.cloud/35.85.205.122, a88q4ot1bn.us-west-2.aws.clickhouse.cloud/44.226.232.172] failed: connect timed out, server ClickHouseNode [uri=https://a88q4ot1bn.us-west-2.aws.clickhouse.cloud:8443/default, options={sslmode=STRICT}]@-1254851478
[Worker-04c93c16f7d15e23e] at com.clickhouse.client.ClickHouseException.of(ClickHouseException.java:164)
[Worker-04c93c16f7d15e23e] at com.clickhouse.client.AbstractClient.lambda$execute$0(AbstractClient.java:275)
[Worker-04c93c16f7d15e23e] at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1700)
[Worker-04c93c16f7d15e23e] ... 3 more
[Worker-04c93c16f7d15e23e] Caused by: org.apache.hc.client5.http.ConnectTimeoutException: Connect to https://a88q4ot1bn.us-west-2.aws.clickhouse.cloud:8443 [a88q4ot1bn.us-west-2.aws.clickhouse.cloud/35.82.252.60, a88q4ot1bn.us-west-2.aws.clickhouse.cloud/35.85.205.122, a88q4ot1bn.us-west-2.aws.clickhouse.cloud/44.226.232.172] failed: connect timed out
[Worker-04c93c16f7d15e23e] at java.base/java.net.PlainSocketImpl.socketConnect(Native Method)
[Worker-04c93c16f7d15e23e] at java.base/java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:412)
[Worker-04c93c16f7d15e23e] at java.base/java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:255)
[Worker-04c93c16f7d15e23e] at java.base/java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:237)
[Worker-04c93c16f7d15e23e] at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[Worker-04c93c16f7d15e23e] at java.base/java.net.Socket.connect(Socket.java:609)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory.lambda$connectSocket$0(SSLConnectionSocketFactory.java:281)
[Worker-04c93c16f7d15e23e] at java.base/java.security.AccessController.doPrivileged(Native Method)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:280)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:240)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.impl.io.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:184)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:447)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.impl.classic.InternalExecRuntime.connectEndpoint(InternalExecRuntime.java:162)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.impl.classic.InternalExecRuntime.connectEndpoint(InternalExecRuntime.java:172)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.impl.classic.ConnectExec.execute(ConnectExec.java:142)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.impl.classic.ProtocolExec.execute(ProtocolExec.java:192)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.impl.classic.HttpRequestRetryExec.execute(HttpRequestRetryExec.java:113)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.impl.classic.RedirectExec.execute(RedirectExec.java:116)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.impl.classic.ExecChainElement.execute(ExecChainElement.java:51)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.impl.classic.InternalHttpClient.doExecute(InternalHttpClient.java:170)
[Worker-04c93c16f7d15e23e] at org.apache.hc.client5.http.impl.classic.CloseableHttpClient.execute(CloseableHttpClient.java:123)
[Worker-04c93c16f7d15e23e] at com.clickhouse.client.http.ApacheHttpConnectionImpl.post(ApacheHttpConnectionImpl.java:241)
[Worker-04c93c16f7d15e23e] at com.clickhouse.client.http.ClickHouseHttpClient.send(ClickHouseHttpClient.java:118)
[Worker-04c93c16f7d15e23e] at com.clickhouse.client.AbstractClient.sendAsync(AbstractClient.java:161)
[Worker-04c93c16f7d15e23e] at com.clickhouse.client.AbstractClient.lambda$execute$0(AbstractClient.java:273)