Local rack host offline and could not find any remote hosts for fallback connection at
Opened this issue · 0 comments
abhinav0101 commented
==> localhost.2021-03-12.log <==
12-03-2021 23:08:28.393 SEVERE [http-nio-8080-exec-15] org.apache.catalina.core.StandardWrapperValve.invoke Servlet.service() for servlet [Jersey REST Service] in context with path [/sambhaashan] threw exception [org.glassfish.jersey.server.ContainerException: com.phonepe.ivr.exceptions.VxmlSerlvletException: com.netflix.dyno.connectionpool.exception.NoAvailableHostsException: NoAvailableHostsException: [host=Host [hostname=UNKNOWN, ipAddress=UNKNOWN, port=0, rack: UNKNOWN, datacenter: UNKNOW, status: Down, hashtag=null, password=null], latency=0(0), attempts=0]Local rack host offline and could not find any remote hosts for fallback connection] with root cause
com.netflix.dyno.connectionpool.exception.NoAvailableHostsException: NoAvailableHostsException: [host=Host [hostname=UNKNOWN, ipAddress=UNKNOWN, port=0, rack: UNKNOWN, datacenter: UNKNOW, status: Down, hashtag=null, password=null], latency=0(0), attempts=0]Local rack host offline and could not find any remote hosts for fallback connection
at com.netflix.dyno.connectionpool.impl.lb.HostSelectionWithFallback.getFallbackHostPool(HostSelectionWithFallback.java:252)
at com.netflix.dyno.connectionpool.impl.lb.HostSelectionWithFallback.getConnection(HostSelectionWithFallback.java:133)
at com.netflix.dyno.connectionpool.impl.lb.HostSelectionWithFallback.getConnectionUsingRetryPolicy(HostSelectionWithFallback.java:120)
at com.netflix.dyno.connectionpool.impl.ConnectionPoolImpl.executeWithFailover(ConnectionPoolImpl.java:305)
at com.netflix.dyno.jedis.DynoJedisClient.d_hset(DynoJedisClient.java:942)
at com.netflix.dyno.jedis.DynoJedisClient.hset(DynoJedisClient.java:932)
at com.phonepe.ivr.util.CallContextUtil.persist(CallContextUtil.java:156)
at com.phonepe.ivr.util.CallContextUtil.updateHashSet(CallContextUtil.java:143)
at com.phonepe.ivr.util.CallContextUtil.updateCallContextCustomValue(CallContextUtil.java:90)
at com.phonepe.ivr.services.VxmlProcessingService.ingest(VxmlProcessingService.java:180)
at com.phonepe.ivr.services.VxmlProcessingService.storeAvayaData(VxmlProcessingService.java:96)
at com.phonepe.ivr.servlets.VxmlServlet.fetchAvayaData(VxmlServlet.java:156)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory.lambda$static$0(ResourceMethodInvocationHandlerFactory.java:52)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:124)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:167)
at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$TypeOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:219)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:79)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:469)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:391)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:80)
at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:253)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244)
at org.glassfish.jersey.internal.Errors.process(Errors.java:292)
at org.glassfish.jersey.internal.Errors.process(Errors.java:274)
at org.glassfish.jersey.internal.Errors.process(Errors.java:244)
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:265)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:232)
4 hosts currently running in same machine with the following dynomite.yml files
dyn_o_mite:
datacenter: dc
rack: rack1
dyn_listen: 127.0.0.1:8101
dyn_seeds:
- 127.0.0.1:8103:rack2:dc:0
- 127.0.0.1:8105:rack3:dc:0
- 127.0.0.1:8107:rack4:dc:0
listen: 127.0.0.1:8102
servers:
- 127.0.0.1:6382:1
tokens: '0'
secure_server_option: datacenter
pem_key_file: conf/dynomite.pem
data_store: 0
read_consistency : DC_QUORUM
write_consistency : DC_SAFE_QUORUM
stats_listen: 0.0.0.0:22222
` datacenter: dc
rack: rack2
dyn_listen: 127.0.0.1:8103
dyn_seeds:
- 127.0.0.1:8101:rack1:dc:0
- 127.0.0.1:8105:rack3:dc:0
- 127.0.0.1:8107:rack4:dc:0
listen: 127.0.0.1:8104
servers: - 127.0.0.1:6383:1
tokens: '0'
secure_server_option: datacenter
pem_key_file: conf/dynomite.pem
data_store: 0
read_consistency : DC_QUORUM
write_consistency : DC_SAFE_QUORUM
stats_listen: 0.0.0.0:22226
`
`dyn_o_mite:
datacenter: dc
rack: rack3
dyn_listen: 127.0.0.1:8105
dyn_seeds:
- 127.0.0.1:8103:rack2:dc:0
- 127.0.0.1:8101:rack1:dc:0
- 127.0.0.1:8107:rack4:dc:0
listen: 127.0.0.1:8106
servers: - 127.0.0.1:6384:1
tokens: '0'
secure_server_option: datacenter
pem_key_file: conf/dynomite.pem
data_store: 0
read_consistency : DC_QUORUM
write_consistency : DC_SAFE_QUORUM
stats_listen: 0.0.0.0:22231
`
`dyn_o_mite:
datacenter: dc
rack: rack4
dyn_listen: 127.0.0.1:8107
dyn_seeds:
- 127.0.0.1:8103:rack2:dc:0
- 127.0.0.1:8105:rack3:dc:0
- 127.0.0.1:8101:rack1:dc:0
listen: 127.0.0.1:8108
servers: - 127.0.0.1:6385:1
tokens: '0'
secure_server_option: datacenter
pem_key_file: conf/dynomite.pem
data_store: 0
read_consistency : DC_QUORUM
write_consistency : DC_SAFE_QUORUM
stats_listen: 0.0.0.0:22232
`
HERE is my dyno setup:-
Host host1 = new HostBuilder().setHostname("box1")
.setIpAddress("127.0.0.1")
.setPort(8102)
.setRack("rack1")
.setDatacenter("dc").setStatus(Host.Status.Up).createHost();
Host host2 = new HostBuilder().setHostname("box2")
.setIpAddress("127.0.0.1")
.setPort(8104)
.setRack("rack2")
.setDatacenter("dc").setStatus(Host.Status.Up).createHost();
Host host3 = new HostBuilder().setHostname("box3")
.setIpAddress("127.0.0.1")
.setPort(8106)
.setRack("rack3")
.setDatacenter("dc").setStatus(Host.Status.Up).createHost();
Host host4 = new HostBuilder().setHostname("box4")
.setIpAddress("127.0.0.1")
.setPort(8108)
.setRack("rack4")
.setDatacenter("dc")
.setStatus(Host.Status.Up).createHost();
public void setUpDyno() {
try {
final HostSupplier customHostSupplier = new HostSupplier() {
@SneakyThrows
@Override
public List<Host> getHosts() {
return getUpdatefromfile();
}
};
this.hostSuperSupplier = customHostSupplier;
this.dynoClient = new DynoJedisClient.Builder()
.withApplicationName("BEATLE_ON_MY_KNEE")
.withDynomiteClusterName("DYNOMITE_CLUSTER_NAME")
.withHostSupplier(customHostSupplier)
.withTokenMapSupplier(testTokenMapSupplier)
.withCPConfig(getDynoConfig())
.build();
} catch (Exception e ) {
log.error("Error while initialising dyno client", e);
}
}
private List<Host> getUpdatefromfile() throws IOException {
String host1Status = new String(Files.readAllBytes(Paths.get("/abhinav_redis/health_updates/health_of_box1.txt")));
String host2Status = new String(Files.readAllBytes(Paths.get("/abhinav_redis/health_updates/health_of_box2.txt")));
String host3Status = new String(Files.readAllBytes(Paths.get("/abhinav_redis/health_updates/health_of_box3.txt")));
String host4Status = new String(Files.readAllBytes(Paths.get("/abhinav_redis/health_updates/health_of_box4.txt")));
log.info("I read this status for box1 {}", host1Status);
host1.setStatus((host1Status.equals("1")) ? Host.Status.Up : Host.Status.Down);
log.info("I read this status for box2 {}", host2Status);
host2.setStatus((host2Status.equals("1")) ? Host.Status.Up : Host.Status.Down);
log.info("I read this status for box3 {}", host3Status);
host3.setStatus((host3Status.equals("1")) ? Host.Status.Up : Host.Status.Down);
log.info("I read this status for box4 {}", host4Status);
host4.setStatus((host4Status.equals("1")) ? Host.Status.Up : Host.Status.Down);
List<Host> hosts = new ArrayList<>();
hosts.add(host1);
hosts.add(host2);
hosts.add(host3);
hosts.add(host4);
return hosts;
}
final String json = "[\n" +
"{\n" +
" \"token\": \"0\",\n" +
" \"hostname\": \"box1\",\n" +
" \"zone\": \"rack1\",\n" +
" \"port\": \"8102\",\n" +
" \"dc\": \"dc\",\n" +
" \"ip\": \"127.0.0.1\"\n" +
"},\n" +
"{\n" +
" \"token\": \"0\",\n" +
" \"hostname\": \"box2\",\n" +
" \"zone\": \"rack2\",\n" +
" \"port\": \"8104\",\n" +
" \"dc\": \"dc\",\n" +
" \"ip\": \"127.0.0.1\"\n" +
"},\n" +
"{\n" +
" \"token\": \"0\",\n" +
" \"hostname\": \"box3\",\n" +
" \"zone\": \"rack3\",\n" +
" \"port\": \"8106\",\n" +
" \"dc\": \"dc\",\n" +
" \"ip\": \"127.0.0.1\"\n" +
"},\n" +
"{\n" +
" \"token\": \"0\",\n" +
" \"hostname\": \"box4\",\n" +
" \"zone\": \"rack4\",\n" +
" \"port\": \"8108\",\n" +
" \"dc\": \"dc\",\n" +
" \"ip\": \"127.0.0.1\"\n" +
"}]" ;
private TokenMapSupplier testTokenMapSupplier = new AbstractTokenMapSupplier() {
@Override
public String getTopologyJsonPayload(Set<Host> set) {
return json;
}
@Override
public String getTopologyJsonPayload(String hostname) {
return json;
}
};
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
DynoJedisClient dynoClient = (DynoJedisClient) servletContext.getAttribute(DYNO_CLIENT);
if (dynoClient != null) dynoClient.stopClient();
servletContext.removeAttribute(DYNO_CLIENT);
}
private ConnectionPoolConfigurationImpl getDynoConfig() {
ConnectionPoolConfigurationImpl config = new ConnectionPoolConfigurationImpl("config")
.setMaxConnsPerHost(5)
.setMaxFailoverCount(3)
.withHostSupplier(hostSuperSupplier)
.setLoadBalancingStrategy(ConnectionPoolConfiguration.LoadBalancingStrategy.TokenAware)
.setMaxTimeoutWhenExhausted((int) Duration.ofSeconds(10).toMillis())
.setSocketTimeout(12000);
return config;
}