Some help
eoli3n opened this issue · 3 comments
eoli3n commented
Hi,
I'm trying to setup ES 7.4.1 cluster on swarm.
1° I have a question concerning the way you do.
As you use endpoint_mode, and published port in mode: host, you disabled mesh routing.
But you did not constraint coordination containers placement on a node.
As you don't know on which node coordination containers runs, and mesh routing is disabled, how to configure elasticsearch clients ?
2° Is there any way to configure es cluster using mesh routing ?
3° I have some DNS issues
Here's my docker-stack.yml
version: '3.7'
services:
es_coordination:
image: docker.elastic.co/elasticsearch/elasticsearch:7.4.1
healthcheck:
test: curl -fs http://localhost:9200/_cat/health || exit 1
interval: 30s
timeout: 5s
retries: 3
start_period: 45s
ports:
- target: 9200
published: 9200
protocol: tcp
mode: host
configs:
- source: elasticsearch_coordination_config
target: /usr/share/elasticsearch/config/elasticsearch.yml
environment:
node.name: es-coordination
ES_JAVA_OPTS: "-Xmx2G -Xms2G"
ELASTIC_PASSWORD: changeme
discovery.seed_hosts: es-master1, es-master2, es-master3
cluster.initial_master_nodes: es-master1, es-master2, es-master3
networks:
- elk
deploy:
endpoint_mode: dnsrr
mode: replicated
replicas: 1
resources:
limits:
memory: 2G
es-master1:
image: docker.elastic.co/elasticsearch/elasticsearch:7.4.1
healthcheck:
test: curl -fs http://localhost:9200/_cat/health || exit 1
interval: 30s
timeout: 5s
retries: 3
start_period: 45s
configs:
- source: elasticsearch_config
target: /usr/share/elasticsearch/config/elasticsearch.yml
environment:
node.name: es-master1
cluster.initial_master_nodes: es-master1, es-master2, es-master3
ES_JAVA_OPTS: "-Xmx2G -Xms2G"
networks:
- elk
deploy:
placement:
constraints: [ node.hostname == tspeda-swarm-manager1 ]
endpoint_mode: dnsrr
mode: replicated
replicas: 1
resources:
limits:
memory: 2G
volumes:
- type: bind
source: /data/elasticsearch
target: /usr/share/elasticsearch/data
es-master2:
image: docker.elastic.co/elasticsearch/elasticsearch:7.4.1
healthcheck:
test: curl -fs http://localhost:9200/_cat/health || exit 1
interval: 30s
timeout: 5s
retries: 3
start_period: 45s
configs:
- source: elasticsearch_config
target: /usr/share/elasticsearch/config/elasticsearch.yml
environment:
node.name: es-master2
cluster.initial_master_nodes: es-master1, es-master2, es-master3
ES_JAVA_OPTS: "-Xmx2G -Xms2G"
networks:
- elk
deploy:
placement:
constraints: [ node.hostname == tspeda-swarm-worker1 ]
endpoint_mode: dnsrr
mode: replicated
replicas: 1
resources:
limits:
memory: 2G
volumes:
- type: bind
source: /data/elasticsearch
target: /usr/share/elasticsearch/data
es-master3:
image: docker.elastic.co/elasticsearch/elasticsearch:7.4.1
healthcheck:
test: curl -fs http://localhost:9200/_cat/health || exit 1
interval: 30s
timeout: 5s
retries: 3
start_period: 45s
configs:
- source: elasticsearch_config
target: /usr/share/elasticsearch/config/elasticsearch.yml
environment:
node.name: es-master3
cluster.initial_master_nodes: es-master1, es-master2, es-master3
ES_JAVA_OPTS: "-Xmx2G -Xms2G"
networks:
- elk
deploy:
placement:
constraints: [ node.hostname == tspeda-swarm-worker2 ]
endpoint_mode: dnsrr
mode: replicated
replicas: 1
resources:
limits:
memory: 2G
volumes:
- type: bind
source: /data/elasticsearch
target: /usr/share/elasticsearch/data
logstash:
image: docker.elastic.co/logstash/logstash:7.4.1
ports:
- "5000:5000"
- "9600:9600"
configs:
- source: logstash_config
target: /usr/share/logstash/config/logstash.yml
- source: logstash_pipeline
target: /usr/share/logstash/pipeline/logstash.conf
environment:
LS_JAVA_OPTS: "-Xmx1G -Xms1G"
networks:
- elk
deploy:
mode: replicated
replicas: 1
kibana:
image: docker.elastic.co/kibana/kibana:7.4.1
ports:
- "5601:5601"
configs:
- source: kibana_config
target: /usr/share/kibana/config/kibana.yml
networks:
- elk
deploy:
mode: replicated
replicas: 1
configs:
elasticsearch_coordination_config:
file: ./elasticsearch/config/elasticsearch_coordination.yml
elasticsearch_config:
file: ./elasticsearch/config/elasticsearch.yml
logstash_config:
file: ./logstash/config/logstash.yml
logstash_pipeline:
file: ./logstash/pipeline/logstash.conf
kibana_config:
file: ./kibana/config/kibana.yml
networks:
elk:
driver: overlay
elasticsearch.yml
---
## Default Elasticsearch configuration from Elasticsearch base image.
## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml
#
cluster.name: "docker-cluster"
network.host: 0.0.0.0
discovery.type: ''
## X-Pack settings
## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html
#
xpack.license.self_generated.type: basic
xpack.security.enabled: true
xpack.monitoring.collection.enabled: true
elasticsearch_coordination.yml
---
## Default Elasticsearch configuration from Elasticsearch base image.
## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml
#
cluster.name: "docker-cluster"
network.host: 0.0.0.0
discovery.type: ''
node.master: false
node.data: false
node.ingest: false
## X-Pack settings
## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html
#
xpack.license.self_generated.type: basic
xpack.security.enabled: true
xpack.monitoring.collection.enabled: true
When i run the stack, on es-coordination logs i get
elk_es_coordination.1.r7gs9q542tkt@tspeda-swarm-worker2 | {"type": "server", "timestamp": "2019-12-10T09:52:00,757Z", "level": "WARN", "component": "o.e.d.SeedHostsResolver", "cluster.name": "docker-cluster", "node.name": "es-coordination", "message": "failed to resolve host [es-master3]"
Any idea ?
eoli3n commented
solved all my issues.
Thx for your repo which helped a lot !
TamilTintin commented
HI @eoli3n ,
How you solved this issue , can you please explain
eoli3n commented
I don't remember well, but here's my docker-stack file:
version: '3.7'
services:
es-master1:
image: docker.elastic.co/elasticsearch/elasticsearch:7.5.0
configs:
- source: elasticsearch_config
target: /usr/share/elasticsearch/config/elasticsearch.yml
environment:
node.name: elk_elasticsearch.1
network.publish_host: _eth0_
discovery.seed_hosts: tasks.es-master2,tasks.es-master3
cluster.initial_master_nodes: elk_elasticsearch.1,elk_elasticsearch.2,elk_elasticsearch.3
ELASTIC_PASSWORD: changeme
ES_JAVA_OPTS: "-Xmx2G -Xms2G"
ports:
- '9200:9200'
networks:
- elk
deploy:
placement:
constraints: [ node.hostname == tspeda-swarm-manager1 ]
mode: replicated
replicas: 1
resources:
limits:
memory: 2G
volumes:
- type: bind
source: /data/elasticsearch
target: /usr/share/elasticsearch/data
es-master2:
image: docker.elastic.co/elasticsearch/elasticsearch:7.5.0
configs:
- source: elasticsearch_config
target: /usr/share/elasticsearch/config/elasticsearch.yml
environment:
node.name: elk_elasticsearch.2
network.publish_host: _eth0_
discovery.seed_hosts: tasks.es-master1,tasks.es-master3
cluster.initial_master_nodes: elk_elasticsearch.1,elk_elasticsearch.2,elk_elasticsearch.3
ES_JAVA_OPTS: "-Xmx2G -Xms2G"
networks:
- elk
deploy:
placement:
constraints: [ node.hostname == tspeda-swarm-manager2 ]
mode: replicated
replicas: 1
resources:
limits:
memory: 2G
volumes:
- type: bind
source: /data/elasticsearch
target: /usr/share/elasticsearch/data
es-master3:
image: docker.elastic.co/elasticsearch/elasticsearch:7.5.0
configs:
- source: elasticsearch_config
target: /usr/share/elasticsearch/config/elasticsearch.yml
environment:
node.name: elk_elasticsearch.3
network.publish_host: _eth0_
discovery.seed_hosts: tasks.es-master1,tasks.es-master2
cluster.initial_master_nodes: elk_elasticsearch.1,elk_elasticsearch.2,elk_elasticsearch.3
ES_JAVA_OPTS: "-Xmx2G -Xms2G"
networks:
- elk
deploy:
placement:
constraints: [ node.hostname == tspeda-swarm-manager3 ]
mode: replicated
replicas: 1
resources:
limits:
memory: 2G
volumes:
- type: bind
source: /data/elasticsearch
target: /usr/share/elasticsearch/data
logstash:
image: docker.elastic.co/logstash/logstash:7.5.0
ports:
- "5000:5000"
- "9600:9600"
configs:
- source: logstash_config
target: /usr/share/logstash/config/logstash.yml
environment:
LS_JAVA_OPTS: "-Xmx1G -Xms1G"
networks:
- elk
deploy:
mode: replicated
replicas: 1
volumes:
- type: bind
source: /mnt/pipeline/
target: /usr/share/logstash/pipeline/
kibana:
image: docker.elastic.co/kibana/kibana:7.5.0
ports:
- "5601:5601"
configs:
- source: kibana_config
target: /usr/share/kibana/config/kibana.yml
networks:
- elk
- traefik-public
deploy:
mode: replicated
replicas: 1
redis:
image: redis:5.0.7-alpine
command: redis-server /usr/local/etc/redis/redis.conf
ports:
- "6379:6379"
configs:
- source: redis_config
target: /usr/local/etc/redis/redis.conf
networks:
- elk
deploy:
mode: replicated
replicas: 1
configs:
elasticsearch_config:
file: ./elasticsearch/config/elasticsearch.yml
logstash_config:
file: ./logstash/config/logstash.yml
kibana_config:
file: ./kibana/config/kibana.yml
redis_config:
file: ./redis/config/redis.conf
networks:
elk:
driver: overlay
# DEBUG
# docker run --network=elk_elk busybox busybox hostname
#attachable: true
traefik-public:
external: true