Authentication exception (401) when connecting to elasticsearch: 'unable to authenticate user [elastic] for REST request [/]'
Opened this issue · 1 comments
PGSync version: 3.1.0
Postgres version:
Elasticsearch/OpenSearch version: elastic search 8.11.1
Redis version: 7.2.3
Python version: 3.11.7
Problem Description:
I'm having trouble with elastic search authentication though I'm setting the same password and SSL certificates, the three of them: client, CA and key certificates. If I turn off ssl and https as schema everything works fine, but I need to use secure connection. Here below is a reproduceable example. I can login to elasticsearch through kibana and curl just fine (I haven't tested yet with curl supplying the ca and client certificates).
Any feedback on this would be appreciated.
Docker YAML File:
version: '3.9'
services:
setup:
networks:
- elastic
image: elasticsearch:${STACK_VERSION}
volumes:
- certs:/usr/share/elasticsearch/config/certs
user: '0'
command: >
bash -c '
if [ x${ELASTIC_PASSWORD} == x ]; then
echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
exit 1;
elif [ x${KIBANA_PASSWORD} == x ]; then
echo "Set the KIBANA_PASSWORD environment variable in the .env file";
exit 1;
fi;
if [ ! -f config/certs/ca.zip ]; then
echo "Creating CA";
bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
unzip config/certs/ca.zip -d config/certs;
fi;
if [ ! -f config/certs/certs.zip ]; then
echo "Creating certs";
echo -ne \
"instances:\n"\
" - name: es01\n"\
" dns:\n"\
" - es01\n"\
" - localhost\n"\
" ip:\n"\
" - 127.0.0.1\n"\
> config/certs/instances.yml;
bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
unzip config/certs/certs.zip -d config/certs;
fi;
echo "Setting file permissions"
chown -R root:root config/certs;
find . -type d -exec chmod 750 \{\} \;;
find . -type f -exec chmod 640 \{\} \;;
echo "Waiting for Elasticsearch availability";
until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
echo "Setting kibana_system password";
until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
echo "All done!";
'
healthcheck:
test: ['CMD-SHELL', '[ -f config/certs/es01/es01.crt ]']
interval: 1s
timeout: 5s
retries: 120
es01:
depends_on:
setup:
condition: service_healthy
networks:
- elastic
- frontend
- data-center
image: elasticsearch:${STACK_VERSION}
volumes:
- certs:/usr/share/elasticsearch/config/certs
- esdata01:/usr/share/elasticsearch/data
ports:
- ${ES_PORT}:9200
environment:
- node.name=es01
- cluster.name=${CLUSTER_NAME}
- cluster.initial_master_nodes=es01
- ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
- bootstrap.memory_lock=true
- xpack.security.enabled=true
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.key=certs/es01/es01.key
- xpack.security.http.ssl.certificate=certs/es01/es01.crt
- xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.key=certs/es01/es01.key
- xpack.security.transport.ssl.certificate=certs/es01/es01.crt
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.license.self_generated.type=${LICENSE}
mem_limit: ${MEM_LIMIT}
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
test:
[
'CMD-SHELL',
"curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
]
interval: 10s
timeout: 10s
retries: 120
kibana:
depends_on:
es01:
condition: service_healthy
networks:
- elastic
image: kibana:${STACK_VERSION}
volumes:
- certs:/usr/share/kibana/config/certs
- kibanadata:/usr/share/kibana/data
ports:
- ${KIBANA_PORT}:5601
environment:
- SERVERNAME=kibana
- ELASTICSEARCH_HOSTS=https://es01:9200
- ELASTICSEARCH_USERNAME=kibana_system
- ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
- ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
mem_limit: ${MEM_LIMIT}
healthcheck:
test:
[
'CMD-SHELL',
"curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
]
interval: 10s
timeout: 10s
retries: 120
postgres:
ports:
- 5432:5432
image: postgres:16.1
volumes:
- "./init_db:/docker-entrypoint-initdb.d"
# - "../postgres/data:/var/lib/postgresql/data"
environment:
- "POSTGRES_USER=revista"
- "POSTGRES_PASSWORD=${ELASTIC_PASSWORD}"
restart: always
networks:
- data-center
command:
- "postgres"
- "-c"
- "wal_level=logical"
- "-c"
- "max_replication_slots=3"
redis:
image: redis:7.2.3
ports:
- 6379:6379
command: redis-server --requirepass ${ELASTIC_PASSWORD}
networks:
- data-center
pgsync:
build:
context: ../pgsync
dockerfile: ./Dockerfile-pgsync
restart: on-failure
labels:
org.label-schema.name: "pgsync"
org.label-schema.description: "Postgres to Elasticsearch sync"
com.label-schema.service-type: "daemon"
sysctls:
- net.ipv4.tcp_keepalive_time=200
- net.ipv4.tcp_keepalive_intvl=200
- net.ipv4.tcp_keepalive_probes=5
depends_on:
- postgres
- redis
environment:
- CHECKPOINT_PATH=/pgsync
- PG_USER=revista
- PG_HOST=postgres
- PG_PORT=5432
- PG_PASSWORD=${ELASTIC_PASSWORD}
- ELASTICSEARCH_SCHEME=https
- ELASTICSEARCH_HOST=es01
- ELASTICSEARCH_PORT=9200
- ELASTICSEARCH_USER=elastic
- ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
- ELASTICSEARCH_VERIFY_CERTS=True
- ELASTICSEARCH_USE_SSL=True
- REDIS_HOST=redis
- REDIS_PORT=6379
- REDIS_AUTH=${ELASTIC_PASSWORD}
- ELASTICSEARCH=True
- OPENSEARCH=False
- SCHEMA=/pgsync/schema.json
- LOG_LEVEL=INFO
- ELASTICSEARCH_CA_CERTS=/usr/share/elasticsearch/config/certs/ca/ca.crt
- ELASTICSEARCH_CLIENT_CERT=/usr/share/elasticsearch/config/certs/es01/es01.crt
- ELASTICSEARCH_CLIENT_KEY=/usr/share/elasticsearch/config/certs/es01/es01.key
command: ./runserver.sh && sleep inf
volumes:
# - ../pgsync/example-schema.json:/pgsync/schema.json
- ../pgsync/schema.json:/pgsync/schema.json
- certs:/usr/share/elasticsearch/config/certs
networks:
- data-center
volumes:
certs:
driver: local
esdata01:
driver: local
kibanadata:
driver: local
networks:
networks:
data-center:
frontend:
elastic:
Error Message (if any):
pgsync-1 | Traceback (most recent call last):
pgsync-1 | File "/usr/local/bin/pgsync", line 7, in <module>
pgsync-1 | 0:00:00.202205 (0.20 sec)
pgsync-1 | sync.main()
pgsync-1 | File "/usr/local/lib/python3.11/site-packages/click/core.py", line 1157, in __call__
pgsync-1 | return self.main(*args, **kwargs)
pgsync-1 | ^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1 | File "/usr/local/lib/python3.11/site-packages/click/core.py", line 1078, in main
pgsync-1 | rv = self.invoke(ctx)
pgsync-1 | ^^^^^^^^^^^^^^^^
pgsync-1 | File "/usr/local/lib/python3.11/site-packages/click/core.py", line 1434, in invoke
pgsync-1 | return ctx.invoke(self.callback, **ctx.params)
pgsync-1 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1 | File "/usr/local/lib/python3.11/site-packages/click/core.py", line 783, in invoke
pgsync-1 | return __callback(*args, **kwargs)
pgsync-1 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1 | File "/usr/local/lib/python3.11/site-packages/pgsync/sync.py", line 1480, in main
pgsync-1 | sync: Sync = Sync(
pgsync-1 | ^^^^^
pgsync-1 | File "/usr/local/lib/python3.11/site-packages/pgsync/singleton.py", line 36, in __call__
pgsync-1 | cls._instances[key] = super(Singleton, cls).__call__(
pgsync-1 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1 | File "/usr/local/lib/python3.11/site-packages/pgsync/sync.py", line 88, in __init__
pgsync-1 | self.search_client: SearchClient = SearchClient()
pgsync-1 | ^^^^^^^^^^^^^^
pgsync-1 | File "/usr/local/lib/python3.11/site-packages/pgsync/search_client.py", line 48, in __init__
pgsync-1 | self.__client.info()["version"]["number"].split(".")[0]
pgsync-1 | ^^^^^^^^^^^^^^^^^^^^
pgsync-1 | File "/usr/local/lib/python3.11/site-packages/elasticsearch/_sync/client/utils.py", line 402, in wrapped
pgsync-1 | return api(*args, **kwargs)
pgsync-1 | ^^^^^^^^^^^^^^^^^^^^
pgsync-1 | File "/usr/local/lib/python3.11/site-packages/elasticsearch/_sync/client/__init__.py", line 2278, in info
pgsync-1 | return self.perform_request( # type: ignore[return-value]
pgsync-1 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1 | File "/usr/local/lib/python3.11/site-packages/elasticsearch/_sync/client/_base.py", line 320, in perform_request
pgsync-1 | raise HTTP_EXCEPTIONS.get(meta.status, ApiError)(
pgsync-1 | elasticsearch.AuthenticationException: AuthenticationException(401, 'security_exception', 'unable to authenticate user [elastic] for REST request [/]')
Try setting the ElasticSearch password to not include special characters.