- Up and Down scaling of docker swarm services
- AWS ECR access to update distributed swarm images
When setting your alerts with Prometheus introduce an additional annotation for service into your rules file
service: "{{ $labels.container_label_com_docker_swarm_service_name }}"
Full example
groups:
- name: containers
rules:
- alert: container_eating_memory
expr: sum(container_memory_rss{container_label_com_docker_swarm_task_name=~".+"}) by (container_label_com_docker_swarm_service_name,instance,image) / 1000000 > 3000
for: 1m
labels:
severity: critical
annotations:
service: "{{ $labels.container_label_com_docker_swarm_service_name }}"
summary: "Container {{ $labels.container_label_com_docker_swarm_service_name }} has high memory consumption"
description: "{{ $labels.name }} with image {{ $labels.image }} of service {{ $labels.container_label_com_docker_swarm_service_name }} on {{ $labels.instance }} is eating up memory. Usage is {{ humanize $value}}"
In order to feed the scaler with alerts you need to introduce a webhook receiver
route:
receiver: 'receiver'
receivers:
- name: 'receiver'
webhook_configs:
- send_resolved: true
url: http://[SCALER URL]:8083
Alert json payload from Alertmanger will contain the name of your service
{
"annotations": {"service": "[YOUR DOCKER SWARM SERVICE]", "summary": "test", "description": "test"}
}
Full example of valid Alertmanger Webhook json payload
{
"version": "4",
"groupKey": "1",
"status": "firing",
"alerts": [
{
"labels": {"summary": "test", "description": "test"},
"annotations": {"service": "YOUR DOCKER SWARM SERVICE", "summary": "test", "description": "test"},
"startsAt": "2012-11-01T22:08:41+00:00",
"endsAt": "2012-11-01T22:08:41+00:00"
}
]
}
version: '3.4'
services:
scaler_dev:
image: petertilsen1/docker-tools_scaler:latest
environment:
- AWS_ACCESS_KEY_ID=[YOUR ID]
- AWS_SECRET_ACCESS_KEY=[YOUR ACCESS KEY]
ports:
- "8083:8083"
volumes:
- /var/run/docker.sock:/var/run/docker.sock
deploy:
mode: replicated
replicas: 1
restart_policy:
condition: any
logging:
driver: json-file
Scaler was created by Peter Tilsen