base image (if version changed, make change to down-stream Dockerfile in line: FROM spark_base/<version_id>)
docker build -f docker/Dockerfile_spark_base -t spark_base:v1 .
docker build -f docker/Dockerfile_spark_master -t spark_master:v1 .
docker build -f docker/Dockerfile_spark_worker -t spark_worker:v1 .
docker build -f docker/Dockerfile_spark_submit -t spark_submit:v1 .
docker swarm init
docker network create -d overlay --attachable spark-cluster-network
Confirm the overlay network docker network ls
docker run -it --name spark-master --network spark-cluster-network -p 8080:8080 spark_master:v1
Confirm the master alive by visiting http://localhost:8080/
docker run -it --name spark-worker-1 --network spark-cluster-network -p 8081:8081 --env MEMORY=2G --env CORES=2 --env MASTER_CONTAINER_NAME=spark-master spark_worker:v1
docker run -it --name spark-worker-2 --network spark-cluster-network -p 8082:8081 --env MEMORY=2G --env CORES=2 --env MASTER_CONTAINER_NAME=spark-master spark_worker:v1
docker run -it --name spark-submit --network spark-cluster-network -p 4040:4040 --env MASTER_CONTAINER_NAME
=spark-master spark_submit:v1 bash