From d4d8397c6e93776fcbc58a03623cf2a8ab82922c Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 31 May 2024 16:12:37 +0200 Subject: [PATCH 01/13] Using RABBITMQ_ERLANG_COOKIE to resolve "Cookie file /var/lib/rabbitmq/.erlang.cookie must be accessible by owner only" --- .env | 3 ++- .erlang.cookie | 1 - cluster-entrypoint.sh | 3 --- docker-compose.yml | 6 +++--- 4 files changed, 5 insertions(+), 8 deletions(-) delete mode 100644 .erlang.cookie diff --git a/.env b/.env index a2c76ac..0dc86cb 100644 --- a/.env +++ b/.env @@ -1,3 +1,4 @@ RABBITMQ_DEFAULT_USER=guest RABBITMQ_DEFAULT_PASS=guest -RABBITMQ_DEFAULT_VHOST=/ \ No newline at end of file +RABBITMQ_DEFAULT_VHOST=/ +RABBITMQ_ERLANG_COOKIE=12345 diff --git a/.erlang.cookie b/.erlang.cookie deleted file mode 100644 index bd41cba..0000000 --- a/.erlang.cookie +++ /dev/null @@ -1 +0,0 @@ -12345 \ No newline at end of file diff --git a/cluster-entrypoint.sh b/cluster-entrypoint.sh index 8812021..aba2cb7 100755 --- a/cluster-entrypoint.sh +++ b/cluster-entrypoint.sh @@ -2,9 +2,6 @@ set -e -# Change .erlang.cookie permission -chmod 400 /var/lib/rabbitmq/.erlang.cookie - # Get hostname from enviromant variable HOSTNAME=`env hostname` echo "Starting RabbitMQ Server For host: " $HOSTNAME diff --git a/docker-compose.yml b/docker-compose.yml index a002a82..5c655ed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,8 +7,8 @@ services: - RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER} - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS} - RABBITMQ_DEFAULT_VHOST=${RABBITMQ_DEFAULT_VHOST} + - RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE} volumes: - - ./.erlang.cookie:/var/lib/rabbitmq/.erlang.cookie - ./cluster-entrypoint.sh:/usr/local/bin/cluster-entrypoint.sh entrypoint: /usr/local/bin/cluster-entrypoint.sh @@ -19,8 +19,8 @@ services: - rabbitmq1 environment: - JOIN_CLUSTER_HOST=rabbitmq1 + - RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE} volumes: - - ./.erlang.cookie:/var/lib/rabbitmq/.erlang.cookie - ./cluster-entrypoint.sh:/usr/local/bin/cluster-entrypoint.sh entrypoint: /usr/local/bin/cluster-entrypoint.sh @@ -31,8 +31,8 @@ services: - rabbitmq1 environment: - JOIN_CLUSTER_HOST=rabbitmq1 + - RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE} volumes: - - ./.erlang.cookie:/var/lib/rabbitmq/.erlang.cookie - ./cluster-entrypoint.sh:/usr/local/bin/cluster-entrypoint.sh entrypoint: /usr/local/bin/cluster-entrypoint.sh From 30cc71080df9c044118247469e156ac226c1cf7c Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 31 May 2024 16:13:28 +0200 Subject: [PATCH 02/13] Upgrade HAProxy to `haproxy:3.0` --- docker-compose.yml | 2 +- haproxy.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5c655ed..fe443ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: entrypoint: /usr/local/bin/cluster-entrypoint.sh haproxy: - image: haproxy:1.7 + image: haproxy:3.0 volumes: - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro depends_on: diff --git a/haproxy.cfg b/haproxy.cfg index 91b466f..4382731 100644 --- a/haproxy.cfg +++ b/haproxy.cfg @@ -1,5 +1,5 @@ global - log 127.0.0.1 local1 + log stdout format raw local0 maxconn 4096 defaults From 7409f4c537efcbe6f492567ed338d5ebf0af6dd9 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 31 May 2024 16:14:19 +0200 Subject: [PATCH 03/13] Configure HAProxy to check health every 200 milliseconds and very quickly add/remove (un)healthy nodes --- haproxy.cfg | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/haproxy.cfg b/haproxy.cfg index 4382731..b421397 100644 --- a/haproxy.cfg +++ b/haproxy.cfg @@ -28,9 +28,9 @@ listen rabbitmq timeout client 3h timeout server 3h option clitcpka - server rabbitmq1 rabbitmq1:5672 check inter 5s rise 2 fall 3 - server rabbitmq2 rabbitmq2:5672 check inter 5s rise 2 fall 3 - server rabbitmq3 rabbitmq3:5672 check inter 5s rise 2 fall 3 + server rabbitmq1 rabbitmq1:5672 check inter 200ms rise 1 fall 1 + server rabbitmq2 rabbitmq2:5672 check inter 200ms rise 1 fall 1 + server rabbitmq3 rabbitmq3:5672 check inter 200ms rise 1 fall 1 listen mgmt bind *:15672 @@ -39,6 +39,6 @@ listen mgmt timeout client 3h timeout server 3h option clitcpka - server rabbitmq1 rabbitmq1:15672 check inter 5s rise 2 fall 3 - server rabbitmq2 rabbitmq2:15672 check inter 5s rise 2 fall 3 - server rabbitmq3 rabbitmq3:15672 check inter 5s rise 2 fall 3 \ No newline at end of file + server rabbitmq1 rabbitmq1:15672 check inter 200ms rise 1 fall 1 + server rabbitmq2 rabbitmq2:15672 check inter 200ms rise 1 fall 1 + server rabbitmq3 rabbitmq3:15672 check inter 200ms rise 1 fall 1 From 2b7e7d16467e115ea7f1b41325579d3f665e1012 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 31 May 2024 16:15:20 +0200 Subject: [PATCH 04/13] Docker: Remove obsolete version attribute --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index fe443ad..03889a1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' services: rabbitmq1: image: rabbitmq:3-management From 36b18e8e0c7c9d33f7cf75548ab61b2ed7381d1b Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 31 May 2024 16:19:40 +0200 Subject: [PATCH 05/13] Docker: Set `restart` to `unless-stopped` --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 03889a1..7559444 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ services: rabbitmq1: image: rabbitmq:3-management + restart: unless-stopped hostname: rabbitmq1 environment: - RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER} @@ -13,6 +14,7 @@ services: rabbitmq2: image: rabbitmq:3-management + restart: unless-stopped hostname: rabbitmq2 depends_on: - rabbitmq1 @@ -25,6 +27,7 @@ services: rabbitmq3: image: rabbitmq:3-management + restart: unless-stopped hostname: rabbitmq3 depends_on: - rabbitmq1 @@ -37,6 +40,7 @@ services: haproxy: image: haproxy:3.0 + restart: unless-stopped volumes: - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro depends_on: From 0704a8df74b17f174b9daae153fdf4d17f751b74 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 31 May 2024 16:20:35 +0200 Subject: [PATCH 06/13] Ensure `.sh` use `LR` on Windows as CRLF causes "exec /usr/local/bin/cluster-entrypoint.sh: no such file or directory" --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..526c8a3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf \ No newline at end of file From 735ba29b734d6af2c74a899aab5dcc49b0c65e49 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 31 May 2024 16:22:08 +0200 Subject: [PATCH 07/13] Replace tail with sleep infinity --- cluster-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-entrypoint.sh b/cluster-entrypoint.sh index aba2cb7..fd646f4 100755 --- a/cluster-entrypoint.sh +++ b/cluster-entrypoint.sh @@ -20,4 +20,4 @@ else fi # Keep foreground process active ... -tail -f /dev/null +sleep infinity \ No newline at end of file From 4668bee2358aa834b78474e3559a9a876dfc4e23 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 31 May 2024 16:23:38 +0200 Subject: [PATCH 08/13] Apply `docker-entrypoint.sh` output change from https://github.com/serkodev/rabbitmq-cluster-docker/pull/3 --- cluster-entrypoint.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cluster-entrypoint.sh b/cluster-entrypoint.sh index fd646f4..67fa6a7 100755 --- a/cluster-entrypoint.sh +++ b/cluster-entrypoint.sh @@ -5,14 +5,12 @@ set -e # Get hostname from enviromant variable HOSTNAME=`env hostname` echo "Starting RabbitMQ Server For host: " $HOSTNAME +/usr/local/bin/docker-entrypoint.sh rabbitmq-server & +sleep 5 +rabbitmqctl wait /var/lib/rabbitmq/mnesia/rabbit\@$HOSTNAME.pid -if [ -z "$JOIN_CLUSTER_HOST" ]; then - /usr/local/bin/docker-entrypoint.sh rabbitmq-server & - sleep 5 - rabbitmqctl wait /var/lib/rabbitmq/mnesia/rabbit\@$HOSTNAME.pid -else - /usr/local/bin/docker-entrypoint.sh rabbitmq-server -detached - sleep 5 +if [ -n "$JOIN_CLUSTER_HOST" ]; then + echo "Starting RabbitMQ Server For host: " $HOSTNAME rabbitmqctl wait /var/lib/rabbitmq/mnesia/rabbit\@$HOSTNAME.pid rabbitmqctl stop_app rabbitmqctl join_cluster rabbit@$JOIN_CLUSTER_HOST From 928e0a4c783d8c46af5c8f0bbc38e39413600c19 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 31 May 2024 16:25:20 +0200 Subject: [PATCH 09/13] Docker: Use `rabbitmq` virtual network --- docker-compose.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 7559444..7db9f8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,8 @@ services: volumes: - ./cluster-entrypoint.sh:/usr/local/bin/cluster-entrypoint.sh entrypoint: /usr/local/bin/cluster-entrypoint.sh + networks: + - rabbitmq rabbitmq2: image: rabbitmq:3-management @@ -24,6 +26,8 @@ services: volumes: - ./cluster-entrypoint.sh:/usr/local/bin/cluster-entrypoint.sh entrypoint: /usr/local/bin/cluster-entrypoint.sh + networks: + - rabbitmq rabbitmq3: image: rabbitmq:3-management @@ -37,6 +41,8 @@ services: volumes: - ./cluster-entrypoint.sh:/usr/local/bin/cluster-entrypoint.sh entrypoint: /usr/local/bin/cluster-entrypoint.sh + networks: + - rabbitmq haproxy: image: haproxy:3.0 @@ -50,3 +56,8 @@ services: ports: - 15672:15672 - 5672:5672 + networks: + - rabbitmq + +networks: + rabbitmq: From 1c583d26c5ed6b75855fe61b9e7dd26b531eb266 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 31 May 2024 16:26:19 +0200 Subject: [PATCH 10/13] Docker: Use durable volumes for rabbitmq nodes --- docker-compose.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 7db9f8e..f1a48ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,8 @@ services: - RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE} volumes: - ./cluster-entrypoint.sh:/usr/local/bin/cluster-entrypoint.sh + - rabbitmq1-data:/var/lib/rabbitmq + - rabbitmq1-logs:/var/log/rabbitmq entrypoint: /usr/local/bin/cluster-entrypoint.sh networks: - rabbitmq @@ -25,6 +27,8 @@ services: - RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE} volumes: - ./cluster-entrypoint.sh:/usr/local/bin/cluster-entrypoint.sh + - rabbitmq2-data:/var/lib/rabbitmq + - rabbitmq2-logs:/var/log/rabbitmq entrypoint: /usr/local/bin/cluster-entrypoint.sh networks: - rabbitmq @@ -40,6 +44,8 @@ services: - RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE} volumes: - ./cluster-entrypoint.sh:/usr/local/bin/cluster-entrypoint.sh + - rabbitmq3-data:/var/lib/rabbitmq + - rabbitmq3-logs:/var/log/rabbitmq entrypoint: /usr/local/bin/cluster-entrypoint.sh networks: - rabbitmq @@ -61,3 +67,11 @@ services: networks: rabbitmq: + +volumes: + rabbitmq1-data: + rabbitmq1-logs: + rabbitmq2-data: + rabbitmq2-logs: + rabbitmq3-data: + rabbitmq3-logs: From 3f0958e424e608c3b0e865f25e80f064da0483dd Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 31 May 2024 16:28:22 +0200 Subject: [PATCH 11/13] Docker: Map HA Proxy port `1936` --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index f1a48ad..bdb51f8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,6 +62,7 @@ services: ports: - 15672:15672 - 5672:5672 + - 1936:1936 networks: - rabbitmq From 3ab0f6a202950b0d504222e3bf0464de8c5c6734 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 31 May 2024 16:41:34 +0200 Subject: [PATCH 12/13] Increase cluster to 5 nodes --- advanced.config | 5 +++++ docker-compose.yml | 43 +++++++++++++++++++++++++++++++++++++++++++ haproxy.cfg | 4 ++++ 3 files changed, 52 insertions(+) create mode 100644 advanced.config diff --git a/advanced.config b/advanced.config new file mode 100644 index 0000000..b354304 --- /dev/null +++ b/advanced.config @@ -0,0 +1,5 @@ +[ + %% five replicas by default, only makes sense for nine node clusters + {rabbit, [{quorum_cluster_size, 5}, + {quorum_commands_soft_limit, 512}]} +]. \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index bdb51f8..e279d41 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: - RABBITMQ_DEFAULT_VHOST=${RABBITMQ_DEFAULT_VHOST} - RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE} volumes: + - ./advanced.config:/etc/rabbitmq/advanced.config - ./cluster-entrypoint.sh:/usr/local/bin/cluster-entrypoint.sh - rabbitmq1-data:/var/lib/rabbitmq - rabbitmq1-logs:/var/log/rabbitmq @@ -26,6 +27,7 @@ services: - JOIN_CLUSTER_HOST=rabbitmq1 - RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE} volumes: + - ./advanced.config:/etc/rabbitmq/advanced.config - ./cluster-entrypoint.sh:/usr/local/bin/cluster-entrypoint.sh - rabbitmq2-data:/var/lib/rabbitmq - rabbitmq2-logs:/var/log/rabbitmq @@ -43,6 +45,7 @@ services: - JOIN_CLUSTER_HOST=rabbitmq1 - RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE} volumes: + - ./advanced.config:/etc/rabbitmq/advanced.config - ./cluster-entrypoint.sh:/usr/local/bin/cluster-entrypoint.sh - rabbitmq3-data:/var/lib/rabbitmq - rabbitmq3-logs:/var/log/rabbitmq @@ -50,6 +53,42 @@ services: networks: - rabbitmq + rabbitmq4: + image: rabbitmq:3-management + restart: unless-stopped + hostname: rabbitmq4 + depends_on: + - rabbitmq1 + environment: + - JOIN_CLUSTER_HOST=rabbitmq1 + - RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE} + volumes: + - ./advanced.config:/etc/rabbitmq/advanced.config + - ./cluster-entrypoint.sh:/usr/local/bin/cluster-entrypoint.sh + - rabbitmq4-data:/var/lib/rabbitmq + - rabbitmq4-logs:/var/log/rabbitmq + entrypoint: /usr/local/bin/cluster-entrypoint.sh + networks: + - rabbitmq + + rabbitmq5: + image: rabbitmq:3-management + restart: unless-stopped + hostname: rabbitmq5 + depends_on: + - rabbitmq1 + environment: + - JOIN_CLUSTER_HOST=rabbitmq1 + - RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE} + volumes: + - ./advanced.config:/etc/rabbitmq/advanced.config + - ./cluster-entrypoint.sh:/usr/local/bin/cluster-entrypoint.sh + - rabbitmq5-data:/var/lib/rabbitmq + - rabbitmq5-logs:/var/log/rabbitmq + entrypoint: /usr/local/bin/cluster-entrypoint.sh + networks: + - rabbitmq + haproxy: image: haproxy:3.0 restart: unless-stopped @@ -76,3 +115,7 @@ volumes: rabbitmq2-logs: rabbitmq3-data: rabbitmq3-logs: + rabbitmq4-data: + rabbitmq4-logs: + rabbitmq5-data: + rabbitmq5-logs: diff --git a/haproxy.cfg b/haproxy.cfg index b421397..0e9fbc7 100644 --- a/haproxy.cfg +++ b/haproxy.cfg @@ -31,6 +31,8 @@ listen rabbitmq server rabbitmq1 rabbitmq1:5672 check inter 200ms rise 1 fall 1 server rabbitmq2 rabbitmq2:5672 check inter 200ms rise 1 fall 1 server rabbitmq3 rabbitmq3:5672 check inter 200ms rise 1 fall 1 + server rabbitmq4 rabbitmq4:5672 check inter 200ms rise 1 fall 1 + server rabbitmq5 rabbitmq5:5672 check inter 200ms rise 1 fall 1 listen mgmt bind *:15672 @@ -42,3 +44,5 @@ listen mgmt server rabbitmq1 rabbitmq1:15672 check inter 200ms rise 1 fall 1 server rabbitmq2 rabbitmq2:15672 check inter 200ms rise 1 fall 1 server rabbitmq3 rabbitmq3:15672 check inter 200ms rise 1 fall 1 + server rabbitmq4 rabbitmq4:15672 check inter 200ms rise 1 fall 1 + server rabbitmq5 rabbitmq5:15672 check inter 200ms rise 1 fall 1 From 982dac7971d89ad00f57180aa47d4e28725b67eb Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 31 May 2024 17:39:17 +0200 Subject: [PATCH 13/13] Updated README to document erlang cooking envvar and cluster size increase --- README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9c8f186..3dec586 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,11 @@ Setup a RabbitMQ Cluster environment on your device using the pure [RabbitMQ](ht ## Quick start -``` +```con docker compose up ``` -Open http://localhost:15672 to login RabbitMQ dashboard. +Open to login RabbitMQ dashboard. > Username: `guest` > Password: `guest` @@ -24,13 +24,15 @@ Open http://localhost:15672 to login RabbitMQ dashboard. ### `docker-compose.yml` -Docker [compose](https://docs.docker.com/compose/compose-file/) config file, including 3 RabbitMQ service cluster and a HAProxy. +Docker [compose](https://docs.docker.com/compose/compose-file/) config file, including 5 RabbitMQ service cluster and a HAProxy. | Service | Description | | ----------- | ------------------------- | | `rabbitmq1` | RabbitMQ (cluster) | | `rabbitmq2` | RabbitMQ (cluster member) | | `rabbitmq3` | RabbitMQ (cluster member) | +| `rabbitmq4` | RabbitMQ (cluster member) | +| `rabbitmq5` | RabbitMQ (cluster member) | | `haproxy` | Load Balancer | #### Default expose ports @@ -39,6 +41,7 @@ Docker [compose](https://docs.docker.com/compose/compose-file/) config file, inc | ----------------- | --------------------------------------------------- | | `localhost:5672` | AMQP 0-9-1 and AMQP 1.0 clients | | `localhost:15672` | HTTP API clients, management UI and `rabbitmqadmin` | +| `localhost:1936` | HAProxy statistics | ### `.env` @@ -46,11 +49,11 @@ Docker [compose](https://docs.docker.com/compose/compose-file/) config file, inc | ------------------------ | ------- | | `RABBITMQ_DEFAULT_USER` | guest | | `RABBITMQ_DEFAULT_PASS` | guest | -| `RABBITMQ_DEFAULT_VHOST` | / | +| `RABBITMQ_ERLANG_COOKIE` | 12345 | ### `.erlang.cookie` -Put your custom [Erlang Cookie](https://www.rabbitmq.com/clustering.html#erlang-cookie) inside this file (default: `12345`) for the nodes in cluster communicate with each other. +Put your custom [Erlang Cookie](https://www.rabbitmq.com/clustering.html#erlang-cookie) value in the `RABBITMQ_ERLANG_COOKIE` environment variable file (`.env`). ### `haproxy.cfg`