From 7ffa366c9689bf157496c9cf804cd3f33e699886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Varga?= Date: Thu, 7 Aug 2025 11:52:02 +0200 Subject: [PATCH 1/4] Add retry mechanism to splitbrain script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Balázs Varga --- charts/redis-ha/templates/_configs.tpl | 21 ++++++++++++++++++--- charts/redis-ha/values.yaml | 1 + 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/charts/redis-ha/templates/_configs.tpl b/charts/redis-ha/templates/_configs.tpl index e1222ad2..0db1ce33 100644 --- a/charts/redis-ha/templates/_configs.tpl +++ b/charts/redis-ha/templates/_configs.tpl @@ -475,12 +475,28 @@ if [ "$MASTER" = "$ANNOUNCE_IP" ]; then redis_role if [ "$ROLE" != "master" ]; then - reinit + echo "waiting for redis to become master" + sleep {{ .Values.splitBrainDetection.retryInterval }} + identify_master + redis_role + echo "Redis role is $ROLE, expected role is master. No need to reinitialize." + if [ "$ROLE" != "master" ]; then + echo "Redis role is $ROLE, expected role is master, reinitializing" + reinit + fi fi elif [ "${MASTER}" ]; then identify_redis_master if [ "$REDIS_MASTER" != "$MASTER" ]; then - reinit + echo "Redis master and local master are not the same. waiting." + sleep {{ .Values.splitBrainDetection.retryInterval }} + identify_master + identify_redis_master + echo "Redis master is ${MASTER}, expected master is ${REDIS_MASTER}. No need to reinitialize." + if [ "${REDIS_MASTER}" != "${MASTER}" ]; then + echo "Redis master is ${MASTER}, expected master is ${REDIS_MASTER}, reinitializing" + reinit + fi fi fi done @@ -727,4 +743,3 @@ fi echo "response=$response" {{- end }} - diff --git a/charts/redis-ha/values.yaml b/charts/redis-ha/values.yaml index e857f158..9fd5a97a 100644 --- a/charts/redis-ha/values.yaml +++ b/charts/redis-ha/values.yaml @@ -1006,5 +1006,6 @@ networkPolicy: splitBrainDetection: # -- Interval between redis sentinel and server split brain checks (in seconds) interval: 60 + retryInterval: 10 # -- splitBrainDetection resources resources: {} From 3cf0c08369efb977efd3db20dcc46e1181b5e0bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Varga?= Date: Thu, 7 Aug 2025 11:52:10 +0200 Subject: [PATCH 2/4] Update readme MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Balázs Varga --- charts/redis-ha/README.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/charts/redis-ha/README.md b/charts/redis-ha/README.md index e1562b08..2a32e666 100644 --- a/charts/redis-ha/README.md +++ b/charts/redis-ha/README.md @@ -460,14 +460,11 @@ Should your Pod require additional egress rules, define them in a `egressRules` Under not entirely known yet circumstances redis sentinel and its corresponding redis server reach a condition that this chart authors call "split brain" (for short). The observed behaviour is the following: the sentinel switches to the new re-elected master, but does not switch its redis server. Majority of original discussion on the problem has happened at the . The proposed solution is currently implemented as a sidecar container that runs a bash script with the following logic: - -1. Every `splitBrainDetection.interval` seconds a master (as known by sentinel) is determined -1. If it is the current node: ensure the redis server's role is master as well. -1. If it is not the current node: ensure the redis server also replicates from the same node. - -If any of the checks above fails - the redis server reinitialisation happens (it regenerates configs the same way it's done during the pod init), and then the redis server is instructed to shutdown. Then kubernetes restarts the container immediately. - -# Change Log +1. At intervals defined by `splitBrainDetection.interval`, the sidecar checks which node is recognized as master by Sentinel. +2. If the current pod is the master according to Sentinel, it verifies that the local Redis server is also running as master. +3. If the current pod is not the master, it ensures the local Redis server is replicating from the correct master node. +4. If any of these checks fail, the sidecar will retry the check at intervals defined by `splitBrainDetection.retryInterval`. +5. If the checks continue to fail after the retry attempts, the sidecar triggers a reinitialization: it regenerates the Redis configuration and instructs the Redis server to shut down. Kubernetes will then automatically restart the container. ## 4.14.9 - ** POTENTIAL BREAKING CHANGE. ** Introduced the ability to change the Haproxy Deployment container pod From 9d5c2fe4a5c7de972fb27358a27159a21c035116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Varga?= Date: Thu, 7 Aug 2025 11:53:12 +0200 Subject: [PATCH 3/4] Bump chart version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Balázs Varga --- charts/redis-ha/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/redis-ha/Chart.yaml b/charts/redis-ha/Chart.yaml index 2cced456..4bac142e 100644 --- a/charts/redis-ha/Chart.yaml +++ b/charts/redis-ha/Chart.yaml @@ -5,7 +5,7 @@ keywords: - redis - keyvalue - database -version: 4.33.8 +version: 4.33.9 appVersion: 7.2.7 description: This Helm chart provides a highly available Redis implementation with a master/slave configuration and uses Sentinel sidecars for failover management icon: https://upload.wikimedia.org/wikipedia/en/thumb/6/6b/Redis_Logo.svg/1200px-Redis_Logo.svg.png From 37abd12879d4067b2ef8d370b2a74ce104a4c166 Mon Sep 17 00:00:00 2001 From: Aaron Layfield Date: Sat, 13 Sep 2025 08:05:37 +0900 Subject: [PATCH 4/4] Incrementing Chart.yaml Signed-off-by: Aaron Layfield --- charts/redis-ha/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/redis-ha/Chart.yaml b/charts/redis-ha/Chart.yaml index 40cbbba5..aef9b856 100644 --- a/charts/redis-ha/Chart.yaml +++ b/charts/redis-ha/Chart.yaml @@ -5,7 +5,7 @@ keywords: - redis - keyvalue - database -version: 4.34.5 +version: 4.34.6 appVersion: 8.2.1 description: This Helm chart provides a highly available Redis implementation with a master/slave configuration and uses Sentinel sidecars for failover management icon: https://img.icons8.com/external-tal-revivo-shadow-tal-revivo/24/external-redis-an-in-memory-data-structure-project-implementing-a-distributed-logo-shadow-tal-revivo.png