diff --git a/charts/redis-ha/Chart.yaml b/charts/redis-ha/Chart.yaml index 40cbbba..aef9b85 100644 --- a/charts/redis-ha/Chart.yaml +++ b/charts/redis-ha/Chart.yaml @@ -5,7 +5,7 @@ keywords: - redis - keyvalue - database -version: 4.34.5 +version: 4.34.6 appVersion: 8.2.1 description: This Helm chart provides a highly available Redis implementation with a master/slave configuration and uses Sentinel sidecars for failover management icon: https://img.icons8.com/external-tal-revivo-shadow-tal-revivo/24/external-redis-an-in-memory-data-structure-project-implementing-a-distributed-logo-shadow-tal-revivo.png diff --git a/charts/redis-ha/README.md b/charts/redis-ha/README.md index d3cf799..daf2e6f 100644 --- a/charts/redis-ha/README.md +++ b/charts/redis-ha/README.md @@ -464,14 +464,11 @@ Should your Pod require additional egress rules, define them in a `egressRules` Under not entirely known yet circumstances redis sentinel and its corresponding redis server reach a condition that this chart authors call "split brain" (for short). The observed behaviour is the following: the sentinel switches to the new re-elected master, but does not switch its redis server. Majority of original discussion on the problem has happened at the . The proposed solution is currently implemented as a sidecar container that runs a bash script with the following logic: - -1. Every `splitBrainDetection.interval` seconds a master (as known by sentinel) is determined -1. If it is the current node: ensure the redis server's role is master as well. -1. If it is not the current node: ensure the redis server also replicates from the same node. - -If any of the checks above fails - the redis server reinitialisation happens (it regenerates configs the same way it's done during the pod init), and then the redis server is instructed to shutdown. Then kubernetes restarts the container immediately. - -# Change Log +1. At intervals defined by `splitBrainDetection.interval`, the sidecar checks which node is recognized as master by Sentinel. +2. If the current pod is the master according to Sentinel, it verifies that the local Redis server is also running as master. +3. If the current pod is not the master, it ensures the local Redis server is replicating from the correct master node. +4. If any of these checks fail, the sidecar will retry the check at intervals defined by `splitBrainDetection.retryInterval`. +5. If the checks continue to fail after the retry attempts, the sidecar triggers a reinitialization: it regenerates the Redis configuration and instructs the Redis server to shut down. Kubernetes will then automatically restart the container. ## 4.14.9 - ** POTENTIAL BREAKING CHANGE. ** Introduced the ability to change the Haproxy Deployment container pod diff --git a/charts/redis-ha/templates/_configs.tpl b/charts/redis-ha/templates/_configs.tpl index e1222ad..0db1ce3 100644 --- a/charts/redis-ha/templates/_configs.tpl +++ b/charts/redis-ha/templates/_configs.tpl @@ -475,12 +475,28 @@ if [ "$MASTER" = "$ANNOUNCE_IP" ]; then redis_role if [ "$ROLE" != "master" ]; then - reinit + echo "waiting for redis to become master" + sleep {{ .Values.splitBrainDetection.retryInterval }} + identify_master + redis_role + echo "Redis role is $ROLE, expected role is master. No need to reinitialize." + if [ "$ROLE" != "master" ]; then + echo "Redis role is $ROLE, expected role is master, reinitializing" + reinit + fi fi elif [ "${MASTER}" ]; then identify_redis_master if [ "$REDIS_MASTER" != "$MASTER" ]; then - reinit + echo "Redis master and local master are not the same. waiting." + sleep {{ .Values.splitBrainDetection.retryInterval }} + identify_master + identify_redis_master + echo "Redis master is ${MASTER}, expected master is ${REDIS_MASTER}. No need to reinitialize." + if [ "${REDIS_MASTER}" != "${MASTER}" ]; then + echo "Redis master is ${MASTER}, expected master is ${REDIS_MASTER}, reinitializing" + reinit + fi fi fi done @@ -727,4 +743,3 @@ fi echo "response=$response" {{- end }} - diff --git a/charts/redis-ha/values.yaml b/charts/redis-ha/values.yaml index 1dc8b23..e914eeb 100644 --- a/charts/redis-ha/values.yaml +++ b/charts/redis-ha/values.yaml @@ -1014,5 +1014,6 @@ networkPolicy: splitBrainDetection: # -- Interval between redis sentinel and server split brain checks (in seconds) interval: 60 + retryInterval: 10 # -- splitBrainDetection resources resources: {}