I installed a Redis Server + Sentinel cluster with 3 nodes.
Sentinel is making the failover when the master Redis goes down, but what I realised is that the live slaves are not following the new master.
Servers:
- redis1 (192.168.69.91): cluster node 1 (Redis Server + Sentinel)
- redis2 (192.168.69.92): cluster node 2 (Redis Server + Sentinel)
- redis3 (192.168.69.93): cluster node 3 (Redis Server + Sentinel)
Actually the master node is redis1, redis2 and redis3 are slaves of redis1:
redis1:6379> info replication
# Replication
role:master
connected_slaves:2
slave0:ip=192.168.69.93,port=6379,state=online,offset=332569377,lag=0
slave1:ip=192.168.69.92,port=6379,state=online,offset=332569230,lag=1
master_failover_state:no-failover
master_replid:6735b3ef1c29926ee9f6c0ace75bc169150c2fa6
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:332569377
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:331511956
repl_backlog_histlen:1057422
redis2:6379> info replication
# Replication
role:slave
master_host:192.168.69.91
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_read_repl_offset:332556357
slave_repl_offset:332556357
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:6735b3ef1c29926ee9f6c0ace75bc169150c2fa6
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:332556357
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:332555021
repl_backlog_histlen:1337
redis3:6379> info replication
# Replication
role:slave
master_host:192.168.69.91
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_read_repl_offset:332549259
slave_repl_offset:332549259
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:1
slave0:ip=192.168.69.92,port=6379,state=online,offset=332549098,lag=0
master_failover_state:no-failover
master_replid:6735b3ef1c29926ee9f6c0ace75bc169150c2fa6
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:332549259
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:331492545
repl_backlog_histlen:1056715
I get the correct master host from Sentinel:
root@redis1:~# redis-cli -p 26379 sentinel get-master-addr-by-name mycluster
1) "192.168.69.91"
2) "6379"
This is the Redis Sentinel configuration:
protected-mode no
port 26379
sentinel monitor mycluster 192.168.69.91 6379 2
sentinel down-after-milliseconds mycluster 5000
# Generated by CONFIG REWRITE
latency-tracking-info-percentiles 50 99 99.9
user default on nopass ~* &* +@all
sentinel myid c3d7456b7e06e85f449cfc452ac46a4bb7310a59
sentinel config-epoch mycluster 22
sentinel leader-epoch mycluster 22
sentinel current-epoch 22
sentinel known-replica mycluster 192.168.69.93 6379
sentinel known-replica mycluster 192.168.69.92 6379
sentinel known-sentinel mycluster 192.168.69.92 26379 584784ea586a5954576eaa39be12542d4fae3175
sentinel known-sentinel mycluster 192.168.69.93 26379 ebe43f2f88aba2d3015f05ba960fd7a609bb67e0
When I shutdown the master Redis Server host, the new master redis3 is designated after the automatic failover:
root@redis1:/etc/redis# redis-cli -p 26379 sentinel get-master-addr-by-name mycluster
1) "192.168.69.93"
2) "6379"
redis3:6379> info replication
# Replication
role:master
connected_slaves:0
master_failover_state:no-failover
master_replid:c97cca5168a874341c372b4e7e5eead1a0d30f1a
master_replid2:6735b3ef1c29926ee9f6c0ace75bc169150c2fa6
master_repl_offset:332740970
second_repl_offset:332733598
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:331676505
repl_backlog_histlen:1064466
but the problem is that the still alive redis2 slave is not following the new master but it’s trying to still follow the old master:
redis2:6379> info replication
# Replication
role:slave
master_host:192.168.69.91
master_port:6379
master_link_status:down
master_last_io_seconds_ago:-1
master_sync_in_progress:0
slave_read_repl_offset:332733597
slave_repl_offset:332733597
master_link_down_since_seconds:29
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:6735b3ef1c29926ee9f6c0ace75bc169150c2fa6
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:332733597
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:332555021
repl_backlog_histlen:178577
These are the Sentinel logs during the failover:
1591:X 16 Aug 2022 14:41:18.199 # +sdown master mycluster 192.168.69.91 6379
1591:X 16 Aug 2022 14:41:18.267 # +odown master mycluster 192.168.69.91 6379 #quorum 2/2
1591:X 16 Aug 2022 14:41:18.267 # +new-epoch 23
1591:X 16 Aug 2022 14:41:18.267 # +try-failover master mycluster 192.168.69.91 6379
1591:X 16 Aug 2022 14:41:18.273 * Sentinel new configuration saved on disk
1591:X 16 Aug 2022 14:41:18.273 # +vote-for-leader c3d7456b7e06e85f449cfc452ac46a4bb7310a59 23
1591:X 16 Aug 2022 14:41:18.288 # ebe43f2f88aba2d3015f05ba960fd7a609bb67e0 voted for c3d7456b7e06e85f449cfc452ac46a4bb7310a59 23
1591:X 16 Aug 2022 14:41:18.332 # +elected-leader master mycluster 192.168.69.91 6379
1591:X 16 Aug 2022 14:41:18.332 # +failover-state-select-slave master mycluster 192.168.69.91 6379
1591:X 16 Aug 2022 14:41:18.395 # +selected-slave slave 192.168.69.93:6379 192.168.69.93 6379 @ mycluster 192.168.69.91 6379
1591:X 16 Aug 2022 14:41:18.395 * +failover-state-send-slaveof-noone slave 192.168.69.93:6379 192.168.69.93 6379 @ mycluster 192.168.69.91 6379
1591:X 16 Aug 2022 14:41:18.453 * +failover-state-wait-promotion slave 192.168.69.93:6379 192.168.69.93 6379 @ mycluster 192.168.69.91 6379
1591:X 16 Aug 2022 14:41:19.310 * Sentinel new configuration saved on disk
1591:X 16 Aug 2022 14:41:19.310 # +promoted-slave slave 192.168.69.93:6379 192.168.69.93 6379 @ mycluster 192.168.69.91 6379
1591:X 16 Aug 2022 14:41:19.310 # +failover-state-reconf-slaves master mycluster 192.168.69.91 6379
1591:X 16 Aug 2022 14:41:19.382 # +failover-end master mycluster 192.168.69.91 6379
1591:X 16 Aug 2022 14:41:19.382 # +switch-master mycluster 192.168.69.91 6379 192.168.69.93 6379
1591:X 16 Aug 2022 14:41:19.382 * +slave slave 192.168.69.92:6379 192.168.69.92 6379 @ mycluster 192.168.69.93 6379
1591:X 16 Aug 2022 14:41:19.382 * +slave slave 192.168.69.91:6379 192.168.69.91 6379 @ mycluster 192.168.69.93 6379
1591:X 16 Aug 2022 14:41:19.387 * Sentinel new configuration saved on disk
1591:X 16 Aug 2022 14:41:24.397 # +sdown slave 192.168.69.91:6379 192.168.69.91 6379 @ mycluster 192.168.69.93 6379
1591:X 16 Aug 2022 14:41:24.397 # +sdown slave 192.168.69.92:6379 192.168.69.92 6379 @ mycluster 192.168.69.93 6379
This is the new Sentinel configuration after the failover:
protected-mode no
port 26379
sentinel monitor mycluster 192.168.69.91 6379 2
sentinel down-after-milliseconds mycluster 5000
# Generated by CONFIG REWRITE
latency-tracking-info-percentiles 50 99 99.9
user default on nopass ~* &* +@all
sentinel myid c3d7456b7e06e85f449cfc452ac46a4bb7310a59
sentinel config-epoch mycluster 23
sentinel leader-epoch mycluster 23
sentinel current-epoch 23
sentinel known-replica mycluster 192.168.69.91 6379
sentinel known-replica mycluster 192.168.69.92 6379
sentinel known-sentinel mycluster 192.168.69.92 26379 584784ea586a5954576eaa39be12542d4fae3175
sentinel known-sentinel mycluster 192.168.69.93 26379 ebe43f2f88aba2d3015f05ba960fd7a609bb67e0
Anyone could help me to understand what’s going on and where I’m wrong, please?
Thank you very much!
Bye