RedisをSentinelで監視する

http://qiita.com/toshiro3/items/931ededeece6b8b1c680

RedisをSentinelで監視する方法を記述します。

SentinelがRedisのレプリケーションの状態を監視し、マスタのダウンを検知すると任意のスレーブがマスタへ昇格されます。

環境

CentOS 6.5
Redis 2.8.16

構成

Redisのレプリケーション(master x 1台, slave x 2台)を3台のSentinelで監視する構成とします。

役割	名称	ポート
master	redis	6379
slave	redis02	6380
slave	redis03	6381
sentinel	sentinel	26379
sentinel	sentinel02	26380
sentinel	sentinel03	26381

Redisのインストール

$ wget http://download.redis.io/releases/redis-2.8.16.tar.gz $ tar zxvf redis-2.8.16.tar.gz
$ cd cd redis-2.8.16
$ make 2>&1 | tee make.log
$ make test 2>&1 | tee make-test.log
$ sudo sh -c "PREFIX=/usr/local/redis make install 2>&1 | tee make-install.log"

$ sudo mkdir -p /usr/local/redis/conf
$ sudo mkdir -p /usr/local/redis/data/6379 /usr/local/redis/data/6380 /usr/local/redis/data/6381

Redisの設定

redis(master)の設定

$ sudo cp redis.conf /usr/local/redis/conf/
$ vi /usr/local/redis/conf/redis.conf
$ diff redis.conf /usr/local/redis/conf/redis.conf
37c37
< daemonize no
---
> daemonize yes
103c103
< logfile ""
---
> logfile /var/log/redis.log
187c187
< dir ./
---
> dir /usr/local/redis/data/6379

redis(master)の起動

$ sudo /usr/local/redis/bin/redis-server /usr/local/redis/conf/redis.conf
$ /usr/local/redis/bin/redis-cli　info replication

# Replication
role:master
connected_slaves:0
master_repl_offset:0
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0

redis02(slave)の設定

$ sudo cp /usr/local/redis/conf/redis.conf /usr/local/redis/conf/redis-6380.conf
$ sudo vi /usr/local/redis/conf/redis-6380.conf
$ diff /usr/local/redis/conf/redis.conf /usr/local/redis/conf/redis-6380.conf
45c45
< port 6379
---
> port 6380
103c103
< logfile /var/log/redis.log
---
> logfile /var/log/redis-6380.log
187c187
< dir /usr/local/redis/data/6379
---
> dir /usr/local/redis/data/6380
205a206
> slaveof 127.0.0.1 6379

* redis03(slave)の設定

$ sudo cp /usr/local/redis/conf/redis.conf /usr/local/redis/conf/redis-6381.conf
$ sudo vi /usr/local/redis/conf/redis-6381.conf
$ diff /usr/local/redis/conf/redis.conf /usr/local/redis/conf/redis-6381.conf
45c45
< port 6379
---
> port 6381
103c103
< logfile /var/log/redis.log
---
> logfile /var/log/redis-6381.log
187c187
< dir /usr/local/redis/data/6379
---
> dir /usr/local/redis/data/6381
205a206
> slaveof 127.0.0.1 6379

redis02(slave)を起動し、レプリケーションの状態を確認します。

$ sudo /usr/local/redis/bin/redis-server /usr/local/redis/conf/redis-6380.conf
$ /usr/local/redis/bin/redis-cli -p 6380 info replication

# Replication
role:slave
master_host:127.0.0.1
master_port:6379
master_link_status:up
master_last_io_seconds_ago:6
master_sync_in_progress:0
slave_repl_offset:29
slave_priority:100
slave_read_only:1
connected_slaves:0
master_repl_offset:0
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0

redis03(slave)を起動し、レプリケーションの状態を確認します。

$ sudo /usr/local/redis/bin/redis-server /usr/local/redis/conf/redis-6381.conf
$ /usr/local/redis/bin/redis-cli -p 6381 info replication

# Replication
role:slave
master_host:127.0.0.1
master_port:6379
master_link_status:up
master_last_io_seconds_ago:6
master_sync_in_progress:0
slave_repl_offset:197
slave_priority:100
slave_read_only:1
connected_slaves:0
master_repl_offset:0
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0

redis(master)へ接続してレプリケーションの状態を確認します。

$ /usr/local/redis/bin/redis-cli info replication

# Replication
role:master
connected_slaves:2
slave0:ip=127.0.0.1,port=6380,state=online,offset=281,lag=1
slave1:ip=127.0.0.1,port=6381,state=online,offset=281,lag=1
master_repl_offset:281
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:2
repl_backlog_histlen:280

Sentinelの設定

sentinelの設定及び起動

$ sudo cp sentinel.conf /usr/local/redis/conf/
$ diff sentinel.conf /usr/local/redis/conf/sentinel.conf
181c181,182
<
---
> daemonize yes
> logfile "/var/log/redis-sentinel.log"

$ sudo /usr/local/redis/bin/redis-server /usr/local/redis/conf/sentinel.conf --sentinel

監視状態の確認

$ /usr/local/bin/redis/redis-cli -p 26379
127.0.0.1:26379> sentinel masters
1)  1) "name"
2) "mymaster"
3) "ip"
4) "127.0.0.1"
5) "port"
6) "6379"
7) "runid"
8) "0d2111054628f7a654e2a43fbb6157d4f7563821"
9) "flags"
10) "master"
11) "pending-commands"
12) "0"
13) "last-ping-sent"
14) "0"
15) "last-ok-ping-reply"
16) "39"
17) "last-ping-reply"
18) "39"
19) "down-after-milliseconds"
20) "30000"
21) "info-refresh"
22) "1928"
23) "role-reported"
24) "master"
25) "role-reported-time"
26) "153550"
27) "config-epoch"
28) "0"
29) "num-slaves"
30) "2"
31) "num-other-sentinels"
32) "0"
33) "quorum"
34) "2"
35) "failover-timeout"
36) "180000"
37) "parallel-syncs"
38) "1"
127.0.0.1:26379> sentinel slaves mymaster
1)  1) "name"
2) "127.0.0.1:6380"
3) "ip"
4) "127.0.0.1"
5) "port"
6) "6380"
7) "runid"
8) "9dbefed61658340d2da2dc041081a2ac7f7398ec"
9) "flags"
10) "slave"
11) "pending-commands"
12) "0"
13) "last-ping-sent"
14) "0"
15) "last-ok-ping-reply"
16) "934"
17) "last-ping-reply"
18) "934"
19) "down-after-milliseconds"
20) "30000"
21) "info-refresh"
22) "2013"
23) "role-reported"
24) "slave"
25) "role-reported-time"
26) "162731"
27) "master-link-down-time"
28) "0"
29) "master-link-status"
30) "ok"
31) "master-host"
32) "127.0.0.1"
33) "master-port"
34) "6379"
35) "slave-priority"
36) "100"
37) "slave-repl-offset"
38) "12288"
2)  1) "name"
2) "127.0.0.1:6381"
3) "ip"
4) "127.0.0.1"
5) "port"
6) "6381"
7) "runid"
8) "f80c280e2b2bcb70429ecf235bfcbf341599dbc3"
9) "flags"
10) "slave"
11) "pending-commands"
12) "0"
13) "last-ping-sent"
14) "0"
15) "last-ok-ping-reply"
16) "934"
17) "last-ping-reply"
18) "934"
19) "down-after-milliseconds"
20) "30000"
21) "info-refresh"
22) "2013"
23) "role-reported"
24) "slave"
25) "role-reported-time"
26) "162731"
27) "master-link-down-time"
28) "0"
29) "master-link-status"
30) "ok"
31) "master-host"
32) "127.0.0.1"
33) "master-port"
34) "6379"
35) "slave-priority"
36) "100"
37) "slave-repl-offset"
38) "12288"
127.0.0.1:26379> quit

sentinel02(sentinel)の設定

$ sudo cp /usr/local/redis/conf/sentinel.conf /usr/local/redis/conf/sentinel-26380.conf
$ sudo vi /usr/local/redis/conf/sentinel-26380.conf
$ diff /usr/local/redis/conf/sentinel.conf /usr/local/redis/conf/sentinel-26380.conf
5c5
< port 26379
---
> port 26380
182c182
< logfile "/var/log/redis-sentinel.log"
---
> logfile "/var/log/redis-sentinel-26380.log"

sentinel03(sentinel)の設定

$ sudo cp /usr/local/redis/conf/sentinel.conf /usr/local/redis/conf/sentinel-26381.conf
$ sudo vi /usr/local/redis/conf/sentinel-26381.conf
$ diff /usr/local/redis/conf/sentinel.conf /usr/local/redis/conf/sentinel-26381.conf
5c5
< port 26379
---
> port 26381
182c182
< logfile "/var/log/redis-sentinel.log"
---
> logfile "/var/log/redis-sentinel-26381.log"

sentinel02(sentinel)の起動

$ sudo /usr/local/redis/bin/redis-server /usr/local/redis/conf/sentinel-26380.conf --sentinel

sentinel03(sentinel)の起動

$ sudo /usr/local/redis/bin/redis-server /usr/local/redis/conf/sentinel-26381.conf --sentinel

動作確認

redis(master)を停止します。

$ sudo lsof -i :6379
COMMAND     PID USER   FD   TYPE DEVICE SIZE/OFF NODE NAME
redis-ser 32226 root    4u  IPv6 480880      0t0  TCP *:6379 (LISTEN)
redis-ser 32226 root    5u  IPv4 480882      0t0  TCP *:6379 (LISTEN)
redis-ser 32226 root    6u  IPv4 480930      0t0  TCP localhost:6379->localhost:50787 (ESTABLISHED)
redis-ser 32226 root    7u  IPv4 480988      0t0  TCP localhost:6379->localhost:50790 (ESTABLISHED)
redis-ser 32226 root    8u  IPv4 481221      0t0  TCP localhost:6379->localhost:50793 (ESTABLISHED)
redis-ser 32226 root    9u  IPv4 481222      0t0  TCP localhost:6379->localhost:50794 (ESTABLISHED)
redis-ser 32226 root   10u  IPv4 481537      0t0  TCP localhost:6379->localhost:50800 (ESTABLISHED)
redis-ser 32226 root   11u  IPv4 481538      0t0  TCP localhost:6379->localhost:50801 (ESTABLISHED)
redis-ser 32226 root   12u  IPv4 481609      0t0  TCP localhost:6379->localhost:50808 (ESTABLISHED)
redis-ser 32226 root   13u  IPv4 481610      0t0  TCP localhost:6379->localhost:50809 (ESTABLISHED)
redis-ser 32232 root    6u  IPv4 480929      0t0  TCP localhost:50787->localhost:6379 (ESTABLISHED)
redis-ser 32242 root    6u  IPv4 480987      0t0  TCP localhost:50790->localhost:6379 (ESTABLISHED)
redis-ser 32278 root    6u  IPv4 481218      0t0  TCP localhost:50793->localhost:6379 (ESTABLISHED)
redis-ser 32278 root    7u  IPv4 481220      0t0  TCP localhost:50794->localhost:6379 (ESTABLISHED)
redis-ser 32308 root    6u  IPv4 481534      0t0  TCP localhost:50800->localhost:6379 (ESTABLISHED)
redis-ser 32308 root    7u  IPv4 481536      0t0  TCP localhost:50801->localhost:6379 (ESTABLISHED)
redis-ser 32313 root    6u  IPv4 481606      0t0  TCP localhost:50808->localhost:6379 (ESTABLISHED)
redis-ser 32313 root    7u  IPv4 481608      0t0  TCP localhost:50809->localhost:6379 (ESTABLISHED)

$ sudo kill 32226

sentinel(sentinel)のログを確認

/var/log/redis-sentinel.log

[32278] 17 Sep 05:52:41.546 # +sdown master mymaster 127.0.0.1 6379
[32278] 17 Sep 05:52:41.617 # +odown master mymaster 127.0.0.1 6379 #quorum 3/2
[32278] 17 Sep 05:52:41.649 # +new-epoch 1
[32278] 17 Sep 05:52:41.649 # +try-failover master mymaster 127.0.0.1 6379
[32278] 17 Sep 05:52:41.714 # +vote-for-leader 413cec268653074dcc0c83797f7b338517843ba0 1
[32278] 17 Sep 05:52:41.729 # 127.0.0.1:26381 voted for 55865d34479acc6872b084fb401c2e4fd220b707 1
[32278] 17 Sep 05:52:41.776 # 127.0.0.1:26380 voted for 55865d34479acc6872b084fb401c2e4fd220b707 1
[32278] 17 Sep 05:52:42.823 # +config-update-from sentinel 127.0.0.1:26381 127.0.0.1 26381 @ mymaster 127.0.0.1 6379
[32278] 17 Sep 05:52:42.824 # +switch-master mymaster 127.0.0.1 6379 127.0.0.1 6380
[32278] 17 Sep 05:52:42.824 * +slave slave 127.0.0.1:6381 127.0.0.1 6381 @ mymaster 127.0.0.1 6380
[32278] 17 Sep 05:52:42.974 * +slave slave 127.0.0.1:6379 127.0.0.1 6379 @ mymaster 127.0.0.1 6380
[32278] 17 Sep 05:53:13.007 # +sdown slave 127.0.0.1:6379 127.0.0.1 6379 @ mymaster 127.0.0.1 6380

レプリケーションの状態を確認

$ /usr/local/redis/bin/redis-cli -p 6380 info replication

# Replication
role:master
connected_slaves:1
slave0:ip=127.0.0.1,port=6381,state=online,offset=41135,lag=0
master_repl_offset:41135
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:2
repl_backlog_histlen:41134

$ /usr/local/redis/bin/redis-cli -p 6381 info replication

# Replication
role:slave
master_host:127.0.0.1
master_port:6380
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_repl_offset:45433
slave_priority:100
slave_read_only:1
connected_slaves:0
master_repl_offset:0
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0

redis(もとのmaster)を起動する

$ sudo /usr/local/redis/bin/redis-server /usr/local/redis/conf/redis.conf

sentinel(sentienl)のログを確認します。

/var/log/redis-sentinel.log

[32278] 17 Sep 05:58:38.187 # -sdown slave 127.0.0.1:6379 127.0.0.1 6379 @ mymaster 127.0.0.1 6380

再度、レプリケーションの状態を確認します。

$ /usr/local/redis/bin/redis-cli info replication

# Replication
role:slave
master_host:127.0.0.1
master_port:6380
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_repl_offset:102919
slave_priority:100
slave_read_only:1
connected_slaves:0
master_repl_offset:0
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0

$ /usr/local/redis/bin/redis-cli info replication -p 6380

# Replication
role:master
connected_slaves:2
slave0:ip=127.0.0.1,port=6381,state=online,offset=114973,lag=1
slave1:ip=127.0.0.1,port=6379,state=online,offset=114973,lag=1
master_repl_offset:114973
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:2
repl_backlog_histlen:114972

参考

Redis Download
Redis Sentinel
Documentation