기록's

#!/bin/sh

#%# family=auto
#%# capabilities=autoconf

#GETNUM=`/bin/sh /home/test/bin/app/munin_rta.sh`

TODAY=`date +"%Y-%m-%d"`
TOMCAT_HOME2=/home/test/SERVER/

RTA1=0


MONITOR_LOG2=$TOMCAT_HOME2/logs/play_MONITOR.$TODAY.log

if [ -e $MONITOR_LOG2 ]; then
#RTA1=$(tail -n 1000 $MONITOR_LOG1 | grep "SESSION\[" | tail -n 5 | awk -F "SESSION" '{print $2}' | tr -d "[]")
RTA1=$(tail -n 3000 $MONITOR_LOG2 | grep WebSocketMonitor | grep SESSION | tail -n 1 | awk '{print $12}' | awk '{print substr($0, 9)}' | awk '{print substr($0, 0, index($0,"]")-1)}')
fi


if [ "$1" = "autoconf" ]; then
        if [ -n ${RTA1} ] ; then
                echo yes
                exit 0
        else
                echo no
                exit 0
        fi
fi

if [ "$1" = "config" ]; then
        echo 'graph_title User Session(Concurrent User)'
        echo 'graph_args -r --lower-limit 0'
        echo 'graph_vlabel users'
        echo 'graph_category Apps'
        echo 'total.label Total users'
        echo 'total.min 0'
        echo 'total.draw LINE2'
        echo 'total.type GAUGE'
        exit 0
fi

if [ -n ${RTA1} ] ; then
    echo "total.value $RTA1";
else
    echo "total.value U";
fi

#!/bin/bash

CUR_YMD=`date '+%Y-%m-%d'`
LOG_DIR=/home/test/logs
LOG_FILE=play.$CUR_YMD.log

LINE_LIMIT=30000
SUB_YMD_HM_0=`date '+%Y-%m-%d %H:%M'`
SUB_YMD_HM_1=`date --date '-1 min' '+%Y-%m-%d %H:%M'`
SUB_YMD_HM_2=`date --date '-2 min' '+%Y-%m-%d %H:%M'`
SUB_YMD_HM_3=`date --date '-3 min' '+%Y-%m-%d %H:%M'`
SUB_YMD_HM_4=`date --date '-4 min' '+%Y-%m-%d %H:%M'`
SUB_YMD_HM_5=`date --date '-5 min' '+%Y-%m-%d %H:%M'`


TOTAL=`tail -n $LINE_LIMIT $LOG_DIR/$LOG_FILE | grep -e "ERROR\|Exception" | grep -e "$SUB_YMD_HM_0\|$SUB_YMD_HM_1\|$SUB_YMD_HM_2\|$SUB_YMD_HM_3\|$SUB_YMD_HM_4\|$SUB_YMD_HM_5" | wc -l`

CATEGORY="Apps"
TITLE="ERROR or Exception"

if [ "$1" = "autoconf" ]; then
        if [ -n ${TOTAL} ] ; then
                echo yes
                exit 0
        else
                echo no
                exit 0
        fi
fi


if [ "$1" = "config" ]; then
    echo "graph_title $TITLE";
    echo "graph_args -r --lower-limit 0"
    echo "graph_vlabel threads";
    echo "graph_category $CATEGORY";
    echo "graph_order total";
    echo "total.label total";
    echo "total.draw LINE2";
    exit 0;
fi


if [ -n ${TOTAL} ] ; then
    echo "total.value $TOTAL";
else
    echo "total.value U";
fi

개요

일반적으로 설정 했던 기준이지만, 해당 값은 변경 될 수 있다.

OS 직접적으로 체크 해야 되는 부분은 다른 모니터링툴 사용으로 처리를 했다. 

예를 들어 메모리나 디스크 사용율 등

EBS

VolumeQueueLength : 0.5

VolumeIOPS : IOPS값의 80% 정도

BurstBalance : 5분 / 70 이상

Lambda

Error : 1

ConcurrentExecutions : 5분 / 500보다 큼

ELB

UnHealthyHostCount : 1분 / 1 보다 크거나 같다

ApplicationELB

HTTPCode_Target_5XX_Count : 1분 / 1보다 크거나 같다.

TargetResponseTime : 1분 / 5보다 크거나 같다.

EC2

StatusCheckFailed_System(AutoRecovery) : 1분 / 0 보다 크다

CPUUtilization : 5분 / 30보다 큼

IO나 VolumeQueue(QueueLength)에 대해 GCP 같은 경우 OS에서 처리할 파라미터

sudo sysctl -w vm.dirty_ratio=10
sudo sysctl -w vm.dirty_background_ratio=5

참조 URL : https://www.blackmoreops.com/2014/09/22/linux-kernel-panic-issue-fix-hung_task_timeout_secs-blocked-120-seconds-problem/

구성도

작업 순서

1. Redis 다운로드 및 설치

wget http://download.redis.io/releases/redis-5.0.5.tar.gz
gzip -d redis-5.0.5.tar.gz
tar xvf redis-5.0.5.tar
cd redis-5.0.5
redis-5.0.5]# make
mv redis-5.0.5 /usr/local/redis

2. Redis 유저 / 그룹 생성

-- redis 유저 / 그룹 생성 공통
groupadd redis
adduser -g redis redis

2-1 폴더 생성

mkdir -p /usr/local/redis/cluster/7000
mkdir -p /usr/local/redis/cluster/conf
cd /usr/local/redis/cluster/conf

3. config 파일 생성

vi redis.7000.conf

################################## INCLUDES ###################################
# include /path/to/local.conf
# include /path/to/other.conf
 
################################## MODULES #####################################
# loadmodule /path/to/my_module.so
# loadmodule /path/to/other_module.so
 
################################## NEkrORK #####################################
bind 1.1.1.1 127.0.0.1
protected-mode yes
port 7000
tcp-backlog 511
# unixsocket /tmp/redis.sock
# unixsocketperm 700
timeout 0
tcp-keepalive 300
 
################################# GENERAL #####################################
dir /usr/local/redis/cluster/7000
daemonize yes
supervised no
pidfile redis.pid
loglevel notice
logfile redis.log
# syslog-enabled no
# syslog-ident redis
# syslog-facility local0
# databases 16
always-show-logo yes
 
################################ SNAPSHOTTING  ################################
save ""
# save 900 1
# save 300 10
# save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.cluster.rdb
 
################################# REPLICATION #################################
#masterauth 비밀번호 # 설정 안함
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync yes
repl-diskless-sync-delay 5
# repl-ping-replica-period 10
repl-timeout 60
repl-disable-tcp-nodelay no
# repl-backlog-size 1mb
# repl-backlog-ttl 3600
replica-priority 100
# min-replicas-to-write 3
# min-replicas-max-lag 10
# min-replicas-max-lag is set to 10.
# replica-announce-ip 5.5.5.5
# replica-announce-port 1234
 
################################## SECURITY ###################################
#requirepass 비밀번호 # 설정 안함
# rename-command CONFIG ""
 
################################### CLIENTS ####################################
# maxclients 10000
 
############################## MEMORY MANAGEMENT ################################
# maxmemory <bytes>
# maxmemory-policy noeviction
# maxmemory-samples 5
# replica-ignore-maxmemory yes
 
############################# LAZY FREEING ####################################
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
 
############################## APPEND ONLY MODE ###############################
appendonly no
appendfilename "appendonly.aof"
# appendfsync always
appendfsync everysec
# appendfsync no
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
 
################################ LUA SCRIPTING  ###############################
lua-time-limit 5000
 
################################ REDIS CLUSTER  ###############################
cluster-enabled yes
cluster-config-file nodes.conf
cluster-node-timeout 5000
# cluster-replica-validity-factor 10
# cluster-migration-barrier 1
# cluster-require-full-coverage yes
# cluster-replica-no-failover no
 
########################## CLUSTER DOCKER/NAT support  ########################
# cluster-announce-ip 10.1.1.5
# cluster-announce-port 6379
# cluster-announce-bus-port 6380
 
################################## SLOW LOG ###################################
slowlog-log-slower-than 10000
slowlog-max-len 128
 
################################ LATENCY MONITOR ##############################
latency-monitor-threshold 0
 
############################# EVENT NOTIFICATION ##############################
notify-keyspace-events ""
 
############################### ADVANCED CONFIG ###############################
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4096
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
# client-query-buffer-limit 1gb
# proto-max-bulk-len 512mb
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes
# lfu-log-factor 10
# lfu-decay-time 1
 
########################### ACTIVE DEFRAGMENTATION #######################
# activedefrag yes
# active-defrag-ignore-bytes 100mb
# active-defrag-threshold-lower 10
# active-defrag-threshold-upper 100
# active-defrag-cycle-min 5
# active-defrag-cycle-max 75
# active-defrag-max-scan-fields 1000

4. Redis 실행

- 경로이동

cd /usr/local/redis/cluster

# 실행 스크립트 생성
vi redis_start.sh
#!/bin/bash
 
/usr/local/redis/src/redis-server /usr/local/redis/cluster/conf/redis.7000.conf &

chmod +x redis_start.sh
chown -R redis:redis /usr/local/redis

5. 클러스터 생성

# 클러스터 생성
# 1.1.1.2:7000를 1.1.1.1:7000의 슬레이브로 지정
./redis-cli --cluster create 1.1.1.1:7000 1.1.2.1:7000 1.1.3.1:7000
./redis-cli --cluster add-node 1.1.1.2:7000 1.1.1.1:7000 --cluster-slave
./redis-cli --cluster add-node 1.1.2.2:7000 1.1.2.1.213:7000 --cluster-slave
./redis-cli --cluster add-node 1.1.3.2:7000 1.1.3.1:7000 --cluster-slave

5-1 유저 스위칭 후 수행

# redis 유저로 변경하여 실행
sudo su redis
./redis_start.sh

6. 클러스터 확인

cd /usr/local/redis/src
./redis-cli --cluster check 1.1.1.1:7000
1.1.1.1:7000 (c6643cb1...) -> 0 keys | 5461 slots | 1 slaves.
1.1.2.1:7000 (1c9669af...) -> 0 keys | 5461 slots | 1 slaves.
1.1.3.1:7000 (f20cc469...) -> 0 keys | 5462 slots | 1 slaves.
[OK] 0 keys in 3 masters.
0.00 keys per slot on average.
>>> Performing Cluster Check (using node xxxx:7000)
M: c6643cb1d0d1ab962b501a2264e6d9ee9083a39b xxxxx:7000
   slots:[0-5460] (5461 slots) master
   1 additional replica(s)
S: 6fe1fa03f42054304913f5214478be2254116736 xxxxx:7000
   slots: (0 slots) slave
   replicates f20cc469580f924916132db20b4b655bec216e0d
S: 390e2fe2dd4945f0498cbcf505c2d8a0c5c8b566 xxxxx:7000
   slots: (0 slots) slave
   replicates c6643cb1d0d1ab962b501a2264e6d9ee9083a39b
S: b56ba5e6b439c30e1aa12859690dd94d511e6d77 xxxx:7000
   slots: (0 slots) slave
   replicates 1c9669af8a75ffdd9a82e23be6bc78db1c38027f
M: 1c9669af8a75ffdd9a82e23be6bc78db1c38027f xxxxx:7000
   slots:[10923-16383] (5461 slots) master
   1 additional replica(s)
M: f20cc469580f924916132db20b4b655bec216e0d xxxxx:7000
   slots:[5461-10922] (5462 slots) master
   1 additional replica(s)
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.

구성도

THP 비활성화

/etc/rc.local 에 아래 내용 추가

echo never > /sys/kernel/mm/transparent_hugepage/enabled

Redis 다운로드 및 설치

Redis 다운로드

cd /usr/local
wget http://download.redis.io/releases/redis-4.0.14.tar.gz
tar xzf redis-4.0.14.tar.gz
mv redis-4.0.14 redis
cd /usr/local/redis/src
make

디렉토리 생성

cd /usr/local/redis/ 
mkdir -p /usr/local/redis/cluster/7000
mkdir -p /usr/local/redis/cluster/conf
touch /usr/local/redis/cluster/redis_start.sh
chmod 700 /usr/local/redis/cluster/redis_start.sh

실행파일 생성

vi /usr/local/redis/cluster/redis_start.sh

#!/bin/sh
/usr/local/redis/src/redis-server /usr/local/redis/cluster/conf/redis.7000.conf

conf 파일 생성

conf 파일 생성

vi  /usr/local/redis/cluster/conf/redis.7000.conf

masterauth , requirepass에는 패스워드 입력

################################## NETWORK #####################################
#bind에는 접속 가능한 서버의 IP주소를 입력합니다.
bind 11.1.1.1 127.0.0.1
protected-mode yes
port 7000
tcp-backlog 511
timeout 0
tcp-keepalive 300
 
################################# GENERAL #####################################
dir /usr/local/redis/cluster/7000
daemonize yes
supervised no
pidfile redis.pid
loglevel notice
logfile redis.log
always-show-logo yes
 
################################ SNAPSHOTTING  ################################
save ""
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.cluster.rdb
 
################################# REPLICATION #################################
masterauth ***********
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync yes
repl-diskless-sync-delay 5
repl-timeout 60
repl-disable-tcp-nodelay no
slave-priority 100
 
 
################################## SECURITY ###################################
requirepass *********
 
############################# LAZY FREEING ####################################
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
slave-lazy-flush no
 
############################## APPEND ONLY MODE ###############################
appendonly no
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble no
 
################################ LUA SCRIPTING  ###############################
lua-time-limit 5000
 
################################ REDIS CLUSTER  ###############################
cluster-enabled yes
cluster-config-file nodes.conf
cluster-node-timeout 5000
 
################################## SLOW LOG ###################################
slowlog-log-slower-than 10000
slowlog-max-len 128
 
################################ LATENCY MONITOR ##############################
latency-monitor-threshold 0
 
############################# EVENT NOTIFICATION ##############################
notify-keyspace-events ""
 
############################### ADVANCED CONFIG ###############################
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes

redis 사용자 및 그룹 생성

groupadd redis

useradd -g redis redis

chown -R redis:redis /usr/local/redis

redis 기동 확인

redis 시작

su - redis 

/usr/local/redis/cluster/redis_start.sh

기동 확인

ps -ef | grep redis

redis cluster 구성

cluster 구성은 전체 서버가 아닌 한대만 설치해도 된다

cluster 구성용 redis-trib-pass 파일 다운로드

cd /usr/local/redis/src

wget https://raw.githubusercontent.com/seekasia/redis/master/src/redis-trib-pass.rb

redis-trib-pass.rb 첫번째 행 ruby2.1 => ruby 수정
vi redis-trib-pass.rb

chmod 700 redis-trib-pass.rb

ruby를 다운로드

cd /usr/local/src

wget https://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.1.tar.gz

tar xvf ruby-2.5.1.tar.gz

ruby 컴파일

yum install -y bzip2 gcc openssl-devel readline-devel zlib-devel

cd ruby-2.5.1/

./configure

make

make install 

ruby --version

gem install redis

마스터 서버 cluster 설정

*redis cluster 구성시 host name 이 아닌 Ip Address를 이용한다.

/usr/local/redis/src/redis-trib-pass.rb create --auth password 1.1.1.1:7000 1.1.1.2:7000 1.1.1.3:7000

cluster 확인 (마스터 ID 확인)

/usr/local/redis/src/redis-cli -p 7000 -a password cluster nodes

d6c9d17ba96137d13d0217a7e62ea7e7250a090e 1.1.1.1:7000@17000 myself,master - 0 0 1 connected 0-5460
5ac01a2c3936ef788fe63f8e9467156734b07aae 1.1.1.2:7000@17000 master - 0 1563850868038 2 connected 5461-10922
64908665fb8d8f966c5b0b3c82b60c7858db6d6e 1.1.1.3:7000@17000 master - 0 1563850868037 3 connected 10923-16383

* 첫번째 키가 마스터 키

slave 추가

#/usr/local/redis/src/redis-trib.rb add-node --slave --master-id <마스터ID>  --auth 비밀번호 <슬레이브IP:port> <현재실행서버IP:port>

예:

Cluster 1 master slave 설정

/usr/local/redis/src/redis-trib-pass.rb add-node --slave --master-id d6c9d17ba96137d13d0217a7e62ea7e7250a090e --auth password 1.1.1.2:7000 1.1.1.1:7000

/usr/local/redis/src/redis-trib-pass.rb add-node --slave --master-id d6c9d17ba96137d13d0217a7e62ea7e7250a090e --auth password 1.1.1.3:7000 1.1.1.1:7000

Cluster 2 master slave 설정

/usr/local/redis/src/redis-trib-pass.rb add-node --slave --master-id 5ac01a2c3936ef788fe63f8e9467156734b07aae --auth password 1.1.1.4:7000 1.1.1.1:7000

/usr/local/redis/src/redis-trib-pass.rb add-node --slave --master-id 5ac01a2c3936ef788fe63f8e9467156734b07aae --auth password 1.1.1.5:7000 1.1.1.1:7000

Cluster 3 master slave 설정

/usr/local/redis/src/redis-trib-pass.rb add-node --slave --master-id 64908665fb8d8f966c5b0b3c82b60c7858db6d6e --auth password 1.1.1.6:7000 1.1.1.1:7000

/usr/local/redis/src/redis-trib-pass.rb add-node --slave --master-id 64908665fb8d8f966c5b0b3c82b60c7858db6d6e --auth password 1.1.1.7:7000 1.1.1.1:7000

CentOS 7에서 redis 6 설치

redis6에서는 C11을 서 포트 하는 compiler가 필요

gcc에서는 4.9이후부터 서포트 한다

gcc version확인

gcc --version

scl을 이용하고 기본 directory를 변경하지 않고 compile한다

sudo yum install centos-release-scl
sudo yum install devtoolset-7-gcc*

scl enable devtoolset-7 bash
which gcc
gcc --version

이전 complie실패했을 때에는 clean

make distclean

redis source를 compile

cd /usr/local/redis/src
make

rocky linux 8.4에서 설치

groupadd redis
useradd -g redis redis



dnf module list redis

redis6을 defaul로 해

dnf module enable redis:6 -y

redis6을 설치

dnf install -y redis

default가 redis5이 되어 있는 경우에는 reset하고 처음부터 다시 

dnf module reset redis

기동

systemctl enable --now redis

상태 확인

systemctl status redis

conf파일 위치

/etc/redis.conf

port변경

Could not create server TCP listening socket 127.0.0.1:7000: bind: Permission denied

vi /etc/selinux/config
의 enforced를 disabled로 변경

auth설정
/etc/redis.conf의 다음 항목을 변경

masterauth
requirepass

환경설정

기동환경

 - IPv6 를 무효로 하다

    ㄴ /etc/sysconfig/named에 옵션 추가.

OPTIONS="-4"

 - named.conf에서 ipv6의 무효화. 아래의 항목을 설정한다.

listen-on-v6 port 53 { none; };

로그설정

출력 로그를 디폴트에서 변경하므로 로테이션 설정이 필요하다.

 - /etc/logrotate.d/named

/var/named/data/query.log
/var/named/data/default.log
/var/named/data/named_stats.txt
{
    daily
    rotate 7
    missingok
    create 0644 named named
    postrotate
        /sbin/service named reload  2> /dev/null > /dev/null || true
    endscript
}