Kubernetes1.25中Redis集群部署实例
1、概述
我们知道在 Kubernetes 容器编排平台中, 我们可以非常方便的进行应用的扩容缩, 同时也能非常方便的进行业务的迭代，本章主要讲解在Kubernetes1.25搭建Redis单实例和Redis集群主从同步的环境流程步骤, 如果是高频访问重要的线上业务我们最好是部署在物理机器上;
2、Kubernetes环境说明
# 集群信息
[root@k8s-master ~]# kubectl cluster-info
Kubernetes control plane is running at https://10.211.55.11:6443
CoreDNS is running at https://10.211.55.11:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxyTo further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.
[root@k8s-master ~]## 集群节点
[root@k8s-master ~]# kubectl get nodes -owide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
k8s-master Ready control-plane 18h v1.25.0 10.211.55.11 <none> CentOS Stream 8 4.18.0-408.el8.x86_64 docker://20.10.22
k8s-node1 Ready <none> 18h v1.25.0 10.211.55.12 <none> CentOS Stream 8 4.18.0-408.el8.x86_64 docker://20.10.22
k8s-node2 Ready <none> 18h v1.25.0 10.211.55.13 <none> CentOS Stream 8 4.18.0-408.el8.x86_64 docker://20.10.22# 动态卷
[root@k8s-master ~]# kubectl get sc -n dev
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-storage storage-nfs Delete Immediate false 6s3、Redis 集群主从
在Kubernetes中部署Redis集群很有挑战，因为每个Redis实例都依赖于一个配置文件，该文件跟踪其他集群实例及其角色。为此，我们需要结合使用Kubernetes状态集（StatefulSets）和持久卷（PersistentVolumes）
3.1、准备工作
3.1.1、Redis 配置文件
配置文件：/root/redis/redis.conf 内容如下
# 监听端口
port 6379
# 启用外部连接关闭安全模式
protected-mode no
requirepass 123456
# 开启Redis的AOF持久化 && 日志文件
appendonly yes
appendfilename appendonly.aof
# AOF持久化文件存在的位置以及其文件名称
dir /root/redis/data
dbfilename dump.rdb
# 每秒钟同步一次折中的方案
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
# 主从认证及其从节点只读
masterauth weiyigeek.top
slave-read-only yes
# 集群模式打开
cluster-enabled yes
cluster-config-file /root/redis/nodes.conf
cluster-node-timeout 5000
# 当负责一个插槽的主库下线且没有相应的从库进行故障恢复时集群仍然可用
cluster-require-full-coverage no
# 只有当一个主节点至少拥有其他给定数量个处于正常工作中的从节点的时候，才会分配从节点给集群中孤立的主节点
cluster-migration-barrier 13.1.2、集群配置更新文件
更新文件：/root/redis/update-node.sh 内容如下
for pod_name in $(kubectl get pod -n dev -l app=redis-cluster -o jsonpath='{ range.items [*]}{.spec.hostname} ');doecho ${pod_name}kubectl exec -it -n dev ${pod_name} -- sh -c "redis-cli -a 123456 cluster nodes" | grep "myself";kubectl exec -it -n dev ${pod_name} -- sh -c "redis-cli -a 123456 info replication" | egrep "role|slave"echo .
done3.1.3、集群节点配置文件
配置文件：/root/redis/nodes.conf
3.1.4、数据存储目录
数据存储目录：/root/redis/data
3.1.5、动态卷制作
制作方式请看这篇文章：https://blog.csdn.net/u011837804/article/details/128692744
4、部署redis集群
4.1、redis-cluster.yaml清单文件内容
apiVersion: v1
kind: ConfigMap
metadata:name: redis-clusternamespace: dev
data:# 外部命令参数传递执行精妙之处值得学习update-node.sh: |#!/bin/shREDIS_NODES="/data/nodes.conf"if [ ! -f /data/nodes.conf ];then touch /data/nodes.conf;fised -i -e "/myself/ s/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/${POD_IP}/" ${REDIS_NODES}exec "$@"redis.conf: |+# 监听端口port 6379# 启用外部连接关闭安全模式protected-mode nomasterauth 123456requirepass 123456# 开启Redis的AOF持久化 && 日志文件appendonly yes appendfilename appendonly.aof # AOF持久化文件存在的位置以及其文件名称dir /datadbfilename dump.rdbslave-read-only yes# 每秒钟同步一次折中的方案appendfsync everysecno-appendfsync-on-rewrite noauto-aof-rewrite-percentage 100auto-aof-rewrite-min-size 64mb# 集群模式打开cluster-enabled yes cluster-config-file /data/nodes.confcluster-node-timeout 5000# 当负责一个插槽的主库下线且没有相应的从库进行故障恢复时集群仍然可用cluster-require-full-coverage no# 只有当一个主节点至少拥有其他给定数量个处于正常工作中的从节点的时候，才会分配从节点给集群中孤立的主节点cluster-migration-barrier 1
---
apiVersion: apps/v1
kind: StatefulSet
metadata:name: redis-clusternamespace: dev
spec:serviceName: redis-clusterreplicas: 6selector:matchLabels:app: redis-clustertemplate:metadata:labels:app: redis-clusterspec:containers:- name: redisimage: redis:7.0imagePullPolicy: IfNotPresentports:- containerPort: 6379name: client- containerPort: 16379name: gossipcommand: ["/conf/update-node.sh", "redis-server", "/conf/redis.conf"]env:- name: POD_IPvalueFrom:fieldRef:fieldPath: status.podIPvolumeMounts:- name: confmountPath: /confreadOnly: false- name: datamountPath: /datareadOnly: false- name: timezonemountPath: /etc/localtime # 在Pod中时区设置(挂载主机的时区)volumes:- name: confconfigMap:name: redis-clusterdefaultMode: 0755- name: timezone hostPath:path: /usr/share/zoneinfo/Asia/ShanghaivolumeClaimTemplates:- metadata:name: dataspec:accessModes: [ "ReadWriteOnce" ]storageClassName: "nfs-storage"resources:requests:storage: 50Mi
---
# headless Service
apiVersion: v1
kind: Service
metadata:name: redis-clusternamespace: dev
spec:clusterIP: "None"ports:- port: 6379targetPort: 6379name: client- port: 16379targetPort: 16379name: gossipselector:app: redis-cluster4.2、执行清单文件查看效果
# 执行清单文件
[root@k8s-master ~]# kubectl apply -f redis-cluster.yaml
configmap/redis-cluster created
statefulset.apps/redis-cluster created
service/redis-cluster created
[root@k8s-master ~]## 查看6个pod已成功运行
[root@k8s-master ~]# kubectl get pods -n dev
NAME READY STATUS RESTARTS AGE
nfs-client-provisioner-79599dbcfb-gpq78 1/1 Running 0 3h26m
redis-cluster-0 1/1 Running 0 64s
redis-cluster-1 1/1 Running 0 60s
redis-cluster-2 1/1 Running 0 57s
redis-cluster-3 1/1 Running 0 54s
redis-cluster-4 1/1 Running 0 51s
redis-cluster-5 1/1 Running 0 47s
[root@k8s-master ~]#4.3、手动配置集群
# 查看所有节点
[root@k8s-master ~]# kubectl get pods -n dev
NAME READY STATUS RESTARTS AGE
nfs-client-provisioner-79599dbcfb-gpq78 1/1 Running 0 3h29m
redis-cluster-0 1/1 Running 0 4m55s
redis-cluster-1 1/1 Running 0 4m51s
redis-cluster-2 1/1 Running 0 4m48s
redis-cluster-3 1/1 Running 0 4m45s
redis-cluster-4 1/1 Running 0 4m42s
redis-cluster-5 1/1 Running 0 4m38s
[root@k8s-master ~]## 通过命令获取所有节点IP
[root@k8s-master ~]# kubectl get pod -n dev -l app=redis-cluster -o jsonpath='{ range.items [*]}{.status.podIP}:6379 '| sed "s# :6379 ##g"
172.17.36.94:6379 172.17.169.142:6379 172.17.36.95:6379 172.17.169.143:6379 172.17.36.96:6379 172.17.169.144:6379[root@k8s-master ~]#
[root@k8s-master ~]## 进入其中一个节点
[root@k8s-master ~]# kubectl exec -it redis-cluster-0 /bin/bash -n dev
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
root@redis-cluster-0:/data## 执行集群配置命令
root@redis-cluster-0:~# redis-cli --cluster create 172.17.36.94:6379 172.17.169.142:6379 172.17.36.95:6379 172.17.169.143:6379 172.17.36.96:6379 172.17.169.144:6379 --cluster-replicas 1 -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
>>> Performing hash slots allocation on 6 nodes...
Master[0] -> Slots 0 - 5460
Master[1] -> Slots 5461 - 10922
Master[2] -> Slots 10923 - 16383
Adding replica 172.17.36.96:6379 to 172.17.36.94:6379
Adding replica 172.17.169.144:6379 to 172.17.169.142:6379
Adding replica 172.17.169.143:6379 to 172.17.36.95:6379
M: 86db26e2a3f1c67b41b2c0020ae01995852ddb24 172.17.36.94:6379slots:[0-5460] (5461 slots) master
M: 8d41a36025b8a1c8b5f454e5201891a35ee5b946 172.17.169.142:6379slots:[5461-10922] (5462 slots) master
M: 66d691db2f2e1702f4d8a659c1ef3858cbd6da29 172.17.36.95:6379slots:[10923-16383] (5461 slots) master
S: 8e09beab9a6ea23fc1861969e13f8a3f4743a012 172.17.169.143:6379replicates 66d691db2f2e1702f4d8a659c1ef3858cbd6da29
S: 05c281495f5626a1d9c55bc5b574774b5063b7db 172.17.36.96:6379replicates 86db26e2a3f1c67b41b2c0020ae01995852ddb24
S: 4b9fc44a31db3ef8d566e382da319c229112fa90 172.17.169.144:6379replicates 8d41a36025b8a1c8b5f454e5201891a35ee5b946
Can I set the above configuration? (type 'yes' to accept): yes4.4、脚本配置集群
# 方式1
export REDIS_POD_IP=$(kubectl get pod -n database -l app=redis-cluster -o jsonpath='{ range.items [*]}{.status.podIP}:6379 '| sed "s# :6379 ##g")
kubectl exec -it -n dev redis-cluster-0 -- sh -c "/usr/local/bin/redis-cli -a 123456 --cluster create --cluster-replicas 1 ${REDIS_POD_IP}"# 方式2
kubectl -n ev exec -it redis-cluster-0 -- redis-cli --cluster create --cluster-replicas 1 $(kubectl get pods -n dev -l app=redis-cluster -o jsonpath='{range.items[*]}{.status.podIP}:6379 '| sed "s# :6379 ##g")