lvs DR模式调试
DS配置：
# cat /etc/keepalived_docker/keepalived.conf
! Configuration File for keepalived
global_defs {router_id LVS_70 # 设置lvs的id，在一个网络内应该是唯一的
}vrrp_instance VI_70 {state MASTER # 两个 DS，一个为 MASTER，一个为 BACKUPinterface ens33 # 当前 IP 对应的网络接口virtual_router_id 70 # 虚拟路由 ID(0-255)，在一个 VRRP 实例中主备服务器 ID 必须一样priority 100 # 优先级值设定：MASTER 要比 BACKUP 的值大advert_int 1 # 通告时间间隔：单位秒，主备要一致authentication { # 认证机制，主从节点保持一致即可auth_type PASSauth_pass 1111}virtual_ipaddress {192.168.137.70 # VIP，可配置多个}
}# LB 配置，定义对外提供服务的LVS的VIP以及port
virtual_server 192.168.137.70 80 {delay_loop 3 # 设置健康状态检查时间lb_algo wlc # 设置负载调度的算法，wrr/rr等lb_kind DR # 这里测试用了 Direct Route 模式persistence_timeout 600 # 持久连接超时时间即会话保持时间，这段时间内同一ip发起的请求将被转发到同一个 RSprotocol TCPvirtualhost LVSreal_server 192.168.137.3 80 {weight 1TCP_CHECK {connect_timeout 3retry 3 # 旧版本为 nb_get_retrydelay_before_retry 2connect_port 80}}real_server 192.168.137.4 80 {weight 1TCP_CHECK {connect_timeout 3retry 3 # 旧版本为 nb_get_retrydelay_before_retry 2connect_port 80}}
}DS开通ip转发：
#vim /etc/sysctl.conf
net.ipv4.ip_forward = 1
sysctl -pDS网卡信息：
[root@el7 lvs]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:af:ed:5f brd ff:ff:ff:ff:ff:ff
inet 192.168.137.2/24 brd 192.168.137.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.137.60/32 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.137.70/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:feaf:ed5f/64 scope link
valid_lft forever preferred_lft forever
RS脚本：
# cat lvs-rs.sh
#!/bin/bash
VIP=192.168.137.70
#/etc/rc.d/init.d/functions
case "$1" instart)echo "Start Real Server ..."ifconfig lo:0 $VIP netmask 255.255.255.255 broadcast $VIP uproute add -host $VIP dev lo:0#如果多个vip继续类似添加：#ifconfig lo:2 $VIP2 netmask 255.255.255.255 broadcast $VIP2 up#route add -host $VIP2 dev lo:2echo "1" > /proc/sys/net/ipv4/conf/lo/arp_ignoreecho "2" > /proc/sys/net/ipv4/conf/lo/arp_announceecho "1" > /proc/sys/net/ipv4/conf/all/arp_ignoreecho "2" > /proc/sys/net/ipv4/conf/all/arp_announcesysctl -p > /dev/null 2>&1systemctl stop firewalldecho "Real Server Started.";;stop)echo "Stop Real Server ..."ifconfig lo:0 down#如果多个vip继续类似添加：#ifconfig lo:2 downroute del $VIP >/dev/null 2>&1echo "0" > /proc/sys/net/ipv4/conf/lo/arp_ignoreecho "0" > /proc/sys/net/ipv4/conf/lo/arp_announceecho "0" > /proc/sys/net/ipv4/conf/all/arp_ignoreecho "0" > /proc/sys/net/ipv4/conf/all/arp_announceecho "Real Server Stopped.";;*)echo "Usage: $0 {start|stop}"exit 1
esac
exit 0# 添加VIP，修改内核参数（实现对内可见、对外隐藏），注意掩码必须是255.255.255.255
# 因为VIP是绑在环回网卡上（回环网卡离内核更近，数据包优先匹配）的，如果掩码不是4个255，则数据包在返回的时候匹配上了回环网卡，匹配不上真实网关，数据包发送不出去
#RS、DS 有同样的vip，同一个网段中拥有两个vip，客户端在网关发送arp广播需找vip时需要让RS不接受响应。
#arp_ignore 设置为1，意味着当别人的arp请求过来的时候，如果接收的设备没有这个ip，就不做出响应(这个ip在lo上，lo不是接收设备的进口)
#使用最好的ip来回应，什么是最好的ip？同一个网段内子网掩码最长的RS网卡信息：
[root@el8 lvs]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet 192.168.137.60/32 brd 192.168.137.60 scope global lo:0
valid_lft forever preferred_lft forever
inet 192.168.137.70/32 brd 192.168.137.70 scope global lo:1
valid_lft forever preferred_lft forever
inet 39.134.209.254/32 brd 39.134.209.254 scope global lo:254
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:3f:10:fe brd ff:ff:ff:ff:ff:ff
inet 192.168.137.3/24 brd 192.168.137.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe3f:10fe/64 scope link
valid_lft forever preferred_lft forever
查看lvs状态：
ipvsadm -Ln --persistent-conn
echo
ipvsadm -Ln --stats如：
[root@el7 lvs]# sh watch-ipvsadm.sh
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Weight PersistConn ActiveConn InActConn
-> RemoteAddress:Port
TCP 192.168.137.60:80 wlc persistent 600
-> 192.168.137.3:80 1 0 0 0
-> 192.168.137.4:80 1 0 0 0
TCP 192.168.137.70:80 wlc persistent 600
-> 192.168.137.3:80 1 0 0 0
-> 192.168.137.4:80 1 0 0 0IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Conns InPkts OutPkts InBytes OutBytes
-> RemoteAddress:Port
TCP 192.168.137.60:80 2 216 0 9006 0
-> 192.168.137.3:80 2 216 0 9006 0
-> 192.168.137.4:80 0 0 0 0 0
TCP 192.168.137.70:80 9 61 0 3097 0
-> 192.168.137.3:80 9 61 0 3097 0
-> 192.168.137.4:80 0 0 0 0 0
ipvsadm管理命令：
#手工添加vip和dr规则
ip addr add $vip dev bond0
ipvsadm -A -t $vip:80 -s wrr
ipvsadm -a -t $vip:80 -r $rs1:80 -g
ipvsadm -a -t $vip:80 -r $rs2:80 -g#将某个RS的权重调为0：
ipvsadm -e -t 192.168.137.70:80 -r 192.168.137.3 -g -w 0#删除某个vip
ipvsadm -D -t 192.168.137.70:80客户端测试：
curl -vo /dev/null "http://mytest.com/test.mp3" -H "User-Agent:mytest" -m 600 -x 192.168.137.60:80vrrp抓包：
tcpdump -i ens33 vrrp -nlvs tcpdump抓包：
tcpdump -i any -e -n -nn -s 0 '(host $lvs_vip or host $client_ip) and tcp port $service_port'如：
tcpdump -i any -e -n -nn -s 0 '(host 192.168.137.70 or host 192.168.137.1) and tcp port 80'
错误处理：
问题1：IPVS (cmd 1159,errno 2): No such file or directory
解决：ipvs cmd 1160 is IP_VS_SO_SET_DELDEST and cmd 1159 is IP_VS_SO_SET_ADDDEST so it appears to be attempting to delete the failed real server and then add the real server.
--end--