1、实验环境的搭建
准备3台主机：主机1主机名：haproxyip：172.25.254.100主机2主机名：hp-RS1ip：172.25.254.10主机3主机名：hp-RS2ip：172.25.254.20
两台RS都安装nginx：
[haproxy-RS1]# dnf install nginx -y
[haproxy-RS2]# dnf install nginx -y
设置开机自启动（两个都设置）
[haproxy-RS1]systemctl enable --now nginx.service
关闭火墙（两个都设置）
[haproxy-RS1]systemctl disable --now firewalld.service
两台RS设置nginx的index.html内容
[haproxy-RS1]echo "RS1-172.25.254.10" > /usr/share/nginx/html/index.html
连通测试
[haproxy-haproxy]curl 172.25.254.10
2、haproxy的安装和frontend区
安装haproxy
[haproxy-haproxy]dnf install haproxy -y
启动
[haproxy-haproxy]systemctl enable --now haproxy.service
进入haproxy配置文件
[haproxy-haproxy]vim /etc/haproxy/haproxy.cfg
frontend webcluster
bind *:80
mode http
balance roundrobinuse
backend webserver
backend webserver
server webl 172.25.254.10:80
server web2 172.25.254.20:80重启服务
[haproxy-haproxy]systemctl restart haproxy.service
3、global配置（多进程与多线程）
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
globallog 127.0.0.1 local2chroot /var/lib/haproxypidfile /var/run/haproxy.pidmaxconn 100000user haproxygroup haproxydaemon# turn on stats unix socketstats socket /var/lib/haproxy/haproxy.sock1 mode 600 level admin process 1 #
启用多个sock文件stats socket /var/lib/haproxy/haproxy.sock2 mode 600 level admin process 2nbproc 2 #启用多进程cpu-map 1 0 #进程和cpu核心绑定防止cpu抖动从而减少系统资源消耗cpu-map 2 1 #2 表示第二个进程，1表示第二个cpu核心...下面内容省略 ...
查看多进程信息
haproxy haproxy]# pstree -p | grep haproxy|-haproxy(4816)-+-haproxy(4820)| `-haproxy(4821)启用多线程haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
log 127.0.0.1 local2chroot /var/lib/haproxypidfile /var/run/haproxy.pidmaxconn 100000user haproxygroup haproxydaemon# turn on stats unix socketstats socket /var/lib/haproxy/haproxy.sock1 mode 600 level admin process 1 #
启用多个sock文件stats socket /var/lib/haproxy/haproxy.sock2 mode 600 level admin process 2#nbproc 2 #cpu-map 1 0 #cpu-map 2 1 nbthread 2 #启用多线程...下面内容省略...多线程对比
未开启多线程
haproxy ~]# cat /proc/xxxx(haproxy子进程id)/status
...上面内容省略...
Threads: 1
...下面内容省略...
开启后
haproxy ~]# cat /proc/xxxx(haproxy子进程id)/status
...上面内容省略...
Threads: 2
...下面内容省略...
4、socat工具
#修改配置文件
[root@haproxy ~]# vim /etc/haproxy/haproxy.cfg
stats socket /var/lib/haproxy/stats mode 600 level admin
#查看haproxy状态
[root@haproxy ~]# echo "show info" | socat stdio /var/lib/haproxy/stats
Name: HAProxy
Version: 2.4.22-f8e3218
Release_date: 2023/02/14
Nbthread: 1
Nbproc: 1
Process_num: 1
Pid: 33542
Uptime: 0d 0h03m43s
Uptime_sec: 223
Memmax_MB: 0
PoolAlloc_MB: 0
#查看集群状态
[root@haproxy ~]# echo "show servers state" | socat stdio /var/lib/haproxy/stats
1
# be_id be_name srv_id srv_name srv_addr srv_op_state srv_admin_state srv_uweight
srv_iweight srv_time_since_last_change srv_check_status srv_check_result
srv_check_health srv_check_state srv_agent_state bk_f_forced_id srv_f_forced_id
srv_fqdn srv_port srvrecord srv_use_ssl srv_check_port srv_check_addr
srv_agent_addr srv_agent_port
2 webcluster 1 web1 172.25.254.20 2 0 2 2 188 6 3 7 6 0 0 0 - 80 - 0 0 - - 0
2 webcluster 2 web2 172.25.254.30 2 0 1 1 188 6 3 7 6 0 0 0 - 80 - 0 0 - - 0
4 static 1 static 127.0.0.1 0 0 1 1 187 8 2 0 6 0 0 0 - 4331 - 0 0 - - 0
5 app 1 app1 127.0.0.1 0 0 1 1 187 8 2 0 6 0 0 0 - 5001 - 0 0 - - 0
5 app 2 app2 127.0.0.1 0 0 1 1 187 8 2 0 6 0 0 0 - 5002 - 0 0 - - 0
5 app 3 app3 127.0.0.1 0 0 1 1 186 8 2 0 6 0 0 0 - 5003 - 0 0 - - 0
5 app 4 app4 127.0.0.1 0 0 1 1 186 8 2 0 6 0 0 0 - 5004 - 0 0 - - 0
#查看集群权重
[root@haproxy ~]# echo get weight webcluster/web1 | socat stdio
/var/lib/haproxy/stats
2 (initial 2)
[root@haproxy ~]# echo get weight webcluster/web2 | socat stdio
/var/lib/haproxy/stats
1 (initial 1)#设置权重
[root@haproxy ~]# echo "set weight webcluster/web1 1 " | socat stdio
/var/lib/haproxy/stats
[root@haproxy ~]# echo "set weight webcluster/web1 2 " | socat stdio
/var/lib/haproxy/stats
#下线后端服务器
[root@haproxy ~]# echo "disable server webserver_80/webserver1 " | socat stdio
/var/lib/haproxy/stats
#上线后端服务器
[root@haproxy ~]# echo "enable server webserver_80/webserver1 " | socat stdio
/var/lib/haproxy/stats
5、haproxy算法
静态算法
static-rr
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80bind 172.25.254.100:80mode httpbalance static-rrserver webserver1 192.168.0.101:80 weight 2 check inter 3s fall 3 rise 5server webserver2 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...first
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80bind 172.25.254.100:80mode httpbalance first server webserver1 192.168.0.101:80 maxconn 3 check inter 3s fall 3 rise 5 server webserver2 192.168.0.102:80 check inter 3s fall 3 rise 5
...上面内容省略...
#在两台主机上分别执行此循环，可以观察是否102被调度到while true;do curl 172.25.254.100 ; sleep 0.1;done
动态算法
roundrobin
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80bind 172.25.254.100:80mode httpbalance roundrobin server webserver1 192.168.0.101:80 weight 1 check inter 3s fall 3 rise 5 server webserver2 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...动态调整权重[root@haproxy ~]# echo "set weight webserver_80/webserver1 2" | socat stdio
/var/lib/haproxy/haproxy.sockleastconn
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80bind 172.25.254.100:80mode httpbalance leastconnserver webserver1 192.168.0.101:80 weight 1 check inter 3s fall 3 rise 5 server webserver2 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...source
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80bind 172.25.254.100:80mode httpbalance sourceserver webserver1 192.168.0.101:80 weight 1 check inter 3s fall 3 rise 5 server webserver2 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...
[root@node10 ~]# for N in {1..6}; do curl 172.25.254.100; done
RS1 server - 192.168.0.101
RS1 server - 192.168.0.101
RS1 server - 192.168.0.101
RS1 server - 192.168.0.101
RS1 server - 192.168.0.101
RS1 server - 192.168.0.101uri
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80bind 172.25.254.100:80mode httpbalance uriserver webserver1 192.168.0.101:80 weight 1 check inter 3s fall 3 rise 5 server webserver2 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...url_param
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80bind 172.25.254.100:80mode httpbalance url_param name,userid #支持对多个url_param hashserver webserver1 192.168.0.101:80 weight 1 check inter 3s fall 3 rise 5 server webserver2 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...hdr
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80bind 172.25.254.100:80mode httpbalance hdr(User-Agent)hash-type consistentserver webserver1 192.168.0.101:80 weight 1 check inter 3s fall 3 rise 5 server webserver2 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...
[root@node10 ~]# curl -v 172.25.254.100
[root@node10 ~]# curl -vA "firefox" 172.25.254.100
[root@node10 ~]# curl -vA "sougou" 172.25.254.100
6、cookie
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80bind 172.25.254.100:80option forwardformode httpbalance roundrobincookie WEBCOOKIE insert nocache indirectserver webserver1 192.168.0.101:80 cookie web1 weight 1 check inter 3s fall
3 rise 5server webserver2 192.168.0.102:80 cookie web2 weight 1 check inter 3s fall
3 rise 5
...上面内容省略...[root@node10 ~]# curl -i 172.25.254.100
HTTP/1.1 200 OK
date: Wed, 10 Jul 2024 16:36:17 GMT
server: Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips
last-modified: Thu, 04 Jul 2024 11:18:39 GMT
etag: "1b-61c6a1bd2408d"
accept-ranges: bytes
content-length: 27
content-type: text/html; charset=UTF-8
set-cookie: WEBCOOKIE=web2; path=/
cache-control: private
RS2 server - 192.168.0.102
[Administrator.WIN-20240602BIS] ➤ curl -i 172.25.254.100
HTTP/1.1 200 OK
server: nginx/1.20.1
date: Wed, 10 Jul 2024 08:36:43 GMT
content-type: text/html
content-length: 18
last-modified: Wed, 10 Jul 2024 04:09:05 GMT
etag: "668e0961-12"
accept-ranges: bytes
set-cookie: WEBCOOKIE=web1; path=/
cache-control: private
RS1 192.168.0.101#curl访问时指定cookie
[root@node10 ~]# curl -b WEBCOOKIE=web1 172.25.254.100
RS1 server - 192.168.0.101
[root@node10 ~]# curl -b WEBCOOKIE=web2 172.25.254.100
RS2 server - 192.168.0.102
[root@node10 ~]# curl -vb WEBCOOKIE=web1 172.25.254.100
* About to connect() to 172.25.254.100 port 80 (#0)
* Trying 172.25.254.100...
* Connected to 172.25.254.100 (172.25.254.100) port 80 (#0)
> GET / HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 172.25.254.100
> Accept: */*
> Cookie: WEBCOOKIE=web1
>
< HTTP/1.1 200 OK
< server: nginx/1.20.1
< date: Wed, 10 Jul 2024 08:38:25 GMT
< content-type: text/html
< content-length: 18
< last-modified: Wed, 10 Jul 2024 04:09:05 GMT
< etag: "668e0961-12"
< accept-ranges: bytes
<
RS1 192.168.0.101
* Connection #0 to host 172.25.254.100 left intact
7、HAProxy状态页
启用状态页
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen stats：mode httpbind 0.0.0.0:8888stats enablelog globalstats uri /status #自定义stats page uristats auth lee:lee #认证，此行可以出现多次
...上面内容省略...
测试：
浏览器访问：172.25.254.100:8888/status登录状态页
#pid为当前pid号，process为当前进程号，nbproc和nbthread为一共多少进程和每个进程多少个线程
pid = 27134 (process #1, nbproc = 1, nbthread = 1)
#启动了多长时间
uptime = 0d 0h00m04s
#系统资源限制：内存/最大打开文件数/
system limits: memmax = unlimited; ulimit-n = 200029
#最大socket连接数/单进程最大连接数/最大管道数maxpipes
maxsock = 200029; maxconn = 100000; maxpipes = 0
#当前连接数/当前管道数/当前连接速率
current conns = 2; current pipes = 0/0; conn rate = 2/sec; bit rate = 0.000 kbps
#运行的任务/当前空闲率
Running tasks: 1/14; idle = 100 %
active UP： #在线服务器
backup UP： #标记为backup的服务器
active UP, going down： #监测未通过正在进入down过程
backup UP, going down： #备份服务器正在进入down过程
active DOWN, going up： #down的服务器正在进入up过程
backup DOWN, going up： #备份服务器正在进入up过程
active or backup DOWN： #在线的服务器或者是backup的服务器已经转换成了down状态
not checked： #标记为不监测的服务器
#active或者backup服务器人为下线的
active or backup DOWN for maintenance (MAINT)
#active或者backup被人为软下线(人为将weight改成0)
active or backup SOFT STOPPED for maintenance
8、IP透传
四层IP透传
#未开启透传的四层代理
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80bind 172.25.254.100:80mode tcpbalance roundrobinserver webserver1 192.168.0.101:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...
#正常的nginx配置
[root@rs1 ~]# vim /etc/nginx/nginx.conf
。。。内容省略。。。
http {log_format main '$remote_addr - $remote_user [$time_local] "$request"''$status $body_bytes_sent "$http_referer" ''"$http_user_agent" "$http_x_forwarded_for"';
。。。内容省略。。。server {listen 80;listen [::]:80;server_name _;root /usr/share/nginx/html;。。。内容省略。。。}
}
#在访问haproxy后查看nginx日志
[root@rs1 ~]# tail -n 3 /var/log/nginx/access.log
192.168.0.10 - - [10/Jul/2024:15:21:00 +0800] "GET / HTTP/1.1"200 18 "-"
"curl/7.29.0" "-"192.168.0.10 - - [10/Jul/2024:15:26:11 +0800] "GET /
HTTP/1.1"200 18 "-" "curl/7.29.0" "-"
在此日志中是无法看到真实访问源地址的开启四层透传
#nginx 配置：在访问日志中通过变量$proxy_protocol_addr 记录透传过来的客户端IP
[root@rs1 ~]# vim /etc/nginx/nginx.conf
。。。内容省略。。。
http {log_format main '$remote_addr - $remote_user [$time_local] "$request"'' "$proxy_protocol_addr"''$status $body_bytes_sent "$http_referer" ''"$http_user_agent" "$http_x_forwarded_for"';
。。。内容省略。。。server {
listen 80 proxy_protocol; #启用此项，将无法直接访问此网站，只能通过四层代理
访问listen [::]:80;server_name _;root /usr/share/nginx/html;。。。内容省略。。。}
}
#修改haproxy
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80bind 172.25.254.100:80mode tcpbalance roundrobin server webserver1 192.168.0.101:80 send-proxy weight 1 check inter 3s fall 3
rise 5
...上面内容省略...
#查看日志内容
[root@rs1 ~]# tail -n 3 /var/log/nginx/access.log
192.168.0.10 - - [10/Jul/2024:15:21:00 +0800] "GET / HTTP/1.1"200 18 "-"
"curl/7.29.0" "-"
192.168.0.10 - - [10/Jul/2024:15:26:11 +0800] "GET / HTTP/1.1"200 18 "-"
"curl/7.29.0" "-"
192.168.0.10 - - [10/Jul/2024:15:41:56 +0800] "GET / HTTP/1.1" "172.25.254.10"200
18 "-" "curl/7.29.0"七层IP透传
#修改haproxy
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80option forwardforbind 172.25.254.100:80mode httpbalance roundrobin server webserver1 192.168.0.101:80 send-proxy weight 1 check inter 3s fall 3
rise 5server webserver1 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...
9、ACL
域名匹配
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
frontend testaclbind :80mode http########### ACL settings #######################acl web_host hdr_dom(host) www.timinglee.org########### host ###########################use_backend timinglee_host if web_host########### default server ###################default_backend default_webserver
backend timinglee_hostmode httpserver web1 192.168.0.101:80 check weight 1 inter 3s fall 3 rise 5server web2 192.168.0.102:80 check weight 1 inter 3s fall 3 rise 5
backend default_webservermode httpserver web1 172.25.254.10:80 check weight 1 inter 3s fall 3 rise 5
...上面内容省略...#在浏览器所在主机中做地址解析
[root@node10 html]# vim /etc/hosts
172.25.254.100 www.timinglee.org
#测试结果
[root@node10 html]# curl www.timinglee.org
RS1 192.168.0.101
[root@node10 html]# curl www.timinglee.org
RS2 server - 192.168.0.102
[root@node10 html]# curl 172.25.254.100
default web server node10基于源IP或子网调度访问
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
frontend testaclbind :80mode http########### ACL settings #######################acl ip_test src 172.25.254.1 192.168.0.0/24########### host ###########################use_backend ip_test-host if ip_test########### default server ###################default_backend default_webserver
backend ip_test-hostmode httpserver web1 192.168.0.101:80 check weight 1 inter 3s fall 3 rise 5
backend default_webservermode httpserver web1 172.25.254.10:80 check weight 1 inter 3s fall 3 rise 5
测试结果[172.25.254.10 root@node10 html]# curl 172.25.254.100
default web server node10
[172.25.254.1 Administrator.WIN-20240602BIS] ➤ curl 172.25.254.100
RS1 192.168.0.101
[192.168.0.102 root@rs1 ~]# curl 192.168.0.101
RS1 192.168.0.101
10、自定义错误页面
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
defaults
mode http...内容省略...timeout client 1mtimeout server 1mtimeout http-keep-alive 10stimeout check 10smaxconn 1000000errorfile 503 /haproxy/errorpages/503page.http
[root@haproxy ~]# mkdir /haproxy/errorpages/ -p
[root@haproxy ~]# cp /usr/share/haproxy/503.http /haproxy/errorpages/503page.http
[root@haproxy ~]# vim /haproxy/errorpages/503page.http
HTTP/1.0 503 Service Unavailable
Cache-Control: no-cache
Connection: close
Content-Type: text/html;charset=UTF-8^M
<html><body><h1>什么动物生气最安静</h1>
大猩猩！！
</body></html>
测试：
关闭后端的RS主机
然后用浏览器去访问172.25.254.100
11、HAProxy 四层负载
对 MySQL 服务实现四层负载
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen mysql_portbind :3306mode tcpbalance leastconnserver mysql1 192.168.0.101:3306 checkserver mysql2 192.168.0.102:3306 check
#或者使用frontend和backend实现
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
frontend mysql_portbind :3306mode tcpuse_backend mysql_rs
backend mysql_rsmode tcpbalance leastconnserver mysql1 192.168.0.101:3306 checkserver mysql2 192.168.0.102:3306 check
haproxy ~]# systemctl restart haproxy.service
#在后端服务器安装和配置mariadb服务
rs1 ~]# yum install mariadb-server -y
rs2 ~]# yum install mariadb-server -y
rs1 ~]# vim /etc/my.cnf
[mysqld]
server-id=1 #在另一台主机为
rs2 ~]# vim /etc/my.cnf
[mysqld]
server-id=2 #在另一台主机为
rs1 ~]# systemctl start mariadb
rs2 ~]# systemctl start mariadb
rs1 ~]# mysql -e "grant all on *.* to lee@'%' identified by 'lee';"
rs2 ~]# mysql -e "grant all on *.* to lee@'%' identified by 'lee';"
#测试
[root@node10 ~]# mysql -ulee -plee -h 172.25.254.100 -e "show variables like
'hostname'"
+---------------+-------+
| Variable_name | Value |
+---------------+-------+
| hostname | rs2 |
+---------------+-------+
[root@node10 ~]# mysql -ulee -plee -h 172.25.254.100 -e "show variables like
'hostname'"
+---------------+-------+
| Variable_name | Value |
+---------------+-------+
| hostname | rs1 |
+---------------+-------+
[root@node10 ~]# mysql -ulee -plee -h172.25.254.100 -e "select @@server_id"
+-------------+
| @@server_id |
+-------------+
| 1 |
+-------------+
[root@node10 ~]# mysql -ulee -plee -h172.25.254.100 -e "select @@server_id"
+-------------+
| @@server_id |
+-------------+
| 2 |
+-------------+
12、证书
证书制作
haproxy ~]# mkdir /etc/haproxy/certs/
haproxy ~]# openssl req -newkey rsa:2048 \
-nodes -sha256 –keyout /etc/haproxy/certs/timinglee.org.key \
-x509 -days 365 -out /etc/haproxy/certs/timinglee.org.crthttps配置示例
haproxy ~]# vim /etc/haproxy/haproxy.cfg
frontend webserverbind *:80redirect scheme https if !{ ssl_fc }mode httpuse_backend webcluster
frontend webserver-httpsbind *:443 ssl crt /etc/haproxy/timinglee.org.pemmode httpuse_backend webcluster
backend webclustermode httpbalance roundrobinserver web1 172.25.254.200:80 check inter 3s fall 3 rise 5server web2 172.25.254.201:80 check inter 3s fall 3 rise 5
[root@客户端 ~]#curl -IkL http://172.25.254.100
HTTP/1.1 302 Found
content-length: 0
location: https://www.timinglee.org/
cache-control: no-cache
HTTP/1.1 200 OK
date: Sat, 04 Apr 2020 02:31:31 GMT
server: Apache/2.4.6 (CentOS) PHP/5.4.16
last-modified: Thu, 02 Apr 2020 01:44:13 GMT
etag: "a-5a244f01f8adc"
accept-ranges: bytes
content-length: 10
content-type: text/html; charset=UTF-8
[root@centos6 ~]#curl -Ik https://www.timinglee.org
HTTP/1.1 200 OK
date: Sat, 04 Apr 2020 02:31:50 GMT
server: Apache/2.4.6 (CentOS) PHP/5.4.16
last-modified: Thu, 02 Apr 2020 01:44:28 GMT
etag: "a-5a244f0fd5175"
accept-ranges: bytes
content-length: 10
content-type: text/html; charset=UTF-8
