当前位置：首页 > news >正文

Linux grep技巧结合awk查询

news 2025/8/1 14:31:35

一. 前提

1.1 数据准备

⏹file1.log

140 2024/07/08 12:35:01.547 c1server2	5485	[ERROR]	SPLREQUEST	seqNo=11459,eventController=PMT.payinfoforprc.test.search,oldest_data_search=2
110	2024/07/08 12:34:56.457 c1server1	5892	[INFO]	MYCODE2005	测试方法被调用。(method=selectSvc_Test_Id param cpId=16xx2 status=OK)	userid=adminUser
150 2024/07/08 12:35:02.231 c1server3	5634	[INFO]	SPLREQUEST	seqNo=11460,eventController=PMT.payinfoforprc.test.search,oldest_data_search=1
120	2024/07/08 12:34:57.235 c1server1	5675	[INFO]	MYCODE2005	测试方法被调用。(method=selectSvc_Test_Id param cpId=16xx2 status=OK)	userid=adminUser
160 2024/07/08 12:36:22.534 c1server3	2564	[WARN]	SPLREQUEST	seqNo=11461,eventController=PMT.payinfoforprc.test.search,oldest_data_search=1
130	2024/07/08 12:34:58.546 c1server2	6354	[INFO]	MYCODE2005	测试方法被调用。(method=selectSvc_Test_Id param cpId=16xx2 status=OK)	userid=adminUser
110 2024/07/08 12:34:56.456 c1server1	5892	[INFO]	SPLREQUEST	seqNo=11456,eventController=PMT.payinfoforprc.test.search,oldest_data_search=1
140	2024/07/08 12:35:01.548 c1server2	5485	[ERROR]	MYCODE2005	测试方法被调用。(method=selectSvc_Test_Id param cpId=45xx2 status=OK)	userid=adminUser
120 2024/07/08 12:34:57.234 c1server1	5675	[INFO]	SPLREQUEST	seqNo=11457,eventController=PMT.payinfoforprc.test.search,oldest_data_search=2
150	2024/07/08 12:35:02.232 c1server3	5634	[INFO]	MYCODE2005	测试方法被调用。(method=selectSvc_Test_Id param cpId=45xx2 status=OK)	userid=adminUser
130 2024/07/08 12:34:58.545 c1server2	6354	[INFO]	SPLREQUEST	seqNo=11458,eventController=PMT.payinfoforprc.test.search,oldest_data_search=1
160	2024/07/08 12:36:22.535 c1server3	2564	[WARN]	MYCODE2005	测试方法被调用。(method=selectSvc_Test_Id param cpId=45xx2 status=OK)	userid=adminUser

⏹file2.log

210 2024/07/08 12:34:56.456 c1server1	5892	[INFO]	SPLREQUEST	seqNo=21456,eventController=PMT.payinfoforprc.test.search,oldest_data_search=1
240	2024/07/08 12:35:01.548 c1server2	5485	[ERROR]	MYCODE2005	测试方法被调用。(method=selectSvc_Test_Id param cpId=89xx2 status=OK)	userid=adminUser
260 2024/07/08 12:34:57.234 c1server1	5675	[INFO]	SPLREQUEST	seqNo=21457,eventController=PMT.payinfoforprc.test.search,oldest_data_search=2
210	2024/07/08 12:35:02.232 c1server3	5634	[INFO]	MYCODE2005	测试方法被调用。(method=selectSvc_Test_Id param cpId=16xx2 status=OK)	userid=adminUser
240 2024/07/08 12:34:58.545 c1server2	6354	[INFO]	SPLREQUEST	seqNo=21458,eventController=PMT.payinfoforprc.test.search,oldest_data_search=1
260	2024/07/08 12:36:22.535 c1server3	2564	[WARN]	MYCODE2005	测试方法被调用。(method=selectSvc_Test_Id param cpId=89xx2 status=OK)	userid=adminUser

1.2 数据说明

第一列用户存储线程号，一个完整的用户请求要分为很多步骤，每一个步骤的线程号都是相同的
但是由于多个用户同时访问服务器，因此一个请求所产生的各个步骤的日志并不是连续打印，而是错落间隔在日志的各部分中
我们可以通过线程号，来将一个请求的完整内容抽取出来
第二列存储时间
第三列存储服务器名称，一个系统可能有多个服务器
eventController：功能的名称
cpId：加盟店ID

二. 查询

查询PMT.payinfoforprc.test.search功能
然后通过oldest_data_search=1进一步过滤
然后通过awk '{print $1}'获取出第一列(awk默认通过空格分列)
- ./file1.log:150
- ./file1.log:160
再通过awk -F ":" '{print "^"$2 " " $1}'构造查询key
- ^150 ./file1.log
- ^160 ./file1.log
再通过xargs -L 1 grep -a 命令，从前面的命令中读取一行输入，并将其作为参数传递给下一个命令grep -a 构造查询命令
- grep -a ^150 ./file1.log
- grep -a ^160 ./file1.log
最后再从查询的结果中，通过grep -E "SPLREQUEST|MYCODE2005"进一步筛选

grep -a "PMT.payinfoforprc.test.search" ./file*.log | \
grep -a "oldest_data_search=1" | \
awk '{print $1}' | \
awk -F ":" '{print "^"$2 " " $1}' | \
xargs -L 1 grep -a | \
grep -E "SPLREQUEST|MYCODE2005"

2.1 统计每个加盟店搜索的次数

⏹完整命令

cpId=[^ ]*
- 查询不为空的cpId
awk '{$1=$1; print $0}'
- 清除行中的多余空格。通过重新格式化行
- awk 会用单个空格替换多个连续的空格

grep -a "PMT.payinfoforprc.test.search" ./file*.log | \
grep -a "oldest_data_search=1" | \
awk '{print $1}' | \
awk -F ":" '{print "^"$2 " " $1}' | \
xargs -L 1 grep -a | \
grep -E "MYCODE2005" | \
grep -o -e "param cpId=[^ ]*" | \
uniq -c | \
awk '{$1=$1; print $0}' | \
sort

查询拆解1

$ grep -a "PMT.payinfoforprc.test.search" ./file*.log | \
> grep -a "oldest_data_search=1" | \
> awk '{print $1}'./file1.log:150
./file1.log:160
./file1.log:110
./file1.log:130
./file2.log:210
./file2.log:240

查询拆解2

$ grep -a "PMT.payinfoforprc.test.search" ./file*.log | \
> grep -a "oldest_data_search=1" | \
> awk '{print $1}' | \
> awk -F ":" '{print "^"$2 " " $1}'^150 ./file1.log
^160 ./file1.log
^110 ./file1.log
^130 ./file1.log
^210 ./file2.log
^240 ./file2.log

查询拆解3
※为方便打印展示效果，将grep -a替换为echo 'grep -a'

$ grep -a "PMT.payinfoforprc.test.search" ./file*.log | \
> grep -a "oldest_data_search=1" | \
> awk '{print $1}' | \
> awk -F ":" '{print "^"$2 " " $1}' | \
> xargs -L 1 echo 'grep -a'grep -a ^150 ./file1.log
grep -a ^160 ./file1.log
grep -a ^110 ./file1.log
grep -a ^130 ./file1.log
grep -a ^210 ./file2.log
grep -a ^240 ./file2.log

查询拆解4

$ grep -a "PMT.payinfoforprc.test.search" ./file*.log | \
> grep -a "oldest_data_search=1" | \
> awk '{print $1}' | \
> awk -F ":" '{print "^"$2 " " $1}' | \
> xargs -L 1 grep -a | \
> grep -E "MYCODE2005"150     2024/07/08 12:35:02.232 c1server3       5634    [INFO]  MYCODE2005      测试方法被调用。(method=selectSvc_Test_Id param cpId=45xx2 status=OK)    userid=adminUser
160     2024/07/08 12:36:22.535 c1server3       2564    [WARN]  MYCODE2005      测试方法被调用。(method=selectSvc_Test_Id param cpId=45xx2 status=OK)    userid=adminUser
110     2024/07/08 12:34:56.457 c1server1       5892    [INFO]  MYCODE2005      测试方法被调用。(method=selectSvc_Test_Id param cpId=16xx2 status=OK)    userid=adminUser
130     2024/07/08 12:34:58.546 c1server2       6354    [INFO]  MYCODE2005      测试方法被调用。(method=selectSvc_Test_Id param cpId=16xx2 status=OK)    userid=adminUser
210     2024/07/08 12:35:02.232 c1server3       5634    [INFO]  MYCODE2005      测试方法被调用。(method=selectSvc_Test_Id param cpId=16xx2 status=OK)    userid=adminUser
240     2024/07/08 12:35:01.548 c1server2       5485    [ERROR] MYCODE2005      测试方法被调用。(method=selectSvc_Test_Id param cpId=89xx2 status=OK)    userid=adminUser

查询拆解5

$ grep -a "PMT.payinfoforprc.test.search" ./file*.log | \
> grep -a "oldest_data_search=1" | \
> awk '{print $1}' | \
> awk -F ":" '{print "^"$2 " " $1}' | \
> xargs -L 1 grep -a | \
> grep -E "MYCODE2005" | \
> grep -o -e "param cpId=[^ ]*"param cpId=45xx2
param cpId=45xx2
param cpId=16xx2
param cpId=16xx2
param cpId=16xx2
param cpId=89xx2

查询拆解6

$ grep -a "PMT.payinfoforprc.test.search" ./file*.log | \
> grep -a "oldest_data_search=1" | \
> awk '{print $1}' | \
> awk -F ":" '{print "^"$2 " " $1}' | \
> xargs -L 1 grep -a | \
> grep -E "MYCODE2005" | \
> grep -o -e "param cpId=[^ ]*" | \
> uniq -c2 param cpId=45xx23 param cpId=16xx21 param cpId=89xx2

查询拆解7

$ grep -a "PMT.payinfoforprc.test.search" ./file*.log | \
> grep -a "oldest_data_search=1" | \
> awk '{print $1}' | \
> awk -F ":" '{print "^"$2 " " $1}' | \
> xargs -L 1 grep -a | \
> grep -E "MYCODE2005" | \
> grep -o -e "param cpId=[^ ]*" | \
> uniq -c | \
> awk '{$1=$1; print $0}'2 param cpId=45xx2
3 param cpId=16xx2
1 param cpId=89xx2