EmguCV学习笔记 VB.Net 11.9 姿势识别 OpenPose
版权声明：本文为博主原创文章，转载请在显著位置标明本文出处以及作者网名，未经作者允许不得用于商业目的。
EmguCV是一个基于OpenCV的开源免费的跨平台计算机视觉库,它向C#和VB.NET开发者提供了OpenCV库的大部分功能。
教程VB.net版本请访问：EmguCV学习笔记 VB.Net 目录-CSDN博客
教程C#版本请访问：EmguCV学习笔记 C# 目录-CSDN博客
笔者的博客网址：https://blog.csdn.net/uruseibest
教程配套文件及相关说明以及如何获得pdf教程和代码，请移步：EmguCV学习笔记
学习VB.Net知识，请移步： vb.net 教程 目录_vb中如何用datagridview-CSDN博客
学习C#知识，请移步：C# 教程 目录_c#教程目录-CSDN博客
11.9 姿势识别 OpenPose
OpenPose模型是一种用于人体姿态估计的深度学习模型，它能够检测出包括眼睛、鼻子、手臂、腿等18个人体的关键点，并估计它们的坐标位置和置信度。
0：Nose（鼻子）、1：neck（脖子）、2：rshoulder（右肩）、3：relbow（右肘部）、4：rwrist（右手腕）、5：shoulder（左肩）、6：lelbow（左肘部）、7：lwrist（左手腕）、8：rhip（右臀部）、9：rknee（右膝盖）、10：rankle（右脚踝）、11：lhip（左臀部）、12：lknee（左膝盖）、13：lankle（左脚踝）、14：reye（右眼）、15：leye（左眼）、16：rear（右耳）、17：lear（左耳）、18：background（背景，主要是作为下一步扩展使用，在实际中不处理）
在使用OpenPose模型时，通常需要将输入图像作为模型的输入，经过处理后得到一个四维数组作为输出结果。这个四维数组，其维度为(N, P, H, W)，
各个维度的含义：
- N：在输入图像中检测到的人体数量。
- P：估计的关键点数，包括了人体的身体部位和手指关节等，只需要取前18个。
- H：关键点的坐标信息在输出结果中的高度，在实际使用中就是DnnInvoke.BlobFromImage中size参数设置输出的Height，如果最终输出到源图像，那么应该按照比例进行还原。
- W：关键点的坐标信息在输出结果中的宽度，在实际使用中就是DnnInvoke.BlobFromImage中size参数设置输出的Width，如果最终输出到源图像，那么应该按照比例进行还原。
具体到某个元素的值就是该点是人体关键点的置信度，例如（0,2,10,30）返回（0,2,height,width）中的最大值为0.759，那么可以认为坐标(10,30)是右肩的可能性为75.9%。
从上面可以看出，在这个四维数组中，每个元素包含了x坐标（维度W）、y坐标（维度H）和置信度三个值。因此，可以通过遍历该四维数组并解析每个元素来获取所有关键点的坐标信息和置信度，从而进行人体姿态估计的后续处理。
【代码位置：frmChapter11】Button10_Click、getMaxPoint
'关键点信息
Structure Keypoint
Dim conf As Single '置信度
Dim p As Point '关键点坐标
End Structure
'使用openpose显示人体关键点
Private Sub Button10_Click(sender As Object, e As EventArgs) Handles Button10.Click
'人体关键点
Dim body_Keypoint() As String = {"nose", "neck", "rshoulder", "relbow", "rwrist", "lshoulder",
"lelbow", "lwrist", "rhip", "rknee", "rankle", "lhip",
"lknee", "lankle", "reye", "leye", "rear", "lear", "background"}
Dim m As New Mat("C:\learnEmgucv\action.jpg", ImreadModes.Color)
Dim net As Dnn.Net = DnnInvoke.ReadNetFromTensorflow("C:\learnEmgucv\openpose\graph_opt.pb")
Dim blob As Mat = DnnInvoke.BlobFromImage(m, 1.0, New Drawing.Size(360, 360), New MCvScalar(127.5, 127.5, 127.5), False, False)
net.SetInput(blob)
Dim mout As Mat = net.Forward()
'返回四维数组
Dim fout(,,,) As Single
fout = mout.GetData()
Dim H As Integer = fout.GetLength(2)
Dim W As Integer = fout.GetLength(3)
Dim lkeypoint As New List(Of Keypoint)
'获得关键点信息
lkeypoint = getMaxPoint(fout)
Dim x, y As Single
For i As Integer = 0 To lkeypoint.Count - 1
'按照比例获得关键点在源图像中的坐标
x = (lkeypoint(i).p.X / W) * m.Width
y = (lkeypoint(i).p.Y / H) * m.Height
'调试时输出信息
'Console.WriteLine(body_Keypoint(i) & " " & lkeypoint(i).conf & " " & lkeypoint（i).p.X & "-" & lkeypoint(i).p.Y)
'置信度超过某个值才能认为是正确的结果
If lkeypoint(i).conf > 0.1 Then
CvInvoke.Circle(m, New Point(x, y), 4, New MCvScalar(255, 0, 0), -1)
End If
Next
CvInvoke.Imshow("m", m)
End Sub
'获得人体18个关键点列表，这里考虑只有一个人体的情况
Private Function getMaxPoint(ByVal inputarray(,,,) As Single) As List(Of Keypoint)
Dim lkeypoint As New List(Of Keypoint)
Dim peoplecount As Integer = 1 '考虑只有一个人体的情况，如果多个人体，请使用 inputarray.GetLength(0)
Dim modecount As Integer = 18 '只考虑18个人体关键点， inputarray.GetLength(1)
Dim dim3 As Integer = inputarray.GetLength(2) '图像高度
Dim dim4 As Integer = inputarray.GetLength(3) '图像宽度
'循环，检测到的人体个数
For i As Integer = 0 To peoplecount - 1
'循环，检测到的人体关键点
For j As Integer = 0 To modecount - 1
Dim maxvalue As Single = 0
Dim maxX As Integer = 0
Dim maxY As Integer = 0
'循环，图像高度，即对应Y坐标
For k As Integer = 0 To dim3 - 1
'循环，图像宽度，即对应X坐标
For l As Integer = 0 To dim4 - 1
'获得置信度最大的值，并获得其坐标
If maxvalue < inputarray(i, j, k, l) Then
maxvalue = inputarray(i, j, k, l)
maxX = l
maxY = k
End If
Next
Next
'添加到关键点列表
Dim kp As New Keypoint
kp.conf = maxvalue
kp.p = New Point(maxX, maxY)
lkeypoint.Add(kp)
Next
Next
Return lkeypoint
End Function
输出结果如下图所示：
图11-9 获得人体关键点
下面的代码通过人体关键点的关联，建立关键点的连线。
【代码位置：frmChapter11】Button11_Click、PointFToPoint
'关键点关联
Structure Relation
'开始关键点
Dim startpoint As Integer
'结束关键点
Dim endpoint As Integer
Sub New(ByVal startpoint As Integer, ByVal endpoint As Integer)
Me.startpoint = startpoint
Me.endpoint = endpoint
End Sub
End Structure
'获得人体关键点，并将关键点关联起来
Private Sub Button11_Click(sender As Object, e As EventArgs) Handles Button11.Click
Dim body_Keypoint() As String = {"nose", "neck", "rshoulder", "relbow", "rwrist", "lshoulder",
"lelbow", "lwrist", "rhip", "rknee", "rankle", "lhip",
"lknee", "lankle", "reye", "leye", "rear", "lear", "background"}
'18个关键点关联
Dim body_Relations As New List(Of Relation)
body_Relations.Add(New Relation(16, 14))
body_Relations.Add(New Relation(14, 0))
body_Relations.Add(New Relation(17, 15))
body_Relations.Add(New Relation(15, 0))
body_Relations.Add(New Relation(0, 1))
body_Relations.Add(New Relation(1, 2))
body_Relations.Add(New Relation(2, 3))
body_Relations.Add(New Relation(3, 4))
body_Relations.Add(New Relation(1, 5))
body_Relations.Add(New Relation(5, 6))
body_Relations.Add(New Relation(6, 7))
body_Relations.Add(New Relation(1, 8))
body_Relations.Add(New Relation(8, 9))
body_Relations.Add(New Relation(9, 10))
body_Relations.Add(New Relation(1, 11))
body_Relations.Add(New Relation(11, 12))
body_Relations.Add(New Relation(12, 13))
Dim m As New Mat("C:\learnEmgucv\action.jpg", ImreadModes.Color)
Dim net As Dnn.Net
net = DnnInvoke.ReadNetFromTensorflow("C:\learnEmgucv\openpose\graph_opt.pb")
Dim blob As Mat
blob = DnnInvoke.BlobFromImage(m, 1.0, New Drawing.Size(360, 360), New MCvScalar(127.5, 127.5, 127.5), False, False)
net.SetInput(blob)
Dim mout As New Mat
mout = net.Forward()
Dim fout(,,,) As Single
fout = mout.GetData()
Dim H As Integer = fout.GetLength(2)
Dim W As Integer = fout.GetLength(3)
Dim lkeypoint As New List(Of Keypoint)
lkeypoint = getMaxPoint(fout)
Dim x, y As Single
For i As Integer = 0 To lkeypoint.Count - 1
'按照比例获得关键点在源图像中的坐标
x = (lkeypoint(i).p.X / W) * m.Width
y = (lkeypoint(i).p.Y / H) * m.Height
'置信度超过某个值才能认为是正确的结果
If lkeypoint(i).conf > 0.1 Then
CvInvoke.Circle(m, New Point(x, y), 5, New MCvScalar(255, 0, 0), -1)
End If
Next
Dim startpoint As PointF
Dim startpointx, startpointy As Single
Dim endpoint As PointF
Dim endpointx, endpointy As Single
For Each body_Relation As Relation In body_Relations
startpointx = (lkeypoint(body_Relation.startpoint).p.X / W) * m.Width
startpointy = (lkeypoint(body_Relation.startpoint).p.Y / H) * m.Height
startpoint = New PointF(startpointx, startpointy)
endpointx = (lkeypoint(body_Relation.endpoint).p.X / W) * m.Width
endpointy = (lkeypoint(body_Relation.endpoint).p.Y / H) * m.Height
endpoint = New PointF(endpointx, endpointy)
'关键点置信度是否符合要求
If lkeypoint(body_Relation.startpoint).conf > 0.1 And lkeypoint(body_Relation.endpoint).conf > 0.1 Then
'关键点建立连线
CvInvoke.Line(m, PointFToPoint(startpoint), PointFToPoint(endpoint), New MCvScalar(0, 255, 0), 3)
End If
Next
CvInvoke.Imshow("m", m)
End Sub
'将PointF转Point方法
Public Shared Function PointFToPoint(ByVal pf As PointF) As Point
Return New Point(CInt(pf.X), CInt(pf.Y))
End Function
输出结果如下图所示：
图11-10 人体关键点连线
下面代码是在上面代码基础上，实现在视频中显示人体关键点的连线。
【代码位置：frmChapter11】Button12_Click、vc_ImageGrabbed
Dim vc As VideoCapture
Dim body_Relations As New List(Of Relation)
'将视频人物标注人体关键点和关键点连线
Private Sub Button12_Click(sender As Object, e As EventArgs) Handles Button12.Click
'18个关键点关联
body_Relations = New List(Of Relation)
body_Relations.Add(New Relation(16, 14))
body_Relations.Add(New Relation(14, 0))
body_Relations.Add(New Relation(17, 15))
body_Relations.Add(New Relation(15, 0))
body_Relations.Add(New Relation(0, 1))
body_Relations.Add(New Relation(1, 2))
body_Relations.Add(New Relation(2, 3))
body_Relations.Add(New Relation(3, 4))
body_Relations.Add(New Relation(1, 5))
body_Relations.Add(New Relation(5, 6))
body_Relations.Add(New Relation(6, 7))
body_Relations.Add(New Relation(1, 8))
body_Relations.Add(New Relation(8, 9))
body_Relations.Add(New Relation(9, 10))
body_Relations.Add(New Relation(1, 11))
body_Relations.Add(New Relation(11, 12))
body_Relations.Add(New Relation(12, 13))
vc = New VideoCapture("C:\learnEmgucv\action.mp4")
If vc.IsOpened = False Then
MessageBox.Show("打开文件失败")
Exit Sub
End If
'添加ImageGrabbed事件
AddHandler vc.ImageGrabbed, AddressOf vc_ImageGrabbed
vc.Start()
End Sub
Private Sub vc_ImageGrabbed(sender As Object, e As EventArgs)
Dim outangle As Double = 0
Dim outpix As Double = 0
Dim nextframe As New Mat
vc.Retrieve(nextframe)
If vc.Get(CapProp.PosFrames) >= vc.Get(CapProp.FrameCount) Then
vc.Stop()
vc.Dispose()
RemoveHandler vc.ImageGrabbed, AddressOf vc_ImageGrabbed
Exit Sub
End If
Dim net As Dnn.Net
net = DnnInvoke.ReadNetFromTensorflow("graph_opt.pb")
Dim blob As Mat
blob = DnnInvoke.BlobFromImage(nextframe, 1.0, New Drawing.Size(360, 360), New MCvScalar(127.5, 127.5, 127.5), True, False)
net.SetInput(blob)
Dim mout As New Mat
mout = net.Forward()
Dim fout(,,,) As Single
fout = mout.GetData()
Dim H As Integer = fout.GetLength(2)
Dim W As Integer = fout.GetLength(3)
Dim lkeypoint As New List(Of Keypoint)
lkeypoint = getMaxPoint(fout)
Dim x, y As Single
For i As Integer = 0 To lkeypoint.Count - 1
x = (lkeypoint(i).p.X / W) * nextframe.Width
y = (lkeypoint(i).p.Y / H) * nextframe.Height
If lkeypoint(i).conf > 0.1 Then
CvInvoke.Circle(nextframe, New Point(x, y), 5, New MCvScalar(0, 0, 255), -1)
End If
Next
Dim startpoint As PointF
Dim startpointx, startpointy As Single
Dim endpoint As PointF
Dim endpointx, endpointy As Single
For Each body_Relation As Relation In body_Relations
startpointx = (lkeypoint(body_Relation.startpoint).p.X / W) * nextframe.Width
startpointy = (lkeypoint(body_Relation.startpoint).p.Y / H) * nextframe.Height
startpoint = New PointF(startpointx, startpointy)
endpointx = (lkeypoint(body_Relation.endpoint).p.X / W) * nextframe.Width
endpointy = (lkeypoint(body_Relation.endpoint).p.Y / H) * nextframe.Height
endpoint = New PointF(endpointx, endpointy)
If lkeypoint(body_Relation.startpoint).conf > 0.15 And lkeypoint(body_Relation.endpoint).conf > 0.15 Then
CvInvoke.Line(nextframe, PointFToPoint(startpoint), PointFToPoint(endpoint), New MCvScalar(0, 255, 0), 4)
End If
Next
ImageBox1.Image = nextframe
Threading.Thread.Sleep(30)
End Sub
输出结果如下图所示：
图11-11 视频中使用人体关键点连线