HPL 源码结构分析
文件夹结构：
$ cd /home/hipper/ex_hpl_hpcg/
$ pwd
$ mkdir ./openmpi
$mkdir ./openblas
$mkdir ./hpl
$ tree
1. 安装openmpi
1.1.1 使用Makefile下载配置编译安装 openmpi
Makefile：
all:wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.6.tar.gz && \tar zxf openmpi-4.1.6.tar.gz && \cd openmpi-4.1.6/ && \./configure --enable-mpi-cxx --with-cuda --prefix=${PWD}/local/ 2>&1 | tee config.out && \make -j all 2>&1 | tee make.outinstall:make -C openmpi-4.1.6/ install 2>&1 | tee install.out.PHONY: clean
clean:-rm -rf ./openmpi-4.1.6/ ./openmpi-4.1.6.tar.gz$ make
$ make install$ ls
1.1.2 注意事项
如果 是 openmpi 5.0及其以上，则./configure 时不能要选项：
--enable-mpi-cxx
mpi 以后仅支持使用相对广泛很多的 C 语言风格的 API，不再支持 C++ 风格的使用方式，虽然有在用，但很不主流，C 风格 api 足够强大，足够使用。
无论4.x还是5.x，如果没有安装cuda，则 ./configure 时不需要该项：
--with-cuda
而，指定安装目录时需要指定绝对路径，例如：
--prefix=${PWD}/local/
或者不使用 Makefile 方式，而使用命令行安装 openmpi：
$ wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.6.tar.g
$ tar zxf openmpi-4.1.6.tar.gz
$ cd openmpi-4.1.6/其中 configure 选项 --prefix=/.../ 需要使用绝对路径，例如：
$ ./configure --enable-mpi-cxx --with-cuda --prefix=/home/hipper/ex_hpl/openmpi/local/ 2>&1 | tee config.out
$ make -j all 2>&1 | tee make.out
$ make install 2>&1 | tee install.out1.1.3 配置环境变量
export PATH=/home/hipper/ex_openmpi/local/bin:$PATHexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/hipper/ex_openmpi/local/libcd examplesmakempirun -np 7 hello_c2. 安装 openblas
all:wget https://github.com/OpenMathLib/OpenBLAS/archive/refs/tags/v0.3.27.tar.gztar zxf v0.3.27.tar.gz make -C OpenBLAS-0.3.27 FC=gfortran -jinstall:make -C OpenBLAS-0.3.27 install PREFIX=../local/
#PREFIX=/opt/OpenBLAS
# PREFIX=../local/clean:-rm -rf ./local/ ./OpenBLAS-0.3.27/ ./v0.3.27.tar.gz$ make
$ make install
$ ls
3. 配置编译运行 hpl
1.3.1 下载 hpl 源代码
$ wget https://www.netlib.org/benchmark/hpl/hpl-2.3.tar.gz
$ tar zxf hpl-2.3.tar.gz
$ cd hpl-2.3/1.3.2 配置
$ cd hpl-2.3/
$ hpl-2.3$ cp ./setup/Make.Linux_PII_CBLAS ./vim ./Make.Linux_PII_CBLAS
# ######################################################################
#
# ----------------------------------------------------------------------
# - shell --------------------------------------------------------------
# ----------------------------------------------------------------------
#
SHELL = /bin/sh
#
CD = cd
CP = cp
LN_S = ln -s
MKDIR = mkdir
RM = /bin/rm -f
TOUCH = touch
#
# ----------------------------------------------------------------------
# - Platform identifier ------------------------------------------------
# ----------------------------------------------------------------------
#
ARCH = Linux_PII_CBLAS
#
# ----------------------------------------------------------------------
# - HPL Directory Structure / HPL library ------------------------------
# ----------------------------------------------------------------------
#
#/home/hipper/ex_hpl_hpcg/hpl/hpl-2.3
TOPdir = $(HOME)/ex_hpl_hpcg/hpl/hpl-2.3
INCdir = $(TOPdir)/include
BINdir = $(TOPdir)/bin/$(ARCH)
LIBdir = $(TOPdir)/lib/$(ARCH)
#
HPLlib = $(LIBdir)/libhpl.a
#
# ----------------------------------------------------------------------
# - Message Passing library (MPI) --------------------------------------
# ----------------------------------------------------------------------
# MPinc tells the C compiler where to find the Message Passing library
# header files, MPlib is defined to be the name of the library to be
# used. The variable MPdir is only used for defining MPinc and MPlib.
#
#MPdir = /usr/local/mpi
MPdir = $(HOME)/ex_hpl_hpcg/openmpi/local
MPinc = -I$(MPdir)/include
MPlib = $(MPdir)/lib/libmpi.so
#
# ----------------------------------------------------------------------
# - Linear Algebra library (BLAS or VSIPL) -----------------------------
# ----------------------------------------------------------------------
# LAinc tells the C compiler where to find the Linear Algebra library
# header files, LAlib is defined to be the name of the library to be
# used. The variable LAdir is only used for defining LAinc and LAlib.
#
#LAdir = $(HOME)/netlib/ARCHIVES/Linux_PII
LAdir = $(HOME)/ex_hpl_hpcg/openblas/local
LAinc =
LAlib = $(LAdir)/lib/libopenblas.a
#
# ----------------------------------------------------------------------
# - F77 / C interface --------------------------------------------------
# ----------------------------------------------------------------------
# You can skip this section if and only if you are not planning to use
# a BLAS library featuring a Fortran 77 interface. Otherwise, it is
# necessary to fill out the F2CDEFS variable with the appropriate
# options. **One and only one** option should be chosen in **each** of
# the 3 following categories:
#
# 1) name space (How C calls a Fortran 77 routine)
#
# -DAdd_ : all lower case and a suffixed underscore (Suns,
# Intel, ...), [default]
# -DNoChange : all lower case (IBM RS6000),
# -DUpCase : all upper case (Cray),
# -DAdd__ : the FORTRAN compiler in use is f2c.
#
# 2) C and Fortran 77 integer mapping
#
# -DF77_INTEGER=int : Fortran 77 INTEGER is a C int, [default]
# -DF77_INTEGER=long : Fortran 77 INTEGER is a C long,
# -DF77_INTEGER=short : Fortran 77 INTEGER is a C short.
#
# 3) Fortran 77 string handling
#
# -DStringSunStyle : The string address is passed at the string loca-
# tion on the stack, and the string length is then
# passed as an F77_INTEGER after all explicit
# stack arguments, [default]
# -DStringStructPtr : The address of a structure is passed by a
# Fortran 77 string, and the structure is of the
# form: struct {char *cp; F77_INTEGER len;},
# -DStringStructVal : A structure is passed by value for each Fortran
# 77 string, and the structure is of the form:
# struct {char *cp; F77_INTEGER len;},
# -DStringCrayStyle : Special option for Cray machines, which uses
# Cray fcd (fortran character descriptor) for
# interoperation.
#
F2CDEFS =
#
# ----------------------------------------------------------------------
# - HPL includes / libraries / specifics -------------------------------
# ----------------------------------------------------------------------
#
HPL_INCLUDES = -I$(INCdir) -I$(INCdir)/$(ARCH) $(LAinc) $(MPinc) -lpthread
HPL_LIBS = $(HPLlib) $(LAlib) $(MPlib) -lpthread
#
# - Compile time options -----------------------------------------------
#
# -DHPL_COPY_L force the copy of the panel L before bcast;
# -DHPL_CALL_CBLAS call the cblas interface;
# -DHPL_CALL_VSIPL call the vsip library;
# -DHPL_DETAILED_TIMING enable detailed timers;
#
# By default HPL will:
# *) not copy L before broadcast,
# *) call the BLAS Fortran 77 interface,
# *) not display detailed timing information.
#
HPL_OPTS = -DHPL_CALL_CBLAS
#
# ----------------------------------------------------------------------
#
HPL_DEFS = $(F2CDEFS) $(HPL_OPTS) $(HPL_INCLUDES)
#
# ----------------------------------------------------------------------
# - Compilers / linkers - Optimization flags ---------------------------
# ----------------------------------------------------------------------
#
CC = $(HOME)/ex_hpl_hpcg/openmpi/local/bin/mpicc
CCNOOPT = $(HPL_DEFS)
CCFLAGS = $(HPL_DEFS) -fomit-frame-pointer -O3 -funroll-loops
#
# On some platforms, it is necessary to use the Fortran linker to find
# the Fortran internals used in the BLAS library.
#
LINKER = $(HOME)/ex_hpl_hpcg/openmpi/local/bin/mpif77
LINKFLAGS = $(CCFLAGS)
#
ARCHIVER = ar
ARFLAGS = r
RANLIB = echo
#
# ----------------------------------------------------------------------前后区别：
$ diff Make.Linux_PII_CBLAS setup/Make.Linux_PII_CBLAS
hipper@hipper-G21:~/ex_hpl_hpcg/hpl/hpl-2.3$ diff Make.Linux_PII_CBLAS setup/Make.Linux_PII_CBLAS
70,71c70
< #/home/hipper/ex_hpl_hpcg/hpl/hpl-2.3
< TOPdir = $(HOME)/ex_hpl_hpcg/hpl/hpl-2.3
---
> TOPdir = $(HOME)/hpl
85,86c84
< #MPdir = /usr/local/mpi
< MPdir = /home/hipper/ex_hpl_hpcg/openmpi/local
---
> MPdir = /usr/local/mpi
88c86
< MPlib = $(MPdir)/lib/libmpi.so
---
> MPlib = $(MPdir)/lib/libmpich.a
97,98c95
< #LAdir = $(HOME)/netlib/ARCHIVES/Linux_PII
< LAdir = $(HOME)/ex_hpl_hpcg/openblas/local
---
> LAdir = $(HOME)/netlib/ARCHIVES/Linux_PII
100c97
< LAlib = $(LAdir)/lib/libopenblas.a
---
> LAlib = $(LAdir)/libcblas.a $(LAdir)/libatlas.a
147,148c144,145
< HPL_INCLUDES = -I$(INCdir) -I$(INCdir)/$(ARCH) $(LAinc) $(MPinc) -lpthread
< HPL_LIBS = $(HPLlib) $(LAlib) $(MPlib) -lpthread
---
> HPL_INCLUDES = -I$(INCdir) -I$(INCdir)/$(ARCH) $(LAinc) $(MPinc)
> HPL_LIBS = $(HPLlib) $(LAlib) $(MPlib)
172c169
< CC = $(HOME)/ex_hpl_hpcg/openmpi/local/bin/mpicc
---
> CC = /usr/bin/gcc
179c176
< LINKER = $(HOME)/ex_hpl_hpcg/openmpi/local/bin/mpif77
---
> LINKER = /usr/bin/g77
hipper@hipper-G21:~/ex_hpl_hpcg/hpl/hpl-2.3$1.3.3编译运行
make arch=Linux_PII_CBLAS$ ls
单节点运行：
$ /home/hipper/ex_hpl_hpcg/openmpi/local/bin/mpirun -np 4 ./xhpl > HPL-Benchmark.txt效果： 多节点： mpirun -np 8 --host node1,node2 ./xhpl > HPL-Benchmark.txt
https://www.advancedclustering.com/act_kb/tune-hpl-dat-file
mpirun 的参数 np 值需要跟 HPL.dat 的 (P x Q) 相等。
2，hpl 源码框架分析
先保存备忘，待续。。。。