此前我们做了一个单机多卡的nccl-tests部署测试脚本，多集卡多卡的我们也需要经常测试，所以这次记录了一个自己用的多机多卡openmpi+nccl+nccl-tests编译安装脚本，注意不含测试，这个脚本最终编译完成以后会打印出多机测试命令，只需要进行简单修改然后执行即可。单机多卡的可以查看这个笔记：https://sulao.cn/post/1125

需要准备如下源码包和脚本放一起，确保多机集群做好免密，openmpi多机测试可以单向的免密，确保机器cuda库和驱动都已经安装好。

nccl-master.zip
nccl-tests-master.zip
openmpi-4.1.8.tar.gz

脚本内容如下：

#!/bin/bash
#set -xe

CURRENT_PATH=`readlink -f $(dirname $0)`

function INFO(){
    /bin/echo -e "\e[104m\e[97m[INFO]\e[49m\e[39m ${*}"
}

function WARNING(){
    /bin/echo >&2 -e "\e[101m\e[97m[WARNING]\e[49m\e[39m ${*}"
}

function ERROR(){
    /bin/echo >&2 -e "\e[101m\e[97m[ERROR]\e[49m\e[39m ${*}"
}

function CHECK_GPU_COMMAND(){
    if ! command -v "nvidia-smi" >/dev/null 2>&1; then
		ERROR "英伟达驱动未安装！"
        exit 1
    else
        export GPU_TOTAL=`nvidia-smi -L | wc -l`
    fi
}

function CHECK_CUDA_PATH(){
    if [ -d /usr/local/cuda ];then
        export CUDA_PATH=`ls -l /usr/local/cuda | awk '{print $NF}'`
    else
        ERROR "检测没有找到 cuda toolkit 目录，请先安装 cuda toolkit !"
        exit 1
    fi
}

function CHECK_GPU_SM(){
    GPU_NAME=`nvidia-smi -q -i 0 | grep 'Product Name' | awk -F ': ' '{print $2}'`
    if [[ $GPU_NAME =~ A100$ ]] || [[ $GPU_NAME =~ 'A800-SXM4-80GB'$ ]]; then
        COMPUTE_SM=80
    fi
    if [[ $GPU_NAME =~ 3090$ ]]; then
        COMPUTE_SM=86
    fi
    if [[ $GPU_NAME =~ 4090$ ]] || [[ $GPU_NAME =~ 4090D$ ]]; then
        COMPUTE_SM=89
    fi
    if [[ $GPU_NAME =~ 'H100 80GB HBM3'$ ]] || [[ $GPU_NAME =~ H800$ ]] || [[ $GPU_NAME =~ H200$ ]]|| [[ $GPU_NAME =~ H20$ ]]; then
        COMPUTE_SM=90
    fi
    if [[ $GPU_NAME =~ B200$ ]] || [[ $GPU_NAME =~ GB200$ ]]; then
        COMPUTE_SM=100
    fi
    if [[ $GPU_NAME =~ 5090$ ]] || [[ $GPU_NAME =~ 5090D$ ]]; then
        COMPUTE_SM=120
    fi
    if [ -z "${COMPUTE_SM+x}" ]; then
        ERROR "没有获取到算力，请检查！"
        exit 1
    else
		export COMPUTE_SM=${COMPUTE_SM}
    fi
}

function COMPILE_NCCL(){
    if [ ! -d ${CURRENT_PATH}/nccl-master ]; then
        WARNING "${CURRENT_PATH}/nccl-master 目录不存在，开始解压！"
        unzip ${CURRENT_PATH}/nccl-master.zip   
    fi
    cd ${CURRENT_PATH}/nccl-master
    mkdir -p ${CURRENT_PATH}/nccl
    if [ -d ${CURRENT_PATH}/nccl/lib ]; then
        INFO "检测到编译路径 ${CURRENT_PATH}/nccl/lib 存在，开始清理编译文件！"
        make clean
    fi
    make -j$(nproc) src.build BUILDDIR=${CURRENT_PATH}/nccl CUDA_HOME=${CUDA_PATH} NVCC_GENCODE="-gencode=arch=compute_${COMPUTE_SM},code=sm_${COMPUTE_SM}"
    if [ $? -eq 0 ]; then
        INFO "nccl 编译完成！"
    else
        ERROR "nccl 编译失败！"
        exit 1
    fi
	export LD_LIBRARY_PATH=${CURRENT_PATH}/nccl/lib:$LD_LIBRARY_PATH
	export PATH=${CURRENT_PATH}/nccl/bin:$PATH
}

function COMPILE_OPENMPI(){
    cd ${CURRENT_PATH}
    if [ ! -d ${CURRENT_PATH}/openmpi-4.1.8 ]; then
        WARNING "${CURRENT_PATH}/openmpi 目录不存在，开始解压！"
        tar -zxvf ${CURRENT_PATH}/openmpi-4.1.8.tar.gz
    fi
    cd ${CURRENT_PATH}/openmpi-4.1.8
    mkdir -p ${CURRENT_PATH}/openmpi
    if [ -d ${CURRENT_PATH}/openmpi/lib ]; then
        INFO "检测到编译路径 ${CURRENT_PATH}/openmpi/lib 存在，开始清理编译文件！"
        make clean
    fi
    ./configure  --prefix=${CURRENT_PATH}/openmpi --with-cuda=${CUDA_PATH}
    make -j$(nproc)
    make install
    if [ $? -eq 0 ]; then
        INFO "openmpi 编译完成！"
    else
        ERROR "openmpi 编译失败！"
        exit 1
    fi
	export LD_LIBRARY_PATH=${CURRENT_PATH}/openmpi/lib:$LD_LIBRARY_PATH
}

function COMPILE_NCCL_TESTS(){
    cd ${CURRENT_PATH}
    if [ ! -d ${CURRENT_PATH}/nccl-tests-master ]; then
        WARNING "${CURRENT_PATH}/nccl-tests-master 目录不存在，开始解压！"
        unzip ${CURRENT_PATH}/nccl-tests-master.zip
    fi
    cd ${CURRENT_PATH}/nccl-tests-master
    if [ -d ${CURRENT_PATH}/nccl-tests-master/build ]; then
        INFO "检测到编译路径 ${CURRENT_PATH}/nccl-tests-master/build 存在，开始清理编译文件！"
        make clean
    fi
    make MPI=1 MPI_HOME=${CURRENT_PATH}/openmpi CUDA_HOME=${CUDA_PATH} NCCL_HOME=${CURRENT_PATH}/nccl
    if [ $? -eq 0 ]; then
        INFO "nccl-tests 编译完成！"
    else
        ERROR "nccl-tests 编译失败！"
        exit 1
    fi  
}

apt install unzip -y
source /etc/profile
source ~/.bashrc
CHECK_GPU_COMMAND
CHECK_CUDA_PATH
CHECK_GPU_SM
COMPILE_NCCL
COMPILE_OPENMPI
COMPILE_NCCL_TESTS
INFO "请修改以下命令进行多机多卡测试，如果是ROCE网络注意修改GID！"
echo -e "export LD_LIBRARY_PATH=${CURRENT_PATH}/openmpi/lib:\$LD_LIBRARY_PATH"
echo -e "${CURRENT_PATH}/openmpi/bin/mpirun -np 16 \\
-H node1:8,node2:8 \\
--allow-run-as-root -bind-to numa -map-by slot \\
-x NCCL_DEBUG=INFO \\
-x NCCL_ALGO=Ring \\
-x NCCL_MAX_NCHANNELS=16 \\
-x NCCL_MIN_NCHANNELS=16 \\
-x NCCL_IB_HCA=mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1 \\
-x NCCL_IB_GID_INDEX=3 \\
-x NCCL_IB_DISABLE=0 \\
-x NCCL_IB_RETRY_CNT=7 \\
-x NCCL_IB_TIMEOUT=23 \\
-x NCCL_SOCKET_IFNAME=bond0 \\
-x NCCL_NET_GDR_LEVEL=2 \\
-x NCCL_IB_QPS_PER_CONNECTION=4 \\
-x NCCL_IB_TC=160 \\
-x NCCL_CHECKS_DISABLE=1 \\
-x LD_LIBRARY_PATH=${CURRENT_PATH}/openmpi/lib:${CURRENT_PATH}/nccl/lib:\$LD_LIBRARY_PATH \\
-x PATH=\$PATH \\
-mca coll_hcoll_enable 0 \\
-mca pml ob1 \\
-mca btl_tcp_if_include bond0 \\
-mca btl ^openib \\
${CURRENT_PATH}/nccl-tests-master/build/all_reduce_perf -b 1M -e 4G -f 2 -g 1"

最后打印的命令我们需要修改以下几点再进行测试

1.node1和node2测试的节点IP
2.网络接口名是bond0，有几处根据自己实际情况进行修改
3.NCCL_IB_HCA修改为实际的IB网卡名字和端口
4.如果你没绑核的话把numa改成socket

其他自动生成的无需修改，修改好的命令可以进行两两节点的测试