此前我们做了一个单机多卡的nccl-tests部署测试脚本，多集卡多卡的我们也需要经常测试，所以这次记录了一个自己用的多机多卡openmpi+nccl+nccl-tests编译安装脚本，注意不含测试，这个脚本最终编译完成以后会打印出多机测试命令，只需要进行简单修改然后执行即可。单机多卡的可以查看这个笔记：https://sulao.cn/post/1125

需要准备如下源码包和脚本放一起，确保多机集群做好免密，openmpi多机测试可以单向的免密，确保机器cuda库和驱动都已经安装好。

nccl-master.zip
nccl-tests-master.zip
openmpi-4.1.8.tar.gz

脚本内容如下：

#!/bin/bash
#set -xe

CURRENT_PATH=`readlink -f $(dirname $0)`
. ${CURRENT_PATH}/common.sh
HOST_LIST=('10.7.0.12 master' '10.7.0.13 node1' '10.7.0.14 node2' '10.7.0.15 node3')

function INFO(){
    /bin/echo -e "\e[104m\e[97m[I]\e[49m\e[39m ${*}"
}
function WARNING(){
    /bin/echo >&2 -e "\e[101m\e[97m[W]\e[49m\e[39m ${*}"
}
function ERROR(){
    /bin/echo >&2 -e "\e[101m\e[97m[E]\e[49m\e[39m ${*}"
}
function CHECK_GPU_SM(){
    GPU_NAME=`nvidia-smi -q -i 0 | grep 'Product Name' | awk -F ': ' '{print $2}'`
    INFO "当前GPU型号为：${GPU_NAME}"
    if [[ $GPU_NAME =~ A100$ ]] || [[ $GPU_NAME =~ 'A800-SXM4-80GB'$ ]]; then
        COMPUTE_SM=80
    fi
    if [[ $GPU_NAME =~ 3090$ ]]; then
        COMPUTE_SM=86
    fi
    if [[ $GPU_NAME =~ 4090$ ]] || [[ $GPU_NAME =~ 4090D$ ]]; then
        COMPUTE_SM=89
    fi
    if [[ $GPU_NAME =~ 'H100 80GB HBM3'$ ]] || [[ $GPU_NAME =~ H800$ ]] || [[ $GPU_NAME =~ H200$ ]]|| [[ $GPU_NAME =~ H20$ ]]; then
        COMPUTE_SM=90
    fi
    if [[ $GPU_NAME =~ B200$ ]] || [[ $GPU_NAME =~ GB200$ ]]; then
        COMPUTE_SM=100
    fi
    if [[ $GPU_NAME =~ 5090$ ]] || [[ $GPU_NAME =~ 5090D$ ]]; then
        COMPUTE_SM=120
    fi
    if [ -z "${COMPUTE_SM+x}" ]; then
        ERROR "没有获取到算力，请检查！"
        exit 1
    else
        export COMPUTE_SM=${COMPUTE_SM}
    fi
}
function CHECK_GPU_COMMAND(){
    if ! command -v "nvidia-smi" >/dev/null 2>&1; then
        ERROR "英伟达驱动未安装！"
        exit 1
    else
        export GPU_TOTAL=`nvidia-smi -L | wc -l`
    fi
}
function CHECK_CUDA_PATH(){
    if [ -d /usr/local/cuda ];then
        export CUDA_PATH=`ls -l /usr/local/cuda | awk '{print $NF}'`
    else
        ERROR "检测没有找到 cuda toolkit 目录，请先安装 cuda toolkit !"
        exit 1
    fi
}
function COMPILE_NCCL(){
    INFO "当前算力：${COMPUTE_SM}"
    if [ ! -d ${CURRENT_PATH}/nccl-master ]; then
        WARNING "${CURRENT_PATH}/nccl-master 目录不存在，开始解压！"
        unzip ${CURRENT_PATH}/nccl-master.zip
    fi
    cd ${CURRENT_PATH}/nccl-master
    mkdir -p ${CURRENT_PATH}/nccl
    if [ -d ${CURRENT_PATH}/nccl/lib ]; then
        INFO "检测到编译路径 ${CURRENT_PATH}/nccl/lib 存在，开始清理编译文件！"
        make clean
    fi
    INFO "开始编译 nccl..."
    make -j$(nproc) src.build BUILDDIR=${CURRENT_PATH}/nccl CUDA_HOME=${CUDA_PATH} NVCC_GENCODE="-gencode=arch=compute_${COMPUTE_SM},code=sm_${COMPUTE_SM}"
    if [ $? -eq 0 ]; then
        INFO "nccl 编译完成！"
    else
        ERROR "nccl 编译失败！"
        exit 1
    fi
    export LD_LIBRARY_PATH=${CURRENT_PATH}/nccl/lib:$LD_LIBRARY_PATH
    export PATH=${CURRENT_PATH}/nccl/bin:$PATH
}
function COMPILE_OPENMPI(){
    cd ${CURRENT_PATH}
    if [ ! -d ${CURRENT_PATH}/openmpi-4.1.8 ]; then
        WARNING "${CURRENT_PATH}/openmpi 目录不存在，开始解压！"
        tar -zxvf ${CURRENT_PATH}/openmpi-4.1.8.tar.gz
    fi
    cd ${CURRENT_PATH}/openmpi-4.1.8
    mkdir -p ${CURRENT_PATH}/openmpi
    if [ -d ${CURRENT_PATH}/openmpi/lib ]; then
        INFO "检测到编译路径 ${CURRENT_PATH}/openmpi/lib 存在，开始清理编译文件！"
        make clean
    fi
    INFO "开始配置 openmpi..."
    ./configure  --prefix=${CURRENT_PATH}/openmpi --with-cuda=${CUDA_PATH}
    INFO "开始编译 openmpi..."
    make -j$(nproc)
    INFO "开始安装 openmpi..."
    make install
    if [ $? -eq 0 ]; then
        INFO "openmpi 编译完成！"
    else
        ERROR "openmpi 编译失败！"
        exit 1
    fi
    export LD_LIBRARY_PATH=${CURRENT_PATH}/openmpi/lib:$LD_LIBRARY_PATH
}
function COMPILE_NCCL_TESTS(){
    cd ${CURRENT_PATH}
    if [ ! -d ${CURRENT_PATH}/nccl-tests-master ]; then
        WARNING "${CURRENT_PATH}/nccl-tests-master 目录不存在，开始解压！"
        unzip ${CURRENT_PATH}/nccl-tests-master.zip
    fi
    cd ${CURRENT_PATH}/nccl-tests-master
    if [ -d ${CURRENT_PATH}/nccl-tests-master/build ]; then
        INFO "检测到编译路径 ${CURRENT_PATH}/nccl-tests-master/build 存在，开始清理编译文件！"
        make clean
    fi
    INFO "开始编译 nccl-tests..."
    make MPI=1 MPI_HOME=${CURRENT_PATH}/openmpi CUDA_HOME=${CUDA_PATH} NCCL_HOME=${CURRENT_PATH}/nccl
    if [ $? -eq 0 ]; then
        INFO "nccl-tests 编译完成！"
    else
        ERROR "nccl-tests 编译失败！"
        exit 1
    fi
}
function GENERATE_TEST_COMMAND(){
    NET_DEVICE=`ifconfig | grep -B 1 ${HOST_LIST[0]%% *} | head -1 | awk -F ':' '{print $1}'`
    PROCESS_NUM=$((${#HOST_LIST[@]}*${GPU_TOTAL}))
    INFO "请修改以下命令进行多机多卡测试，如果是ROCE网络注意修改GID！"
    SLOT=""
    for HOST in "${HOST_LIST[@]}"
    do
        SLOT="${SLOT},${HOST%% *}:${GPU_TOTAL}"
    done
    RATE=`ibstatus | grep rate | awk '{print $2}' | uniq -c | sort -k2 | head -1`
    MLN_LIST=`ibstatus | grep -E -B 6 "${RATE##* }\s+Gb/sec" | grep device | awk '{print $3":"$5}' | sed ':a;N;$!ba;s/\n/,/g'`
    echo -e "${CURRENT_PATH}/openmpi/bin/mpirun -np ${PROCESS_NUM} \\
    -H ${SLOT:1} \\
    --allow-run-as-root -bind-to numa -map-by slot \\
    -x NCCL_DEBUG=INFO \\
    -x NCCL_ALGO=Ring \\
    -x NCCL_MAX_NCHANNELS=16 \\
    -x NCCL_MIN_NCHANNELS=16 \\
    -x NCCL_IB_HCA=${MLN_LIST//\'/} \\
    -x NCCL_IB_GID_INDEX=3 \\
    -x NCCL_IB_DISABLE=0 \\
    -x NCCL_IB_RETRY_CNT=7 \\
    -x NCCL_IB_TIMEOUT=23 \\
    -x NCCL_SOCKET_IFNAME=${NET_DEVICE} \\
    -x NCCL_NET_GDR_LEVEL=2 \\
    -x NCCL_IB_QPS_PER_CONNECTION=4 \\
    -x NCCL_IB_TC=160 \\
    -x NCCL_CHECKS_DISABLE=1 \\
    -x LD_LIBRARY_PATH=${CURRENT_PATH}/openmpi/lib:${CURRENT_PATH}/nccl/lib:\$LD_LIBRARY_PATH \\
    -x PATH=\$PATH \\
    -mca coll_hcoll_enable 0 \\
    -mca pml ob1 \\
    -mca btl_tcp_if_include ${NET_DEVICE} \\
    -mca btl ^openib \\
    ${CURRENT_PATH}/nccl-tests-master/build/all_reduce_perf -b 1M -e 8G -f 2 -g 1"
}

CHECK_GPU_COMMAND
CHECK_CUDA_PATH
CHECK_GPU_SM
COMPILE_NCCL
COMPILE_OPENMPI
COMPILE_NCCL_TESTS
GENERATE_TEST_COMMAND

上述脚本执行完成以后会自动生成多机多卡nccl-tests测试命令，我们只需要核对命令，然后进行执行即可，注意每个节点都需要执行此脚本进行环境的编译配置。