yezhengmao/nccl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix intermittent 11.6 builds: generate unique .cu file for each object file

Browse Source
This commit is contained in:
Jane Xu 2022-09-13 16:05:21 -04:00 committed by Sylvain Jeaugey
parent c4e2aa6c79
commit 79fb0326ac
2 changed files with 13 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/collectives/device/Makefile
View File

@ -29,7 +29,7 @@ all: $(STATICLIB)
all_deps: $(DEPENDFILES) all_deps: $(DEPENDFILES)
# Auto-generating the rules per op/reduction/datatype/algorithm # Auto-generating the rules per op/reduction/datatype/algorithm
$(RULESFILE) : $(RULESFILE) : gen_rules.sh
@printf "Generating %-35s > %s\n" rules $@ @printf "Generating %-35s > %s\n" rules $@
@mkdir -p $(OBJDIR) @mkdir -p $(OBJDIR)
@CUDA_MAJOR=${CUDA_MAJOR} CUDA_MINOR=${CUDA_MINOR} ./gen_rules.sh $(OBJDIR) > $@ @CUDA_MAJOR=${CUDA_MAJOR} CUDA_MINOR=${CUDA_MINOR} ./gen_rules.sh $(OBJDIR) > $@

14
src/collectives/device/gen_rules.sh
View File

@ -13,6 +13,9 @@ then
datatypes+=" bf16" datatypes+=" bf16"
fi fi
echo "CURDIR := \$(dir \$(realpath \$(word \$(words \$(math\$(MAKEFILE_LIST))-1), \$(MAKEFILE_LIST))))"
+echo ""
targets="GENOBJS := \\\\\n" targets="GENOBJS := \\\\\n"
for base in sendrecv all_reduce all_gather broadcast reduce reduce_scatter; do for base in sendrecv all_reduce all_gather broadcast reduce reduce_scatter; do
@ -21,10 +24,17 @@ for base in sendrecv all_reduce all_gather broadcast reduce reduce_scatter; do
dtn=0 dtn=0
# Order must match that of the ncclDataType_t enum # Order must match that of the ncclDataType_t enum
for dt in ${datatypes}; do for dt in ${datatypes}; do
echo "${dir}/${base}_${op}_${dt}.o : ${base}.cu ${dir}/${base}.dep" # Generate a unique filename for each compilation unit,
# otherwise the __nv_module_id may conflict at link time
echo "${dir}/${base}_${opn}_${dtn}.cu :"
echo " echo \"#include \\\"\$(CURDIR)${base}.cu\\\"\" > \$@"
echo ""
# Compile the file
echo "${dir}/${base}_${op}_${dt}.o : ${dir}/${base}_${opn}_${dtn}.cu ${base}.cu ${dir}/${base}.dep"
echo " @printf \"Compiling %-35s > %s\\\\n\" ${base}.cu ${dir}/${base}_${op}_${dt}.o" echo " @printf \"Compiling %-35s > %s\\\\n\" ${base}.cu ${dir}/${base}_${op}_${dt}.o"
echo " mkdir -p ${dir}" echo " mkdir -p ${dir}"
echo " \${NVCC} -DNCCL_OP=${opn} -DNCCL_TYPE=${dtn} \${NVCUFLAGS} -dc ${base}.cu -o ${dir}/${base}_${op}_${dt}.o" echo " \${NVCC} -DNCCL_OP=${opn} -DNCCL_TYPE=${dtn} \${NVCUFLAGS} -dc \$< -o \$@"
echo "" echo ""
targets="$targets\t${dir}/${base}_${op}_${dt}.o \\\\\n" targets="$targets\t${dir}/${base}_${op}_${dt}.o \\\\\n"
dtn=$(($dtn + 1)) dtn=$(($dtn + 1))