299c554dcc
Add LL128 Protocol. Rewrite the topology detection and tree/ring creation (#179). Improve tree performance by sending/receiving from different GPUs. Add model-based tuning to switch between the different algorithms and protocols. Rework P2P/SHM detection in containers (#155, #248). Detect duplicated devices and return an error (#231). Add tuning for GCP
137 lines
4.6 KiB
Makefile
137 lines
4.6 KiB
Makefile
#
|
|
# Copyright (c) 2015-2019, NVIDIA CORPORATION. All rights reserved.
|
|
#
|
|
# See LICENSE.txt for license information
|
|
#
|
|
|
|
include ../makefiles/common.mk
|
|
include ../makefiles/version.mk
|
|
|
|
##### src files
|
|
INCEXPORTS := nccl.h nccl_net.h
|
|
LIBSRCFILES := init.cc channel.cc bootstrap.cc transport.cc enqueue.cc group.cc debug.cc \
|
|
misc/nvmlwrap.cc misc/ibvwrap.cc misc/utils.cc misc/argcheck.cc \
|
|
transport/p2p.cc transport/shm.cc transport/net.cc transport/net_socket.cc transport/net_ib.cc \
|
|
collectives/all_reduce.cc collectives/all_gather.cc collectives/broadcast.cc collectives/reduce.cc collectives/reduce_scatter.cc \
|
|
graph/topo.cc graph/paths.cc graph/search.cc graph/connect.cc graph/rings.cc graph/trees.cc graph/tuning.cc
|
|
|
|
##### lib files
|
|
LIBNAME := libnccl.so
|
|
STATICLIBNAME := libnccl_static.a
|
|
##### pkgconfig files
|
|
PKGCONFIGFILE := nccl.pc
|
|
##### dirs
|
|
BUILDDIR ?= $(abspath ../build)
|
|
INCDIR := $(BUILDDIR)/include
|
|
LIBDIR := $(BUILDDIR)/lib
|
|
OBJDIR := $(BUILDDIR)/obj
|
|
PKGDIR := $(BUILDDIR)/lib/pkgconfig
|
|
##### target files
|
|
CUDARTLIB ?= cudart_static
|
|
INCTARGETS := $(INCEXPORTS:%=$(INCDIR)/%)
|
|
LIBSONAME := $(LIBNAME:%=%.$(NCCL_MAJOR))
|
|
LIBTARGET := $(LIBNAME:%=%.$(NCCL_MAJOR).$(NCCL_MINOR).$(NCCL_PATCH))
|
|
STATICLIBTARGET := $(STATICLIBNAME)
|
|
PKGTARGET := $(PKGCONFIGFILE)
|
|
LIBOBJ := $(LIBSRCFILES:%.cc=$(OBJDIR)/%.o)
|
|
DEPFILES := $(LIBOBJ:%.o=%.d)
|
|
LDFLAGS += -L${CUDA_LIB} -l$(CUDARTLIB) -lpthread -lrt -ldl
|
|
|
|
DEVICELIB := $(BUILDDIR)/obj/collectives/device/colldevice.a
|
|
|
|
##### rules
|
|
build : lib staticlib
|
|
|
|
lib : $(INCTARGETS) $(LIBDIR)/$(LIBTARGET) $(PKGDIR)/$(PKGTARGET)
|
|
|
|
staticlib : $(LIBDIR)/$(STATICLIBTARGET)
|
|
|
|
$(DEVICELIB): ALWAYS_REBUILD
|
|
$(MAKE) -C collectives/device
|
|
|
|
# Empty target to force rebuild
|
|
ALWAYS_REBUILD:
|
|
|
|
-include $(DEPFILES)
|
|
$(LIBDIR)/$(LIBTARGET) $(LIBDIR)/$(STATICLIBTARGET) : $(LIBOBJ)
|
|
|
|
$(INCDIR)/nccl.h : nccl.h.in
|
|
# NCCL_VERSION(X,Y,Z) ((X) * 1000 + (Y) * 100 + (Z))
|
|
@$(eval NCCL_VERSION := $(shell printf "%d%d%02d" $(NCCL_MAJOR) $(NCCL_MINOR) $(NCCL_PATCH)))
|
|
mkdir -p $(INCDIR)
|
|
@printf "Generating %-35s > %s\n" $< $@
|
|
sed -e "s/\$${nccl:Major}/$(NCCL_MAJOR)/g" \
|
|
-e "s/\$${nccl:Minor}/$(NCCL_MINOR)/g" \
|
|
-e "s/\$${nccl:Patch}/$(NCCL_PATCH)/g" \
|
|
-e "s/\$${nccl:Suffix}/$(NCCL_SUFFIX)/g" \
|
|
-e "s/\$${nccl:Version}/$(NCCL_VERSION)/g" \
|
|
$< > $@
|
|
|
|
$(LIBDIR)/$(LIBTARGET): $(LIBOBJ) $(DEVICELIB)
|
|
@printf "Linking %-35s > %s\n" $(LIBTARGET) $@
|
|
mkdir -p $(LIBDIR)
|
|
$(CXX) $(CXXFLAGS) -shared -Wl,--no-as-needed -Wl,-soname,$(LIBSONAME) -o $@ $(LIBOBJ) $(DEVICELIB) $(LDFLAGS)
|
|
ln -sf $(LIBSONAME) $(LIBDIR)/$(LIBNAME)
|
|
ln -sf $(LIBTARGET) $(LIBDIR)/$(LIBSONAME)
|
|
|
|
$(LIBDIR)/$(STATICLIBTARGET): $(LIBOBJ) $(DEVICELIB)
|
|
@printf "Archiving %-35s > %s\n" $(STATICLIBTARGET) $@
|
|
mkdir -p $(LIBDIR)
|
|
$(eval TMP := $(shell mktemp -d))
|
|
cp $(LIBOBJ) $(TMP)
|
|
cd $(TMP) && ar x $(DEVICELIB) && cd -
|
|
ar cr $@ $(LIBOBJ) $(TMP)/*.o
|
|
rm -Rf $(TMP)
|
|
|
|
$(PKGDIR)/nccl.pc : nccl.pc.in
|
|
mkdir -p $(PKGDIR)
|
|
@printf "Generating %-35s > %s\n" $< $@
|
|
sed -e 's|$${nccl:Prefix}|\$(PREFIX)|g' \
|
|
-e "s/\$${nccl:Major}/$(NCCL_MAJOR)/g" \
|
|
-e "s/\$${nccl:Minor}/$(NCCL_MINOR)/g" \
|
|
-e "s/\$${nccl:Patch}/$(NCCL_PATCH)/g" \
|
|
$< > $@
|
|
|
|
$(INCDIR)/%.h : %.h
|
|
@printf "Grabbing %-35s > %s\n" $< $@
|
|
mkdir -p $(INCDIR)
|
|
install -m 644 $< $@
|
|
|
|
$(INCDIR)/nccl_%.h : include/nccl_%.h
|
|
@printf "Grabbing %-35s > %s\n" $< $@
|
|
mkdir -p $(INCDIR)
|
|
install -m 644 $< $@
|
|
|
|
$(PKGDIR)/%.pc : %.pc
|
|
@printf "Grabbing %-35s > %s\n" $< $@
|
|
mkdir -p $(PKGDIR)
|
|
install -m 644 $< $@
|
|
|
|
$(OBJDIR)/%.o : %.cc
|
|
@printf "Compiling %-35s > %s\n" $< $@
|
|
mkdir -p `dirname $@`
|
|
$(CXX) -I. -I$(INCDIR) $(CXXFLAGS) -Iinclude -c $< -o $@
|
|
@$(CXX) -I. -I$(INCDIR) $(CXXFLAGS) -Iinclude -M $< > $(@:%.o=%.d.tmp)
|
|
@sed "0,/^.*:/s//$(subst /,\/,$@):/" $(@:%.o=%.d.tmp) > $(@:%.o=%.d)
|
|
@sed -e 's/.*://' -e 's/\\$$//' < $(@:%.o=%.d.tmp) | fmt -1 | \
|
|
sed -e 's/^ *//' -e 's/$$/:/' >> $(@:%.o=%.d)
|
|
@rm -f $(@:%.o=%.d.tmp)
|
|
|
|
clean :
|
|
$(MAKE) -C collectives/device clean
|
|
rm -rf ${INCDIR} ${LIBDIR} ${PKGDIR} ${OBJDIR}
|
|
|
|
install : lib
|
|
mkdir -p $(PREFIX)/lib
|
|
mkdir -p $(PREFIX)/lib/pkgconfig
|
|
mkdir -p $(PREFIX)/include
|
|
cp -P -v $(BUILDDIR)/lib/lib* $(PREFIX)/lib/
|
|
cp -P -v $(BUILDDIR)/lib/pkgconfig/* $(PREFIX)/lib/pkgconfig/
|
|
cp -v $(BUILDDIR)/include/* $(PREFIX)/include/
|
|
|
|
FILESTOFORMAT := $(shell find . -name ".\#*" -prune -o \( -name "*.cc" -o -name "*.h" \) -print | grep -v -E 'ibvwrap.h|nvmlwrap.h|nccl.h')
|
|
# Note that formatting.mk defines a new target so in order to not overwrite the default target,
|
|
# it shouldn't be included at the top. Also, it uses the above definition of FILESTOFORMAT as well
|
|
# as the BUILDDIR variable.
|
|
include ../makefiles/formatting.mk
|