### Makefile --- Makefile for ECCcode on GPU

## Author: hoang-trong@minhtuan
## Version: $Id: Makefile,v 1.6 2011/07/16 00:48:02 minhtuan Exp minhtuan $
## Keywords: 
## X-URL: 

#PROG=./execs/ecc_gpu_mtcpu
#PROG=./execs/ecc_gpu_I_tf
#PROG=./execs/ecc_gpu_V_tf
PROG=ecc_gpu

#PGI_PATH=/opt/pgi/linux86-64/11.10
#PGI_PATH=/opt/pgi/linux86-64/12.5
#PGI_PATH=/opt/pgi/linux86-64/12.6
#CUDA_VER=4.1

PGI_PATH=/opt/pgi/linux86-64/13.9
#PGI_PATH=/opt/pgi/linux86-64/13.8
CUDA_VER=4.2
#CUDA_VER=5.5
# Tesla, Fermi, Kepler (MAKE SURE NO TRAILING SPACE)
GPU ?= Fermi
LIB=$(PGI_PATH)/lib
#LIB=$(PGI_PATH)/../2013/acml/5.3.0/lib/
F77=$(PGI_PATH)/bin/pgfortran
F90=$(PGI_PATH)/bin/pgfortran
FC=$(PGI_PATH)/bin/pgfortran
ORDERPACK=orderpack-2.0
# For FreeGLUT library:
GLUTDIR=/usr/lib64


ifeq ($(GPU), Tesla) 
  FFLAGS=-Rlibrary_Tesla/ -Llibrary_Tesla/ -Mcuda=cc13,cuda${CUDA_VER}
else ifeq ($(GPU), Fermi)
# FFLAGS=-Rlibrary_Fermi/ -Llibrary_Fermi/ -Mcuda=cc13,cuda${CUDA_VER}
 FFLAGS=-Rlibrary_Fermi/ -Llibrary_Fermi/ -Mcuda=cc20,cuda${CUDA_VER},ptxinfo,maxregcount:32
else ifeq ($(GPU), Kepler)
 FFLAGS=-Rlibrary_Kepler/ -Llibrary_Kepler/ -Mcuda=cc30,cuda${CUDA_VER},ptxinfo,maxregcount:32
#use 32 if 4 active thread blocks/SM
#use 42 if 3 active thread blocks/SM
endif

##NOTE: Temporary disable optimization 
F_OPTIMIZATION = -g #-O2 -fastsse #-Mipa=fast,inline
FFLAGS+=-L/usr/local/szip/lib \
	-L/usr/local/hdf5/ver_1.8.5/lib \
	-lhdf5 -lhdf5_hl -lhdf5_fortran -lsz -lz -lm \
	-I/usr/local/hdf5/ver_1.8.5/include/ 

FFLAGS+= -L$(LIB) -lacml -lreduction -lsaru -lmycudart
ifeq ($(CUDA_VER),5.0)
FFLAGS+= -L${GLUTDIR} -L/usr/local/cuda$(CUDA_VER)/samples/common/lib/linux/x86_64/ -lGL -lglut -lGLU -lGLEW
else
FFLAGS+= -L${GLUTDIR} -L/usr/local/NVIDIA_GPU_COMPUTING_$(CUDA_VER)/C/common/lib/linux/ -lGL -lglut -lGLU -lGLEW_x86_64
endif
#FFLAGS+= -Munixlogical -Mpreprocess -Mbounds  -Mdclchk  -Mextend  #-Mlarge_arrays
FFLAGS+= -Mpreprocess -Mbounds  -Mdclchk  -Mextend  #-Mlarge_arrays 
FFLAGS+= ${F_OPTIMIZATION} -fPIC

SDIR=src
ODIR=objs
VPATH= $(ODIR)


_FSRCS=makeKs_utility.f95 matrix_utility.f95 math_utility.f95 \
	precisioninfo.f95 ecc_gpu_utility.f95 \
	m_inspar.f95 error_codes.f95 \
        data_global.f95 data_ecc.f95  \
	data_ecc_gpu.f95 \
	rand_utility.f95 hdf5_utility.f95 \
	cuda_interop.f95
#	iso_c_utilities.f95 cudpp.f95 #mtgp64.f95 type_mod.f95 

_FOBJS=$(_FSRCS:.f95=.o)
_FMODS=$(_FSRCS:.f95=.mod)

.SUFFICES:.o .mod .f95

FSRCS = $(addprefix $(SDIR)/, $(_FSRCS))
#FSRCS = $(patsubst %, $(SDIR)/, $(_FSRCS))
#FOBJS = $(patsubst %,$(ODIR)/%,$(_FOBJS))
FOBJS = $(addprefix $(ODIR)/, $(_FOBJS))

FMODS = $(patsubst %,$(ODIR)/%,$(_FMODS))
#FFLAGS+= -module ${ODIR}

COMPILE_CMD=$(FC) -c $< -o $@ $(FFLAGS)

start: $(SDIR)/main.f95 $(FOBJS) buildfreeglut
	$(FC)  -o $(PROG) $< $(FSRCS) $(MODULES) $(FFLAGS) 

$(ODIR)/makeKs_utility.o: $(SDIR)/makeKs_utility.f95 matrix_utility.o math_utility.o  $(ODIR)/m_inspar.o  error_codes.o
	$(COMPILE_CMD)

$(ODIR)/matrix_utility.o: $(SDIR)/matrix_utility.f95 $(ODIR)/precisioninfo.o
	$(COMPILE_CMD)

$(ODIR)/math_utility.o: $(SDIR)/math_utility.f95 matrix_utility.o $(ODIR)/precisioninfo.o
	$(COMPILE_CMD)

$(ODIR)/data_global.o: $(SDIR)/data_global.f95 $(ODIR)/precisioninfo.o error_codes.o cuda_interop.o makeKs_utility.o
	$(COMPILE_CMD)

$(ODIR)/cuda_interop.o: $(SDIR)/cuda_interop.f95 opengl_kinds.o $(ODIR)/precisioninfo.o
	$(COMPILE_CMD)

# mtgp64.o: mtgp64.f95 type_mod.o
# 	$(FC) -c $< $(FFLAGS)

$(ODIR)/data_ecc.o: $(SDIR)/data_ecc.f95 $(ODIR)/precisioninfo.o math_utility.o
	$(COMPILE_CMD)

$(ODIR)/data_ecc_gpu.o: $(SDIR)/data_ecc_gpu.f95 $(ODIR)/precisioninfo.o
	$(COMPILE_CMD)

$(ODIR)/rand_utility.o: $(SDIR)/rand_utility.f95 $(ODIR)/precisioninfo.o
	$(COMPILE_CMD)

$(ODIR)/ecc_gpu_utility.o: $(SDIR)/ecc_gpu_utility.f95 $(ODIR)/cudpp.o \
	$(ODIR)/data_global.o $(ODIR)/data_ecc_gpu.o $(ODIR)/data_ecc.o \
	$(ODIR)/rand_utility.o  $(ODIR)/hdf5_utility.o #mtgp64.o type_mod.o
	$(COMPILE_CMD)

$(ODIR)/%.o: $(SDIR)/%.f95
	$(COMPILE_CMD)
%.o:%.mod


CUDAPATH=/usr/local/cuda$(CUDA_VER)
#CUDAPATH=/usr/local/cuda-$(CUDA_VER)
ifeq ($(CUDA_VER),5.0)
CUDASDK=/usr/local/cuda5.0/samples/common/inc/
else
CUDASDK=/usr/local/NVIDIA_GPU_COMPUTING_$(CUDA_VER)/C
endif
NVCC = $(CUDAPATH)/bin/nvcc
CUDALINK = -I$(CUDAPATH)/include -I$(CUDASDK)/common/inc -lcuda -lcudart  --compiler-options '-fPIC'
#--ptxas-options=-v
ifeq ($(GPU), Tesla) 
  CUDALINK += -arch=compute_13 -code=sm_20 
  #CUDALINK += -arch=compute_13 -code=sm_35
else ifeq ($(GPU), Fermi)
  CUDALINK += -arch=compute_20 -code=sm_20 -Xptxas -dlcm=cg
else ifeq ($(GPU), Kepler)
  CUDALINK += -arch=compute_30 -code=sm_30 -Xptxas -dlcm=cg
endif

# CXX = g++ or gcc
OPTI = -O3 -msse3 -finline-functions -fomit-frame-pointer -fno-strict-aliasing
CXX = gcc 
CXXFLAGS=-I$(CUDAPATH)/include -I$(CUDASDK)/common/inc/ -Wall -Wextra  -fPIC #-lstdc++
LDFLAGS=-L$(CUDAPATH)/lib64  -lcuda -lcudart

_CSRCS = reduction.cpp cudart_utility.cpp
_KERNELS=reduction_kernel.cu saruprngCUDA.cu

CSRCS = $(addprefix $(SDIR)/, $(_CSRCS))
KERNELS=$(addprefix $(SDIR)/, $(_KERNELS))

COBJS=$(_CSRCS:.cpp=.o) $(_KERNELS:.cu=.o)  

sharelib: $(COBJS)
	$(CXX) $(LDFLAGS) -m64 -shared -fPIC -o libreduction.so $(ODIR)/reduction.o $(ODIR)/reduction_kernel.o
	$(CXX) $(LDFLAGS) -m64 -shared -fPIC -o libsaru.so $(ODIR)/saruprngCUDA.o
	$(CXX) $(LDFLAGS) -m64 -shared -fPIC -o libmycudart.so $(ODIR)/cudart_utility.o
#	$(CXX) $(LDFLAGS) -m64 -shared -fPIC -o libreduction.so reduction.o reduction_kernel.o
#	$(CXX) $(LDFLAGS) -m64 -shared -fPIC -o libsaru.so saruprngCUDA.o
	@echo a$(GPU)b
	@echo a$(CUDA_VER)b
ifeq ($(GPU), Tesla) 
	-mkdir -p library_Tesla
	-mv libreduction.so library_Tesla/
else ifeq ($(GPU), Fermi)
	-mkdir -p library_Fermi
	-rm library_Fermi/*
	-mv libreduction.so library_Fermi/
	-mv libsaru.so library_Fermi/
	-mv libmycudart.so library_Fermi/
else ifeq ($(GPU), Kepler)
	-mkdir -p library_Kepler
	-mv libreduction.so library_Kepler/
	-mv libsaru.so library_Kepler/
	-mv libmycudart.so library_Kepler/
endif

$(ODIR)/reduction.o: $(SDIR)/reduction.cpp reduction_kernel.o
	${CXX} $(CXXFLAGS) -lcuda -lcudart  -c $<  -o $@
$(ODIR)/reduction_kernel.o: $(SDIR)/reduction_kernel.cu
	$(NVCC)  -c $< $(CUDALINK) -o $@

$(ODIR)/saruprngCUDA.o: $(SDIR)/saruprngCUDA.cu
	$(NVCC)  -c $< $(CUDALINK) -o $@

$(ODIR)/cudart_utility.o: $(SDIR)/cudart_utility.cpp
	${CXX} $(CXXFLAGS) -lcuda -lcudart  -c $<  -o $@

%.o : %.cpp
	${CXX} ${CXXLAGS}  -c $<

help:
	@echo "make sharelib"
	@echo " to make the lib"
	@echo "make amd=y"
	@echo " to compile on NFAT, NFKB; otherwise"
	@echo "make"
	@echo " to compile on CARDIAC/LEAK"

clean: tidy
	-rm $(PROG) *.mod *.o *.h5 *.8 *.so out $(ODIR)/*.mod  $(ODIR)/*.o
	-@make -C common clean
tidy:
	-@find * | egrep "#" | xargs rm -f
	-@find * | egrep "\~" | xargs rm -f

cleanall: clean tidy
	-rm *.bin *.ptx *.oo

#libcutil.so:
#	@make -C common
#precisioninfo.mod: precisioninfo.f95
#	$(FC)  -c $<  $(FFLAGS)	
#makeKs.mod: matrix_utility.mod math_utility.mod
#	${CC} -o mtgp64.so mtgp64_wrapper.cu mtgp64-param-fast.o mtgp64-fast.o \
#	 ${CUDALINK} -shared

testreduce: testreduce.f95 cudpp.mod
	$(FC)  -o testreduce $<  cudpp.f95 $(FFLAGS)

test_main: test_code.f95 
	$(FC)  -o test_main $<  $(FFLAGS)

#  Comment out for 32-bit architecture
BITS      := 64

#  The directories containing the OpenGL libraries and GLUT libraries
OGLLIBDIR := -L/usr/lib${BITS}


#  Libraries for OpenGL, including GLUT, GLU and OpenGL
F90GLUTLIB := -lglut -lGL -lGLU

#  The X11 libraries
X11LIB := -lXaw -lXt -lXmu -lXi -lXext -lX11 -lm -lXxf86vm

LIBRARIES := ${OGLLIBDIR}  -L/usr/local/lib${BITS}
LIBS      := ${F90GLUTLIB} ${X11LIB} -lpthread -ldl

LIBRARIES := ${OGLLIBDIR}  -L/usr/local/lib
_MODULES    =  opengl_kinds.o
MODULES = $(addprefix $(ODIR)/, $(_MODULES))

buildfreeglut:  ${MODULES}  force

#$(ODIR)/opengl_glew.o $(ODIR)/opengl_freeglut.o $(ODIR)/opengl_glu.o:
#opengl_kinds.o opengl_gl.o
opengl_glew.o opengl_freeglut.o opengl_glu.o: opengl_kinds.o opengl_gl.o

$(ODIR)/opengl%.o: $(SDIR)/opengl%.f95 
	${FC}  -c $< -o $@ $(FFLAGS)


$(ODIR)/GLUT_fonts.o: $(SDIR)/GLUT_fonts.c
	gcc  -c GLUT_fonts.c -o $@

force:

etags:
	cd $(SDIR); find . -name '*.f95' -exec basename {} \; | xargs etags -l fortran -r '/[ \t]*attributes([a-zA-Z]*)[ \t]+subroutine[ \t]+[a-zA-Z0-9_]+/im' -r '/[ \t]*attributes([a-zA-Z]*)[ \t]+function[ \t]+[a-zA-Z0-9_]+/im'
