Makefile

#  MIMD version 7
#  Standard application makefile.
#  Replaces Make_vanilla, Make_linux_mpi, Make_linux_qdp, Make_qcdoc_gcc
#  Do not use for making libraries
#  Copy this file into the application directory and run make there.
#

# This file
MAKEFILE = Makefile

#----------------------------------------------------------------------
#  User choices - edit to suit 
#----------------------------------------------------------------------
# 1. Machine architecture.  Controls optimization flags here and in libraries.
#    Can control BINEXT below, a suffix appended to the name of the executable.

ARCH = # knl knc hsw

#----------------------------------------------------------------------
# 2. Compiler family

COMPILER = gnu # intel, ibm, portland, cray-intel

#----------------------------------------------------------------------
# 3. MPP vs Scalar

# Compiling for a parallel machine?  blank for a scalar machine
MPP = #true

#----------------------------------------------------------------------
# 4. Precision 

# 1 = single precision; 2 = double
PRECISION = 1

#----------------------------------------------------------------------
# 5. Compiler
# Choices include mpicc cc gcc pgcc g++

ifeq ($(strip ${COMPILER}),intel)

  ifeq ($(strip ${MPP}),true)
    CC = mpiicc
    CXX = mpiicpc
  else
    CC  = icc
    CXX = icpc
  endif

else ifeq ($(strip ${COMPILER}),cray-intel)

  ifeq ($(strip ${MPP}),true)
    CC = cc
    CXX = CC
  else
    CC  = icc
    CXX = icpc
  endif

else ifeq ($(strip ${COMPILER}),gnu)

  ifeq ($(strip ${MPP}),true)
    CC = mpicc
    CXX = mpiCC
  else
    CC  = gcc
    CXX = CC
  endif

endif

# Override the above definitions

# ifeq ($(strip ${MPP}),true)
#   CC = mpiicc
#   CXX = mpiicpc
# else
#   CC  = icc
#   CXX = icpc
# endif

#CC = /usr/local/mvapich/bin/mpicc  # FNAL
#CXX =  /usr/local/mvapich/bin/mpiCC  # FNAL

#----------------------------------------------------------------------
# 6. Compiler optimization level
# Choices include -g -O, etc

OPT              = -O3

# OpenMP?

OMP = #true

#----------------------------------------------------------------------
# 7. Other compiler optimization flags.  Uncomment stanza to suit.

#-------------- Gnu C -------------------------------------

ifeq ($(strip ${COMPILER}),gnu)

  OCFLAGS += -std=c99

 
  ifeq ($(strip ${OMP}),true)
    OCFLAGS += -fopenmp
    LDFLAGS = -fopenmp
  endif

# Other Gnu options
#OCFLAGS += -mavx # depends on architecture
#OCFLAGS += -Wall

endif

#------------------------ BlueGene -----------------------------------

ifeq ($(strip ${COMPILER}),ibm)

  OCFLAGS = -qarch=450 -qlanglvl=stdc99 # BG/P BG/Q

endif

#-------------- Intel icc/ecc -----------------------------------

ifeq ($(strip ${COMPILER}),intel)

  OCFLAGS += -std=c99

  ifeq ($(strip ${ARCH}),knl)
  ARCH_FLAG = -xMIC-AVX512
  BINEXT=.knl
  else ifeq ($(strip ${ARCH}),knc)
  ARCH_FLAG = -mmic
  BINEXT=.knc
  else ifeq ($(strip ${ARCH}),hsw)
  ARCH_FLAG = -xCORE-AVX2
  BINEXT=.hsw
  else
  ARCH_FLAG = -mavx
  BINEXT=
  endif

  OCFLAGS += ${ARCH_FLAG}
  LDFLAGS += ${ARCH_FLAG}
  OCFLAGS += -parallel-source-info=2 -debug inline-debug-info -qopt-report=5

  ifeq ($(strip ${OMP}),true)
    OCFLAGS += -qopenmp
    LDFLAGS = -qopenmp
  endif

endif

#-------------- Cray-Intel  -----------------------------------

ifeq ($(strip ${COMPILER}),cray-intel)

  OCFLAGS += -std=c99

  ifeq ($(strip ${ARCH}),knl)
  ARCH_FLAG = -xMIC-AVX512
  BINEXT=.knl
  else ifeq ($(strip ${ARCH}),knc)
  ARCH_FLAG = -mmic
  BINEXT=.knc
  else ifeq ($(strip ${ARCH}),hsw)
  ARCH_FLAG = -xCORE-AVX2
  BINEXT=.hsw
  else
  ARCH_FLAG = -mavx
  BINEXT=
  endif

  OCFLAGS += ${ARCH_FLAG}
  LDFLAGS += ${ARCH_FLAG}
  OCFLAGS += -parallel-source-info=2 -debug inline-debug-info -qopt-report=5

  ifeq ($(strip ${OMP}),true)
    OCFLAGS += -qopenmp
    LDFLAGS = -qopenmp
  endif

endif

#-------------- Portland Group ----------------------------
#OCFLAGS = -tp p6 -Munroll=c:4,n:4
#OCFLAGS= -mpentiumpro -march=pentiumpro -funroll-all-loops -malign-double -D_REENTRANT  # Pentium pro

#----------------------------------------------------------------------
# 8. Choose large file support.

ifeq ($(strip ${COMPILER}),ibm)
  CLFS = -D_LARGE_FILES   # AIX
else
  CLFS = -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE # Large files gcc only
endif

#----------------------------------------------------------------------
# 9. Installation-specific MPI includes and libraries
#    Not needed if using mpicc or on single processor

#----------------- Intel MPI Custom -------------------------------------------

# IMPI = -I/opt/intel/impi/5.1.3.181/compilers_and_libraries_2016.2.181/linux/mpi/intel64/include

# ifeq ($(strip ${ARCH}),knl)
#   LMPI = -L/opt/intel/compiler/latest/compilers_and_libraries_2016/linux/mpi/mic/lib
# else ifeq ($(strip ${ARCH}),knc) 
#   LMPI = -L/opt/intel/compiler/latest/compilers_and_libraries_2016/linux/mpi/mic/lib
# else
#   LMPI = -L/opt/intel/compiler/latest/compilers_and_libraries_2016/linux/mpi/intel64/lib
# endif

#----------------- MVICH ----------------------------------------------
#IMPI = -I/uufs/icebox/sys/src/mpich/1.2.0-via/include  # MVICH
#LMPI = -L/uufs/icebox/sys/pkg/mpich/1.2.0-via/lib -lmpi -lvipl -lpthread # MVICH

#----------------------------------------------------------------------
# 10. I/O routines
# Both io_nonansi and io_ansi should work on a scalar machine
# Solaris 2.6 gave "bad file number" errors with io_ansi.  CD

MACHINE_DEP_IO   = io_ansi.o # (io_ansi.o io_nonansi.o io_dcap.o)

# Uncomment if you have and want to support dcache I/O
# (Forces use of io_dcap.o)

# WANTDCAP = true

# The location of the installed dcap libraries. Uncomment and enter if
# it is not already defined as a system environment variable.

# DCAP_DIR = /usr/local/develop/dcache

# Choose the appropriate library path according to the addressing
# size of the machine

# DCAPLIB  = lib64 # (lib64 lib)

#----------------------------------------------------------------------
# 11. SciDAC package options

# Edit these "wants"

WANTQOP = # true # or blank. Implies HAVEQDP, HAVEQOP, HAVEQMP.

WANTQIO = true # or blank.  Implies HAVEQMP.

WANTQMP = # true or blank.

# Edit these locations for the installed SciDAC packages
# It is assumed that these are the parents of "include" and "lib"

SCIDAC = ${HOME}/scidac/install
# Parallel versions
QMPPAR = ${SCIDAC}/qmp
QIOPAR = $(SCIDAC)/qio
# Single processor versions
QMPSNG = ${SCIDAC}/qmp-single
QIOSNG = $(SCIDAC)/qio-single
QLA = ${SCIDAC}/qla
# Either version
QDP = ${SCIDAC}/qdp
QOPQDP = ${SCIDAC}/qopqdp
#QOPQDP = ${SCIDAC}/qopqdp-lapack # BG/P

QOP = ${QOPQDP}

# Make_template_scidac defines these macros:
# HAVEQOP HAVEQDP HAVEQIO HAVEQMP (Says what we are compiling with)
# LIBSCIDAC INCSCIDAC (The -L and -I compiler and linker lists)
# SCIDAC_LIBRARIES SCIDAC_HEADERS  (Lists for make dependencies)
# CSCIDAC (List of compiler macros for SciDAC modules)

include ../Make_template_scidac

#----------------------------------------------------------------------
# 12. FFTW3 Options

WANTFFTW = #true

ifeq ($(strip ${WANTFFTW}),true)
FFTW=/usr/local/fftw

ifeq ($(strip ${PRECISION}),1)
  INCFFTW = -I${FFTW}/float-mvapich2/include
  LIBFFTW = -L${FFTW}/float-mvapich2/lib
  LIBFFTW += -lfftw3f
else
  INCFFTW = -I${FFTW}/double-mvapich2/include
  LIBFFTW = -L${FFTW}/double-mvapich2/lib
  LIBFFTW += -lfftw3
endif
endif

#----------------------------------------------------------------------
# 13. LAPACK Options (for qopqdp-lapack and arb_overlap )

#LIBLAPACK = -L/opt/ibmcmp/xlf/bg/11.1/lib /soft/apps/LAPACK/liblapack_bgp.a /soft/apps/LIBGOTO/libgoto.a -lxlf90 -lxlsmp # LAPACK on BG/P

# Utah physics and math Redhat-linux
# LIBLAPACK = -L/usr/local/lib64  -llapack-gfortran -lblas-gfortran -L/usr/lib/gcc/x86_64-redhat-linux/4.1.2 -lgfortran

# Utah physics and math Centos-linus
# LIBLAPACK = -L/usr/local/lib64  -llapack-gfortran -lblas-gfortran -lgfortran

# FNAL cluster (Jim's installation of ATLAS)
#LDFLAGS = -Wl,-rpath,"/usr/local/atlas-3.10-lapack-3.4.2/lib" -L/usr/local/atlas-3.10-lapack-3.4.2/lib
#LIBS = $(LDFLAGS) -lprimme -lm  -llapack -lptf77blas -lptcblas -latlas -lgfortran -lpthread

# NERSC Cori
# LIBLAPACK = -L${LIBSCI_BASE_DIR}/INTEL/14.0/haswell/lib -l sci_intel

#----------------------------------------------------------------------
# 14. PRIMME Options (for arb_overlap and ks_eigen).  REQUIRES LAPACK AS WELL.

WANTPRIMME = #true

ifeq ($(strip ${WANTPRIMME}),true)
  LIBPRIMME = -L${HOME}/milc/install/PRIMME -lzprimme
endif

#----------------------------------------------------------------------
# 15. GPU/QUDA Options

WANTQUDA    = #true
WANT_CL_BCG_GPU = #true
WANT_FN_CG_GPU = #true
WANT_FL_GPU = #true
WANT_FF_GPU = #true
WANT_GF_GPU = #true

ifeq ($(strip ${WANTQUDA}),true)

  QUDA_HOME = ${HOME}/quda

  INCQUDA = -I${QUDA_HOME}/include -I/lib -I${QUDA_HOME}/tests
  LIBQUDA = -L${QUDA_HOME}/lib -lquda
  QUDA_LIBRARIES = ${QUDA_HOME}/lib

  CUDA_HOME = /usr/local/cuda
  INCQUDA += -I${CUDA_HOME}/include
  LIBQUDA += -L${CUDA_HOME}/lib64 -lcudart

# Definitions of compiler macros -- don't change.  Could go into a Make_template_QUDA

  CGPU += -DHAVE_QUDA

  ifeq ($(strip ${WANT_CL_BCG_GPU}),true)
    HAVE_CL_GPU = true
    CGPU += -DUSE_CL_GPU
  endif

  ifeq ($(strip ${WANT_FN_CG_GPU}),true)
    HAVE_FN_CG_GPU = true
    CGPU += -DUSE_CG_GPU
  endif

  ifeq ($(strip ${WANT_GF_GPU}),true)
    HAVE_GF_GPU = true
    CGPU += -DUSE_GF_GPU
  endif

  ifeq ($(strip ${WANT_FL_GPU}),true)
    HAVE_FL_GPU = true
    CGPU += -DUSE_FL_GPU
  endif

  ifeq ($(strip ${WANT_FF_GPU}),true)
    HAVE_FF_GPU = true
    CGPU += -DUSE_FF_GPU
  endif

# Verbosity choices: 
# SET_QUDA_SILENT, SET_QUDA_SUMMARIZE, SET_QUDA_VERBOSE, SET_QUDA_DEBUG_VERBOSE

  CGPU += -DSET_QUDA_SUMMARIZE

endif

#----------------------------------------------------------------------
# 16. QPhiX Options

WANTQPHIX = #true

QPHIX_HOME = ${HOME}/QPhiX/mbench

ifeq ($(strip ${WANTQPHIX}), true)

  INCQPHIX = -I${QPHIX_HOME}
  HAVE_QPHIX = true
  CPHI = -DHAVE_QPHIX

  ifeq ($(strip ${MPP}),true)

  # MPI versions of QPHIX

  ifeq ($(strip ${ARCH}),knl)
    LIBQPHIX = -L${QPHIX_HOME} -lqphixmilc_avx512 -lrt
  else ifeq ($(strip ${ARCH}),knc)
    LIBQPHIX = -L${QPHIX_HOME} -lqphixmilc_mic -lrt
  else ifeq ($(strip ${ARCH}),hsw)
    LIBQPHIX = -L${QPHIX_HOME} -lqphixmilc_avx2 -lrt
  else
  endif

  else

  # Non-MPI versions

  ifeq ($(strip ${ARCH}),knl)
    LIBQPHIX = -L${QPHIX_HOME} -lqphixmilc_avx512_single -lrt
  else ifeq ($(strip ${ARCH}),knc)
    LIBQPHIX = -L${QPHIX_HOME} -lqphixmilc_mic_single -lrt
  else ifeq ($(strip ${ARCH}),hsw)
    LIBQPHIX = -L${QPHIX_HOME} -lqphixmilc_avx2_single -lrt
  else
  endif

  endif

endif

#----------------------------------------------------------------------
# 17. Linker (need the C++ linker for QUDA and QPHIX)

ifeq ($(strip ${WANTQUDA}),true)
  LD  = ${CXX}
else ifeq ($(strip ${WANTQPHIX}),true)
  LD  = ${CXX}
else
  LD  = ${CC}
endif

#----------------------------------------------------------------------
# 18. Extra ld flags

# VTune

# VTUNE_VERSION = 2016u2
# INCVTUNE = -I/opt/intel/vtune/${VTUNE_VERSION}/vtune_amplifier_xe_2016/include
# LIBVTUNE = -L/opt/intel/vtune/${VTUNE_VERSION}/vtune_amplifier_xe_2016/lib64 -l ittnotify
# 
# OCFLAGS += -DVTUNE

#----------------------------------------------------------------------
# 19. Inlining choices

# USE INLINE SSE WITH EXTREME CAUTION!  IT MAY GIVE WRONG RESULTS.

# SSE ASM and explicit C inlining is available for some of the library
# functions.

# Use SSE for P3 or P4 or Athlon Thunderbird and compilers, such
# as gcc that accept ASM macros

# Both SSE and C inline macros can be invoked selectively by defining
# SSE_INLINE and C_INLINE and changing the function call to the macro
# name.

# To invoke the inline macros globally (where available) without
# changing the code, define, instead, SSE_GLOBAL_INLINE or
# C_GLOBAL_INLINE.

# You may use SSE and C inlining at the same time.  The SSE version
# takes precedence when both are available.

# See also the libraries Make_SSE_nasm for building non-inline SSE
# Some compilers don't like -DSSE_INLINE with the debugging -g option.

# Choose nothing or
#  [ C_INLINE | C_GLOBAL_INLINE ] [ SSE_INLINE | SSE_GLOBAL_INLINE ]
INLINEOPT = -DC_GLOBAL_INLINE # -DSSE_GLOBAL_INLINE #-DC_INLINE

# There are special single-precision macros for the AMD Opteron
# To get them, uncomment the next line

#INLINEOPT += -DSSEOPTERON

#----------------------------------------------------------------------
# 20. Miscellaneous macros for performance control and metric

#     Define them with a -D prefix.

#------------------------------
# Print timing statistics.
# Applications: many

# Use any combination of these
# CGTIME CG Solver
# FFTIME Fermion force
# FLTIME Link fattening
# GFTIME Gauge force
# IOTIME I/O timing
# PRTIME print time (clover_invert2)

# REMAP  report remapping time for QDP, QOP in conjunction with above

CTIME = # -DCGTIME -DFFTIME -DFLTIME -DGFTIME -DREMAP -DPRTIME -DIOTIME

#------------------------------
# Profiling
# Applications:  QDP

# QDP_PROFILE         Generates a report for all QDP routines

CPROF =#

#------------------------------
# Troubleshooting
# Applications: All

# COM_CRC            Message passing test.  Checksums on all gathers.

#------------------------------
# Debugging and diagnostics
# Applications:  all

# CHECK_MALLOC        Report malloc/free activity.
#                     (Then process stdout using check_malloc.pl)
# CG_DEBUG            Print debugging information for the inverters.
# CG_OK               Print inverter convergence information even when OK
# REMAP_STDIO_APPEND  All nodes append to stdout.
# NO_FREOPEN          Don't use freopen to remap stdin and stdout
#
# HISQ_SVD_VALUES_INFO Print HISQ SVD diagnostics
# HISQ_SVD_COUNTER    Print summary count of SVD uses
# HISQ_FORCE_FILTER_COUNTER Print summary count of force filter applications.

CDEBUG = -DCG_OK -DREMAP_STDIO_APPEND # -DCHECK_MALLOC 

#------------------------------
# Backward compatibility

# As of version qopqdp 0.9.0 the normalization convention for the
# staggered inverter changed.  If you are using a version of QOPQDP
# with the old convention, define this macro:
CCOMPAT += #-DOLD_QOPQDP_NORM

# Prior to version 7.7.2 the conversion from staggeredd to naive was peculiar.
CCOMPAT += #-DOLD_STAGGERED2NAIVE

#------------------------------
# Layout
# Applications: all

# These are currently selected only by editing Make_template.
#  Choices
#    layout_hyper_prime.o          Standard hypercubic
#    layout_timeslices.o           Puts full timeslices on each node.
#    layout_squares.o              Puts 2D slices on each node.
#    layout_hyper_tstretch.o       Rarely used.  Fewer timeslices on last node.
#                                  In case node no is incommensurate with nt
#    layout_timeslices_2.o         Untested.  Supposedly more forgiving.

#------------------------------
# Compute node grid layout

#     layout_hyper_prime views the machine as a grid and selects a layout
#     that distributes the lattice uniformly across the nodes and
#     minimizes the surface to volume ratio.  On fixed grid machines
#     such as the IBM Bluegene it may be better to control the
#     geometry.  In that case some of our applications support
#     the compiler macro FIX_NODE_GEOM.  You must then provide
#     and extra list of dimensions in the parameter input file.
#     See e.g. ks_imp_rhmc.

CGEOM =#-DFIX_NODE_GEOM

#------------------------------
# I/O node grid layout

#     For some applications and architectures it is more efficient to
#     split large files and have a subset of processors handle the I/O.
#     We define the I/O node partitions by hypercubes in the same manner 
#     as the compute node geometry above.  The I/O node geometry must
#     be commensurate with the compute node geometry.
#     For applications that support it, the I/O node geometry is specified
#     by the macro FIX_IONODE_GEOM.  Then the parameter input file
#     includes a list of dimensions.

CGEOM +=# -DFIX_IONODE_GEOM

#------------------------------
# Improved staggered CG inverter and Dslash
# Applications: ks_imp_dyn ks_imp_rhmc ks_imp_invert_multi ks_hl_spectrum

#  Note, some options still require editing Make_template
#  Choices 
#   dslash_fn.o                   Overlaps computation with backward gathers.
#   dslash_fn2.o                  Does all gathers before computation
#                                   but has fewer FORALLSITES loops
#   dslash_fn_dblstore.o          Double store version of dslash_fn.o
#                                   Supports GATHER13

# DBLSTORE_FN    Copies backward links.  Requires more memory.
#                You must also call for dslash_fn_dblstore.o for now.
# D_FN_GATHER13  Combine third and next neighbor gathers in Dslash.
#                For now, works only with dslash_fn_dblstore.o
# FEWSUMS        Fewer CG reduction calls

KSCGSTORE = -DDBLSTORE_FN -DD_FN_GATHER13 -DFEWSUMS

#------------------------------
# Staggered fermion force routines
# Applications: ks_imp_dyn ks_imp_rhmc

# These are currently selected only by editing Make_template

# Choices
#  fermion_force_asqtad.o    Optimized for the Asqtad action
#  fermion_force_general.o   Takes any quark action

#------------------------------
# Prefetching
# Applications: all

# PREFETCH (Not working yet)

CPREFETCH = #

#------------------------------
# Multimass improved KS CG solvers
# Applications: ks_imp_rhmc ks_measure ks_spectrum

# Choices
# KS_MULTICG=OFFSET  The basic multicg solver.
# KS_MULTICG=HYBRID  Solve with multicg and polish off with single mass CG.
# KS_MULTICG=FAKE    Iterate the single mass solver.
# KS_MULTICG=REVERSE Iterate in reverse order
# KS_MULTICG=REVHYB  Same as HYBRID but with vectors in reverse order.

# HALF_MIXED         If PRECISION=2, do multimass solve in single precision
#                    and single-mass refinements in double
# NO_REFINE          No refinements except for masses with nonzero Naik eps
# CPU_REFINE         Refine on CPU only (if at all), not GPU

KSCGMULTI = -DKS_MULTICG=HYBRID # -DHALF_MIXED # -DNO_REFINE

#------------------------------
# Multifermion force routines
# Applications: ks_imp_rhmc

# Choices
# KS_MULTIFF=FNMAT    Construct matrix parallel transporters
#                     and use with sum of outer products of sources 
# KS_MULTIFF=FNMATREV Older version of FNMAT.  Traverses path
#                     in reverse order
# KS_MULTIFF=ASVEC    Use improved Asqtad, parallel transporting
#                     groups of source vectors.  See VECLENGTH.

# Additional options
# VECLENGTH=n        Number of source vectors to process in one group.
#                    Applies only to the ASVEC option

KSFFMULTI = -DKS_MULTIFF=FNMAT


#------------------------------
# RHMC molecular dynamics algorithm
# Applications: ks_imp_rhmc

# Choices
# (Not needed if the make target has a preset value in Make_template)

# INT_ALG=INT_LEAPFROG
# INT_ALG=INT_OMELYAN
# INT_ALG=INT_2EPS_3TO1
# INT_ALG=INT_2EPS_2TO1
# INT_ALG=INT_2G1F
# INT_ALG=INT_3G1F
# INT_ALG=INT_4MN4FP
# INT_ALG=INT_4MN5FV
# INT_ALG=INT_FOURSTEP
# INT_ALG=INT_PLAY

KSRHMCINT =#

#------------------------------

# Staggered spin-taste operator shift for applications that construct
# nonlocal interpolating operators.  By default the shift operator is
# symmetric (forward and backward).  This macro makes it one-sided
# (forward).

# The Fat-Naik variants of the nonlocal operators are currently
# constructed only from the forward fat links.

KSSHIFT = # -DONE_SIDED_SHIFT

#------------------------------
# Clover inverter choice
# Applications: clover_invert, clover_invert2
#

# CL_CG=BICG  Biconjugate gradient
# CL_CG=CG    Standard CG
# CL_CG=MR    Minimum residue
# CL_CG=HOP   Hopping

# HALF_MIXED  Do double-precision inversion with single, or single with half (if supported)
# MAX_MIXED   Do double-precision inversion with half-precision (if supported)
# SCALE_PROP  Do rescaling for the clover propagator

CLCG = -DCL_CG=BICG 

#------------------------------
# Propagator storage
# Applications: clover_invert2
#

# CLOV_LEAN   Write intermediate propagators to disk, saving memory
#             but increasing run time somewhat

CLMEM = #-DCLOV_LEAN

#----------------------------------------------------------------------
# Extra include paths

INCADD = ${INCFFTW} ${INCQUDA} ${INCQPHIX} ${INCVTUNE}

#----------------------------------------------------------------------
#  Extra libraries

LIBADD = ${LIBFFTW} ${LIBPRIMME} ${LIBLAPACK} ${LIBQUDA} ${LIBVTUNE}

ifeq ($(strip ${WANTQPHIX}), true)
  LIBADD += ${LIBQPHIX}
endif

#------------------------------
# Summary

CODETYPE = ${CTIME} ${CPROF} ${CDEBUG} ${CGEOM} ${KSCGSTORE} ${CPREFETCH} \
 ${KSCGMULTI} ${KSFFMULTI} ${KSRHMCINT} ${KSSHIFT} ${CLCG} ${CLMEM} ${CQOP} \
 ${CCOMPAT}

#----------------------------------------------------------------------
# MILC library make file in libraries directory.  
#    CHECK IT FOR FURTHER OPTIONS!

MAKELIBRARIES = Make_vanilla

#----------------------------------------------------------------------
# End of user choices.  Please, also, check choices in include/config.h.
#----------------------------------------------------------------------

ifeq ($(strip ${OMP}),true)
  OCFLAGS += -DOMP
endif

ifeq ($(strip ${MPP}),true)
  ifeq ($(strip ${HAVEQMP}),true)
     COMMTYPE = QMP
     COMMPKG = com_qmp.o
  else
     COMMTYPE = MPI
     COMMPKG = com_mpi.o
  endif
else
  ifeq ($(strip ${HAVEQMP}),true)
     COMMTYPE = SINGLE
     COMMPKG = com_qmp.o
  else
     COMMTYPE = SINGLE
     COMMPKG = com_vanilla.o
  endif
endif

ifeq ($(strip ${HAVEQDP}),true)
  QDPPREC = -DQDP_PrecisionInt=${PRECISION}
endif

ifeq ($(strip ${HAVEQOP}),true)
  QOPPREC = -DQOP_PrecisionInt=${PRECISION}
endif

ifeq ($(strip ${WANTQPHIX}),true)
  QPHIXPREC = -DQPHIX_PrecisionInt=${PRECISION}
endif

ifeq ($(strip ${WANTDCAP}),true)
   MACHINE_DEP_IO = io_dcap.o
   OCFLAGS += -I${DCAP_DIR}/include
   LDFLAGS += -L${DCAP_DIR}/${DCAPLIB} -Wl,--rpath,${DCAP_DIR}/${DCAPLIB} -ldcap
endif

ifeq ($(strip ${WANTFFTW}),true)
  HAVEFFTW = true
endif

ifeq ($(strip ${WANTPRIMME}),true)
  HAVEPRIMME = true
endif

# Make_template_combos defines convenience macros for interdependent
# groups of compilation units.  They are used to specify build lists.

include ../Make_template_combos

CPREC = -DPRECISION=${PRECISION} ${QDPPREC} ${QOPPREC} ${QPHIXPREC}
DARCH = ${CSCIDAC} ${CGPU} ${CPHI}

# Complete set of compiler flags - do not change
CFLAGS = ${OPT} ${OCFLAGS} -D${COMMTYPE} ${CODETYPE} ${INLINEOPT} \
	${CPREC} ${CLFS} ${INCSCIDAC} -I${MYINCLUDEDIR} ${DARCH} \
	${DEFINES} ${ADDDEFINES} ${IMPI} ${INCADD}

ILIB = ${LIBSCIDAC} ${LMPI} ${LIBADD}

.PHONY: time check test_clean
time:
	make -f Make_time time

check: test_clean
	cd test ; perl ../../check.pl ${EXEC} ${CASE} ${PREC} < checklist

test_clean:
	cd test ; make test_clean

include Make_template