Import latest changes (#82)

* reorganize source files
* add debug cmd option
* pass OCCA_CXX and OCCA_CXXFLAGS to cmake
* fix mapB issue for scalars

CMakeLists.txt

@@ -245,7 +245,10 @@ set(THIRD_PARTY_INCLUDES ${THIRD_PARTY_INCLUDES}
   ${CDSDIR}
 )
 
-set(NEKRS_INCLUDES ${CMAKE_CURRENT_SOURCE_DIR}/src
+set(NEKRS_INCLUDES 
+  ${CMAKE_CURRENT_SOURCE_DIR}/src
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/mesh
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/core
   ${CMAKE_CURRENT_SOURCE_DIR}/src/udf
   ${NEKINTERFACEDIR}
 )
@@ -296,23 +299,23 @@ add_custom_command(OUTPUT ${LIBP_BUILD_LIBRARIES}
 ###############################################################################
 set(SRC 
     src/nekrs.cpp
-    src/cfl.cpp
-    src/filter.cpp
-    src/bcMap.cpp
-    src/insSetup.cpp
-    src/runTime.cpp
-    src/tombo.cpp
-    src/cds.cpp
+    src/core/cfl.cpp
+    src/core/filter.cpp
+    src/core/bcMap.cpp
+    src/core/insSetup.cpp
+    src/core/runTime.cpp
+    src/core/tombo.cpp
+    src/core/cds.cpp
     src/udf/udf.cpp
-    src/meshSetup.cpp
-    src/meshNekReader.cpp
-    src/meshPhysicalNodesHex3D.cpp
-    src/occaDeviceConfig.cpp
+    src/mesh/meshSetup.cpp
+    src/mesh/meshNekReader.cpp
+    src/mesh/meshPhysicalNodesHex3D.cpp
+    src/core/occaDeviceConfig.cpp
     src/nekInterface/nekInterfaceAdapter.cpp
-    src/parReader.cpp
-    src/configReader.cpp
-    src/tinyexpr.c
-    src/timer.cpp
+    src/core/parReader.cpp
+    src/core/configReader.cpp
+    src/core/tinyexpr.c
+    src/core/timer.cpp
     ## To get fortran flags
     src/dummy.f
 )

config/nekrs.conf.in

@@ -9,6 +9,6 @@ FFLAGS = @CMAKE_CXX_FLAGS@
 
 NEK5000_PPLIST = @NEK5000_PPLIST@
 LIBP_DEFINES = @LIBP_DEFINES@
-OCCA_CXX = @CMAKE_CXX_COMPILER@
-OCCA_CXXFLAGS = @CMAKE_CXX_FLAGS@
+OCCA_CXX = @CMAKE_OCCA_CXX_COMPILER@
+OCCA_CXXFLAGS = @CMAKE_OCCA_CXX_FLAGS@
 

examples/conj_ht/conj_ht.udf

@@ -72,6 +72,9 @@ void UDF_Setup(ins_t *ins)
 
 void UDF_ExecuteStep(ins_t *ins, dfloat time, int tstep)
 {
-  if (ins->isOutputStep) nek_userchk();
+  if (ins->isOutputStep) {
+    nek_ocopyFrom(ins, time, tstep);
+    nek_userchk();
+  }
   if (ciMode) ciTestErrors(ins, time, tstep);
 }

examples/eddyPeriodic/eddy.udf

@@ -20,5 +20,8 @@ void UDF_Setup(ins_t *ins)
 
 void UDF_ExecuteStep(ins_t *ins, dfloat time, int tstep)
 {
-  if (ins->isOutputStep) nek_userchk();
+  if (ins->isOutputStep) {
+    nek_ocopyFrom(ins, time, tstep);
+    nek_userchk();
+  }
 }

examples/ethier/ethier.udf

@@ -55,6 +55,9 @@ void UDF_Setup(ins_t *ins)
 
 void UDF_ExecuteStep(ins_t *ins, dfloat time, int tstep)
 {
-  if (ins->isOutputStep) nek_userchk();
+  if (ins->isOutputStep) {
+    nek_ocopyFrom(ins, time, tstep);
+    nek_userchk();
+  }
   if (ciMode) ciTestErrors(ins, time, tstep);
 }

examples/hemi/hemi.udf

@@ -18,5 +18,8 @@ void UDF_Setup(ins_t *ins)
 
 void UDF_ExecuteStep(ins_t *ins, dfloat time, int tstep)
 {
-  if (ins->isOutputStep) nek_userchk();
+  if (ins->isOutputStep) {
+    nek_ocopyFrom(ins, time, tstep);
+    nek_userchk();
+  }
 }

examples/lowMach/lowMach.udf

@@ -55,6 +55,9 @@ void UDF_Setup(ins_t *ins)
 
 void UDF_ExecuteStep(ins_t *ins, dfloat time, int tstep)
 {
-  if (ins->isOutputStep) nek_userchk();
+  if (ins->isOutputStep) {
+    nek_ocopyFrom(ins, time, tstep);
+    nek_userchk();
+  }
   if (ciMode) ciTestErrors(ins, time, tstep);
 }

examples/pb146/pb.oudf

@@ -7,10 +7,20 @@ void insVelocityDirichletConditions3D(bcData *bc)
   bc->wP = 1.0;
 }
 
+void cdsDirichletConditions3D(bcData *bc)
+{                                                                        
+  bc->sP = 0.0;
+}
+
+void cdsNeumannConditions3D(bcData *bc)
+{                                                                        
+  bc->sF = 1.0;
+}
+
 // Stabilized outflow (Dong et al)
 void insPressureDirichletConditions3D(bcData *bc)
 {
-  const dfloat iU0delta = 10.0;
+  const dfloat iU0delta = 20.0;
   const dfloat un = bc->uM*bc->nx + bc->vM*bc->ny + bc->wM*bc->nz;
   const dfloat s0 = 0.5 * (1.0 - tanh(un*iU0delta)); 
   bc->pP = -0.5 * (bc->uM*bc->uM + bc->vM*bc->vM + bc->wM*bc->wM) * s0;

examples/pb146/pb.par

@@ -5,20 +5,20 @@ deviceNumber = LOCAL-RANK
 [GENERAL]
 #verbose = true
 polynomialOrder = 7
-#startFrom = r1.fld
+#startFrom = r.fld
 stopAt = endTime
-endTime = 10
+endTime = 20
 
 dt = 2.0e-3
 timeStepper = tombo2
 extrapolation = subCycling
 subCyclingSteps = 2
 
-writeControl = TIMESTEP
-writeInterval = 1000
+writeControl = runTime
+writeInterval = 1
 
 filtering = hpfrt
-filterWeight = 100 
+filterWeight = 0.2/${dt} 
 filterModes = 2
 
 [PRESSURE]
@@ -29,7 +29,16 @@ residualTol = 1e-04
 
 [VELOCITY]
 solver = pcg+block
-boundaryTypeMap = inlet, outlet, wall
+boundaryTypeMap = inlet, outlet, wall, wall
 density = 1.0
-viscosity = -10000.0
+viscosity = -5000.0
 residualTol = 1e-06
+
+[TEMPERATURE]
+boundaryTypeMap = inlet, outlet, insulated, flux 
+residualTol = 1e-06
+rhoCp = 1.0
+conductivity = -5000
+
+[BOOMERAMG]
+iterations = 1

examples/pb146/pb.udf

@@ -19,5 +19,8 @@ void UDF_Setup(ins_t *ins)
 
 void UDF_ExecuteStep(ins_t *ins, dfloat time, int tstep)
 {
-  if (ins->isOutputStep) nek_userchk();
+  if (ins->isOutputStep) {
+    nek_ocopyFrom(ins, time, tstep);
+    nek_userchk();
+  }
 }

examples/pb146/pb.usr

@@ -8,7 +8,8 @@ c-----------------------------------------------------------------------
       ux   = 0.0
       uy   = 0.0
       uz   = 1.0
-
+      temp = 0.0
+
       return
       end
 c-----------------------------------------------------------------------
@@ -47,17 +48,18 @@ c-----------------------------------------------------------------------
         zm1(i,1,1,1)=zm1(i,1,1,1)*scale
       enddo
 
-      do e=1,nelv
-      do f=1,2*ndim
-        if(cbc(f,e,1) .eq. 'SYM') cbc(f,e,1)='W  ' 
-      enddo
-      enddo
-
       do iel=1,nelt
       do ifc=1,2*ndim
          if (cbc(ifc,iel,1) .eq. 'v  ') boundaryID(ifc,iel) = 1
          if (cbc(ifc,iel,1) .eq. 'O  ') boundaryID(ifc,iel) = 2
-         if (cbc(ifc,iel,1) .eq. 'W  ') boundaryID(ifc,iel) = 3
+         if (cbc(ifc,iel,1) .eq. 'SYM') boundaryID(ifc,iel) = 3
+         if (cbc(ifc,iel,1) .eq. 'W  ') boundaryID(ifc,iel) = 4
+      enddo
+      enddo
+
+      do e=1,nelv
+      do f=1,2*ndim
+        if(cbc(f,e,1) .eq. 'SYM') cbc(f,e,1)='W  ' 
       enddo
       enddo
 

examples/rbc/rbc.udf

@@ -28,5 +28,8 @@ void UDF_Setup(ins_t *ins)
 
 void UDF_ExecuteStep(ins_t *ins, dfloat time, int tstep)
 {
-  if (ins->isOutputStep) nek_userchk(); 
+  if (ins->isOutputStep) {
+    nek_ocopyFrom(ins, time, tstep);
+    nek_userchk();
+  }
 }

examples/tgv/tgv.udf

@@ -27,5 +27,8 @@ void UDF_Setup(ins_t *ins)
 
 void UDF_ExecuteStep(ins_t *ins, dfloat time, int tstep)
 {
-  if (ins->isOutputStep) nek_userchk();
+  if (ins->isOutputStep) {
+    nek_ocopyFrom(ins, time, tstep);
+    nek_userchk();
+  }
 }

examples/turbPipe/turbPipe.udf

@@ -37,5 +37,8 @@ void UDF_Setup(ins_t *ins)
 void UDF_ExecuteStep(ins_t *ins, dfloat time, int tstep)
 {
   velRecycling::copy();
-  if (ins->isOutputStep) nek_userchk();
+  if (ins->isOutputStep) {
+    nek_ocopyFrom(ins, time, tstep);
+    nek_userchk();
+  }
 }

examples/turbPipePeriodic/turbPipe.udf

@@ -39,5 +39,8 @@ void UDF_Setup(ins_t *ins)
 
 void UDF_ExecuteStep(ins_t *ins, dfloat time, int tstep)
 {
-  if (ins->isOutputStep) nek_userchk(); 
+  if (ins->isOutputStep) {
+    nek_ocopyFrom(ins, time, tstep);
+    nek_userchk();
+  }
 }

examples/vortexOutflow/vortex.udf

@@ -23,5 +23,8 @@ void UDF_Setup(ins_t *ins)
 
 void UDF_ExecuteStep(ins_t *ins, dfloat time, int tstep)
 {
-  if (ins->isOutputStep) nek_userchk();
+  if (ins->isOutputStep) {
+    nek_ocopyFrom(ins, time, tstep);
+    nek_userchk();
+  }
 }

install_script.in.cmake

@@ -22,6 +22,11 @@ macro(install_glob_recurse_if_newer source pattern install_prefix)
   install_("${src_files}" ${source} ${install_prefix})
 endmacro()
 
+macro(install_glob_recurse_if_newer_flat source pattern install_prefix)
+  file(GLOB_RECURSE src_files "${source}/${pattern}") 
+  install_flat_("${src_files}" ${source} ${install_prefix})
+endmacro()
+
 macro(install_glob_if_newer source pattern install_prefix)
   file(GLOB src_files "${source}/${pattern}") 
   install_("${src_files}" ${source} ${install_prefix})
@@ -86,10 +91,13 @@ install_files_if_newer(@NEKDIR@/bin/nekconfig @CMAKE_INSTALL_PREFIX@/nek5000/bin
 ## nekRS
 message("-- Installing nekRS")
 install_glob_if_newer(@CMAKE_SOURCE_DIR@/scripts * @CMAKE_INSTALL_PREFIX@/bin)
-install_glob_if_newer(@CMAKE_SOURCE_DIR@/src "*.h*" @CMAKE_INSTALL_PREFIX@/include)
-install_glob_if_newer(@CMAKE_SOURCE_DIR@/src/udf "*.hpp" @CMAKE_INSTALL_PREFIX@/include)
-install_glob_if_newer(@CMAKE_SOURCE_DIR@/src/nekInterface "*.hpp" @CMAKE_INSTALL_PREFIX@/include)
-install_glob_if_newer(@CMAKE_SOURCE_DIR@/src/okl * @CMAKE_INSTALL_PREFIX@/okl)
+install_glob_recurse_if_newer(@CMAKE_SOURCE_DIR@/src "*.h" @CMAKE_INSTALL_PREFIX@/include)
+install_glob_recurse_if_newer(@CMAKE_SOURCE_DIR@/src "*.hpp" @CMAKE_INSTALL_PREFIX@/include)
+install_glob_recurse_if_newer(@CMAKE_SOURCE_DIR@/okl "*.okl" @CMAKE_INSTALL_PREFIX@/okl)
+
+message("-- Installing nekRS plugins")
+install_glob_recurse_if_newer(@CMAKE_SOURCE_DIR@/src/avg "*.hpp" @CMAKE_INSTALL_PREFIX@/plugin)
+install_glob_recurse_if_newer(@CMAKE_SOURCE_DIR@/src/velRecycling "*.hpp" @CMAKE_INSTALL_PREFIX@/plugin)
 
 install_files_if_newer(@CMAKE_SOURCE_DIR@/src/udf/CMakeLists.txt @CMAKE_INSTALL_PREFIX@/udf)
 install_files_if_newer("@CMAKE_SOURCE_DIR@/src/nekInterface/NEKINTF;@CMAKE_SOURCE_DIR@/src/nekInterface/nekInterface.f;@CMAKE_SOURCE_DIR@/src/nekInterface/Makefile" @CMAKE_INSTALL_PREFIX@/nekInterface)

makenrs

@@ -10,15 +10,14 @@
 : ${NEKRS_CC:="mpicc"}
 : ${NEKRS_CXX:="mpic++"}
 : ${NEKRS_FC:="mpif77"}
+: ${NEKRS_CXXFLAGS:="-g -O2 -march=native -mtune=native"}
 
-: ${NEKRS_CXXFLAGS:="-g -O2 -march=native -mtune=native -DUSE_OCCA_MEM_BYTE_ALIGN=64"}
+: ${OCCA_CXX:="${NEKRS_CXX}"}
 : ${OCCA_CXXFLAGS:="${NEKRS_CXXFLAGS}"}
-
 : ${OCCA_CUDA_ENABLED:=1}
 : ${OCCA_HIP_ENABLED:=0}
 : ${OCCA_OPENCL_ENABLED:=0}
 
-
 ###############################################################################
 # DONT'T TOUCH WHAT FOLLOWS !!!
 ###############################################################################
@@ -38,6 +37,8 @@ fi
 
 : ${OCCA_METAL_ENABLED:=0}
 
+NEKRS_CXXFLAGS+=" -DUSE_OCCA_MEM_BYTE_ALIGN=64" 
+
 if uname -a | grep 'ppc64'; then
   NEKRS_CXXFLAGS=$(echo "$NEKRS_CXXFLAGS" | sed -e "s/march/mcpu/g") 
 fi
@@ -48,6 +49,7 @@ NEKRS_DEBUG=0
 NEKRS_LIBP_DEFINES="-DUSE_NULL_PROJECTION=1"
 NEKRS_NEK5000_PPLIST="PARRSB DPROCMAP"
 
+export OCCA_CXX
 export OCCA_CXXFLAGS
 export OCCA_CUDA_ENABLED
 export OCCA_HIP_ENABLED
@@ -177,6 +179,8 @@ cmake -DCMAKE_BUILD_TYPE="${NEKRS_BUILD_TYPE}" \
       -DCMAKE_Fortran_FLAGS="${NEKRS_FFLAGS}"  \
       -DCMAKE_C_FLAGS="${NEKRS_CFLAGS}" \
       -DCMAKE_CXX_FLAGS="${NEKRS_CXXFLAGS}" \
+      -DCMAKE_OCCA_CXX_COMPILER="${OCCA_CXX}" \
+      -DCMAKE_OCCA_CXX_FLAGS="${OCCA_CXXFLAGS}" \
       -DCMAKE_INSTALL_PREFIX="${NEKRS_INSTALL_DIR}" \
       -DNEK5000_PPLIST="${NEKRS_NEK5000_PPLIST}" \
       -DLIBP_DEFINES="${NEKRS_LIBP_DEFINES}" \

mktar

@@ -71,7 +71,7 @@ function tar_3rd_party() {
 function tar_nekrs() {
   mkdir nekRS
   cp -r ../3rd_party ../scripts ../CMake*.txt ../examples \
-    ../install_*.cmake ../LICENSE ../make* ../README* ../RELEASE* ../src ../config nekRS
+    ../install_*.cmake ../LICENSE ../make* ../README* ../RELEASE* ../src ../okl ../config nekRS
   tar --exclude=.git -cvzf nekrs.tar.gz nekRS
   if [ $? -ne 0 ]; then
     echo "Failed to creare a .tar out of nekrs."

okl/avg/avg.okl

@@ -0,0 +1,40 @@
+@kernel void avgX_XX(const dlong N,
+                     const dlong offset,
+                     const dlong Yoffset,
+                     const dlong Nfields,
+                     const dfloat a,
+                     const dfloat b,
+                     @restrict const  dfloat * X,
+                     @restrict dfloat * Y){
+
+  for(dlong n=0;n<N;++n;@tile(256,@outer,@inner)){
+    if(n<N){
+      for(dlong fld=0;fld<Nfields;++fld){ 
+        const dlong id  = n + fld*offset;
+        const dlong id2 = id + Yoffset;
+        const dfloat xn = X[id];
+        Y[id]  = a*Y[id]  + b*xn; 
+        Y[id2] = a*Y[id2] + b*xn*xn; 
+      }
+    }
+  }
+}
+
+@kernel void avgXY(const dlong N,
+                   const dlong offset,
+                   const dfloat a,
+                   const dfloat b,
+                   @restrict const dfloat * X,
+                   @restrict dfloat * Y){
+
+  for(dlong n=0;n<N;++n;@tile(256,@outer,@inner)){
+    if(n<N){
+      const dfloat vx = X[n + 0*offset];
+      const dfloat vy = X[n + 1*offset];
+      const dfloat vz = X[n + 2*offset];
+      Y[n + 0*offset] = a*Y[n + 0*offset] + b*vx*vy;
+      Y[n + 1*offset] = a*Y[n + 1*offset] + b*vy*vz;
+      Y[n + 2*offset] = a*Y[n + 2*offset] + b*vz*vx;
+    }
+  }
+}