Revert "import latest changes (#85)" (#87)

This reverts commit 04b3fc6.

3rd_party/libparanumal/libparanumal.makefile

@@ -79,6 +79,7 @@ $(HDRDIR)/src/meshLoadReferenceNodesTet3D.o \
 $(HDRDIR)/src/meshOccaSetup2D.o \
 $(HDRDIR)/src/meshOccaSetup3D.o \
 $(HDRDIR)/src/meshOccaSetupQuad3D.o \
+$(HDRDIR)/src/meshParallelConnectNodes.o \
 $(HDRDIR)/src/meshParallelConnectOpt.o \
 $(HDRDIR)/src/meshParallelConsecutiveGlobalNumbering.o\
 $(HDRDIR)/src/meshParallelGatherScatterSetup.o \
@@ -122,8 +123,7 @@ $(HDRDIR)/src/hash.o\
 $(HDRDIR)/src/setupAide.o \
 $(HDRDIR)/src/readArray.o\
 $(HDRDIR)/src/occaHostMallocPinned.o \
-$(HDRDIR)/src/timer.o \
-#$(HDRDIR)/src/meshParallelConnectNodes.o
+$(HDRDIR)/src/timer.o
 
 ifeq ($(OS),Windows_NT)
     detected_OS := Windows

CMakeLists.txt

@@ -322,7 +322,6 @@ set(SRC
     src/mesh/meshSetup.cpp
     src/mesh/meshNekReader.cpp
     src/mesh/meshPhysicalNodesHex3D.cpp
-    src/mesh/meshParallelConnectNodes.cpp
     src/core/occaDeviceConfig.cpp
     src/nekInterface/nekInterfaceAdapter.cpp
     src/core/parReader.cpp

src/core/insSetup.cpp

@@ -688,8 +688,7 @@ ins_t *insSetup(MPI_Comm comm, setupAide &options, int buildOnly)
     MPI_Barrier(mesh->comm);
   }
 
-  if(!buildOnly) {
-    int err = 0;
+  {
     dlong gNelements = mesh->Nelements;
     MPI_Allreduce(MPI_IN_PLACE, &gNelements, 1, MPI_DLONG, MPI_SUM, mesh->comm);
     const dfloat sum2 = (dfloat)gNelements * mesh->Np;
@@ -704,27 +703,13 @@ ins_t *insSetup(MPI_Comm comm, setupAide &options, int buildOnly)
     ins->o_wrk1.copyTo(tmp, Nlocal*sizeof(dfloat));
     dfloat sum1 = 0;
     for(int i=0; i<Nlocal; i++) sum1 += tmp[i];
+    free(tmp);
     MPI_Allreduce(MPI_IN_PLACE, &sum1, 1, MPI_DFLOAT, MPI_SUM, mesh->comm);
-    sum1 = abs(sum1-sum2)/sum2;
-    if(sum1 > 1e-15) {
-      if(mesh->rank==0) printf("ogsGatherScatter test err=%g!\n", sum1); 
-      fflush(stdout);
-      err++;
-    }
-
-    mesh->ogs->o_invDegree.copyTo(tmp, Nlocal*sizeof(dfloat));
-    double *vmult = (double *) nek_ptr("vmult");
-    sum1 = 0;
-    for(int i=0; i<Nlocal; i++) sum1 += abs(tmp[i] - vmult[i]);
-    MPI_Allreduce(MPI_IN_PLACE, &sum1, 1, MPI_DFLOAT, MPI_SUM, mesh->comm);
-    if(sum1 > 1e-15) {
-      if(mesh->rank==0) printf("multiplicity test err=%g!\n", sum1); 
-      fflush(stdout);
-      err++;
+    const dfloat err = abs(sum1-sum2)/sum2;
+    if(err > 1e-15) {
+      if(mesh->rank==0) cout << "ogsGatherScatter test failed!\n"; 
+      exit(1);
     }
-
-    if(err) exit(1);
-    free(tmp);
   }
 
   return ins;

src/mesh/meshNekReader.cpp

@@ -34,7 +34,7 @@ void meshNekReaderHex3D(int N, mesh_t *mesh, int isMeshT){
   const int vtxmap[8] = {0, 1, 3, 2, 4, 5, 7, 6};
 
   // build vertex numbering
-  mesh->Nnodes = nek_set_glo_num(2, isMeshT);
+  mesh->Nnodes = nek_set_glo_num(mesh->Nverts, isMeshT);
 
   mesh->EToV 
     = (hlong*) calloc(mesh->Nelements * mesh->Nverts, sizeof(hlong));

src/mesh/meshSetup.cpp

@@ -2,8 +2,6 @@
 #include "bcMap.hpp"
 #include "meshNekReader.hpp"
 
-int isTmesh = 0;
-
 void meshVOccaSetup3D(mesh_t *mesh, setupAide &options, occa::properties &kernelInfo);
 
 mesh_t *createMeshDummy(MPI_Comm comm, int N, setupAide &options, occa::properties& kernelInfo) {
@@ -138,7 +136,7 @@ mesh_t *createMeshDummy(MPI_Comm comm, int N, setupAide &options, occa::properti
   libParanumal::meshSurfaceGeometricFactorsHex3D(mesh);
 
   // global nodes
-  meshParallelConnectNodes(mesh);
+  libParanumal::meshParallelConnectNodes(mesh);
 
   meshOccaSetup3D(mesh, options, kernelInfo);
 
@@ -189,9 +187,7 @@ mesh_t *createMeshT(MPI_Comm comm, int N, int isMeshT, setupAide &options, occa:
   libParanumal::meshSurfaceGeometricFactorsHex3D(mesh);
 
   // global nodes
-  isTmesh = 1;
-  meshParallelConnectNodes(mesh); 
-  isTmesh = 0;
+  libParanumal::meshParallelConnectNodes(mesh); 
 
   bcMap::check(mesh);
 
@@ -252,7 +248,7 @@ mesh_t *createMeshV(MPI_Comm comm, int N, mesh_t *meshT, setupAide &options, occ
 
   // uniquely label each node with a global index, used for gatherScatter
   // mesh->globalIds
-  meshParallelConnectNodes(mesh);
+  libParanumal::meshParallelConnectNodes(mesh);
 
   bcMap::check(mesh);
   meshVOccaSetup3D(mesh, options, kernelInfo);

src/nekInterface/nekInterface.f

@@ -101,8 +101,6 @@ subroutine nekf_ptr(ptr,id,len)
          ptr = loc(gett)
       elseif (id .eq. 'ifgetps') then
          ptr = loc(getps)
-      elseif (id .eq. 'vmult') then
-         ptr = loc(vmult)
       else
          write(6,*) 'ERROR: nek_ptr cannot find ', id
          call exitt 
@@ -535,7 +533,7 @@ integer function nekf_nbid(isTmsh)
       return
       end
 c-----------------------------------------------------------------------
-      integer*8 function nekf_set_vert(nx, isTmsh)
+      integer*8 function nekf_set_vert(npts, isTmsh)
 
       include 'SIZE'
       include 'TOTAL'
@@ -548,6 +546,7 @@ integer*8 function nekf_set_vert(nx, isTmsh)
 
       integer*8 ngv
 
+      nx  = npts**(1./ndim)
       nel = nelt
       if (isTmsh.eq.0) nel = nelv
       call set_vert(glo_num,ngv,nx,nel,vertex,.false.)
@@ -557,14 +556,3 @@ integer*8 function nekf_set_vert(nx, isTmsh)
       return
       end
 c-----------------------------------------------------------------------
-      subroutine nekf_dssum(u)
-      include 'SIZE'
-      include 'TOTAL'
-
-      ifld = ifield
-      ifield = 1
-      call dssum(u,lx1,ly1,lz1)
-      ifield = ifld 
-
-      return
-      end

src/nekInterface/nekInterfaceAdapter.cpp

@@ -40,7 +40,6 @@ static void (*nek_setics_ptr)(void);
 static int  (*nek_bcmap_ptr)(int *, int*);
 static int  (*nek_nbid_ptr)(int *);
 static long long (*nek_set_vert_ptr)(int *, int *);
-static void (*nek_dssum_ptr)(double *);
 
 void noop_func(void) {}
 
@@ -60,7 +59,7 @@ void nek_outfld(const char *suffix){
   (*nek_outfld_ptr)((char*)suffix);
 }
 
-void nek_outfld(const char *suffix, dfloat t, int coords, 
+void nek_outfld(const char *suffix, dfloat t, occa::memory o_x, 
                 occa::memory o_u, occa::memory o_p, occa::memory o_s, 
                 int NSfields, int FP64){
 
@@ -75,7 +74,7 @@ void nek_outfld(const char *suffix, dfloat t, int coords,
   int po = 0;
   int so = 0;
 
-  if(coords) {
+  if(o_x.ptr()) {
     xo = 1;
   }
   if(o_u.ptr()) {
@@ -262,8 +261,7 @@ void set_function_handles(const char *session_in,int verbose) {
   check_error(dlerror());
   nek_set_vert_ptr = (long long (*)(int *, int *)) dlsym(handle,fname("nekf_set_vert"));
   check_error(dlerror());
-  nek_dssum_ptr = (void (*)(double *)) dlsym(handle,fname("nekf_dssum"));
-  check_error(dlerror());
+
 
 #define postfix(x) x##_ptr
 #define load_or_noop(s) \
@@ -611,8 +609,10 @@ void nek_ocopyFrom(dfloat time, int tstep) {
 
 void nek_copyFrom(dfloat time, int tstep) {
 
-  nek_copyFrom(time);
-  *(nekData.istep) = tstep;
+  if(time != timeLast) {
+    nek_copyFrom(time);
+    *(nekData.istep) = tstep;
+  }
 }
 
 void nek_ocopyTo(dfloat &time) {
@@ -685,12 +685,6 @@ int nek_bcmap(int bid, int ifld) {
   return (*nek_bcmap_ptr)(&bid, &ifld);
 }
 
-long long nek_set_glo_num(int nx, int isTMesh) {
-  return (*nek_set_vert_ptr)(&nx, &isTMesh);
+long long nek_set_glo_num(int npts, int isTMesh) {
+  return (*nek_set_vert_ptr)(&npts, &isTMesh);
 }
-
-void nek_dssum(dfloat *u) {
-
-  (*nek_dssum_ptr)(u);
-}
-

src/nekInterface/nekInterfaceAdapter.hpp

@@ -89,7 +89,7 @@ DECLARE_USER_FUNC(userqtl)
 void*  nek_ptr(const char *id);
 void   nek_outfld(void);
 void   nek_outfld(const char *suffix);
-void   nek_outfld(const char *suffix, dfloat t, int coords, 
+void   nek_outfld(const char *suffix, dfloat t, occa::memory o_x, 
                   occa::memory o_u, occa::memory o_p, occa::memory o_s, 
                   int NSfields, int FP64);
 void   nek_uic(int ifield);
@@ -112,6 +112,5 @@ void nek_copyTo(dfloat &time);
 void nek_ocopyTo(dfloat &time);
 void nek_copyRestart();
 long long nek_set_glo_num(int npts, int isTMesh);
-void nek_dssum(dfloat *u);
 
 #endif