add parallel timers

Author: pmartorell

.gitignore

@@ -19,3 +19,4 @@ tmp/*
 *.nav
 *.snm
 *.toc
+*.jld2

src/Distributed.jl

@@ -1,29 +1,39 @@
 
 function cut(cutter::STLCutter,bgmodel::DistributedDiscreteModel,args...)
   D = map(num_dims,local_views(bgmodel)) |> PartitionedArrays.getany
-  cutter = STLCutter(;all_defined=false,cutter.options...)
+  timers,cutter = setup_distributed_cutter(cutter,bgmodel)
   cell_gids = get_cell_gids(bgmodel)
   facet_gids = get_face_gids(bgmodel,D-1)
 
+  tic!(timers["global"],barrier=true)
   bcells = map(compute_boundary_cells,local_views(bgmodel),local_views(cell_gids))
   icells = map(compute_interior_cells,local_views(bgmodel),local_views(cell_gids))
 
+  # Setup cutters
+  tic!(timers["fine"],barrier=true)
+  cell_to_facets = map(local_views(bgmodel)) do bgmodel
+    compute_cell_to_facets(bgmodel,args...)
+  end
+  toc!(timers["fine"],"setup")
+
   # Cut touched parts (only boundary cells)
+  tic!(timers["fine"],barrier=true)
   cuts = map(
     local_views(bgmodel),
-    local_views(cell_gids),
     local_views(facet_gids),
-    bcells) do bgmodel,cell_gids,facet_gids,bcells
+    bcells,
+    cell_to_facets) do bgmodel,facet_gids,bcells,cell_to_facets
 
-    # bcells = compute_boundary_cells(bgmodel,cell_gids)
     ownmodel = DiscreteModelPortion(bgmodel,bcells)
-    # ownmodel = remove_ghost_cells(bgmodel,cell_gids)
     cell_to_pcell = get_cell_to_parent_cell(ownmodel)
     facet_to_pfacet = get_face_to_parent_face(ownmodel,D-1)
-    cutgeo = cut(cutter,ownmodel,args...)
+    cell_to_facets = lazy_map(Reindex(cell_to_facets),bcells)
+    _cutter = STLCutter(;cell_to_facets,cutter.options...)
+    cutgeo = cut(_cutter,ownmodel,args...)
     cutgeo = change_bgmodel(cutgeo,bgmodel,cell_to_pcell,facet_to_pfacet)
     remove_ghost_subfacets(cutgeo,facet_gids)
   end
+  toc!(timers["fine"],"interface")
 
   # Setup coarse inout and graph
   facet_to_inoutcut = map(compute_bgfacet_to_inoutcut,cuts)
@@ -38,6 +48,8 @@ function cut(cutter::STLCutter,bgmodel::DistributedDiscreteModel,args...)
   part_to_lpart_to_ioc = gather(facet_neighbor_to_ioc,destination=root)
 
   # Propagate at coarse level (and complete intersections)
+  tic!(timers["coarse"],barrier=true)
+  tic!(timers["fine"],barrier=true)
   part_to_ioc = map(
       part_to_parts,
       part_to_lpart_to_ioc,
@@ -50,22 +62,26 @@ function cut(cutter::STLCutter,bgmodel::DistributedDiscreteModel,args...)
       Int8[]
     end
   end
+  toc!(timers["coarse"],"coarse")
 
   # Complete cut (interior cells)
   icuts = map(
     local_views(bgmodel),
-    local_views(cell_gids),
     local_views(facet_gids),
-    icells) do bgmodel,cell_gids,facet_gids,icells
+    icells,
+    cell_to_facets) do bgmodel,facet_gids,icells,cell_to_facets
 
     # icells = compute_interior_cells(bgmodel,cell_gids)
     ownmodel = DiscreteModelPortion(bgmodel,icells)
     cell_to_pcell = get_cell_to_parent_cell(ownmodel)
     facet_to_pfacet = get_face_to_parent_face(ownmodel,D-1)
-    cutgeo = cut(cutter,ownmodel,args...)
+    cell_to_facets = lazy_map(Reindex(cell_to_facets),icells)
+    _cutter = STLCutter(;cell_to_facets,cutter.options...)
+    cutgeo = cut(_cutter,ownmodel,args...)
     cutgeo = change_bgmodel(cutgeo,bgmodel,cell_to_pcell,facet_to_pfacet)
     remove_ghost_subfacets(cutgeo,facet_gids)
   end
+  toc!(timers["fine"],"interior")
 
   # Merge discretizations
   cuts = map(cuts,icuts,bcells,icells) do bcut,icut,bcells,icells
@@ -87,10 +103,37 @@ function cut(cutter::STLCutter,bgmodel::DistributedDiscreteModel,args...)
   # Nearest neighbor communication
   consistent_bgcell_to_inoutcut!(cuts,cell_gids)
   consistent_bgfacet_to_inoutcut!(cuts,facet_gids)
+
+  toc!(timers["global"],"global")
   DistributedEmbeddedDiscretization(cuts,bgmodel)
 end
 
 
+function setup_distributed_cutter(a::STLCutter,bgmodel::DistributedDiscreteModel)
+  parts = map(part_id,get_cell_gids(bgmodel).partition)
+  timers,options = _setup_distributed_cutter(parts;a.options...)
+  cutter = STLCutter(;all_defined=false,options...)
+  timers,cutter
+end
+
+function _setup_distributed_cutter(
+  parts;
+  verbose=false,
+  timers=default_timers(parts;verbose),
+  showprogress=false,
+  kwargs...)
+
+  kwargs = (;showprogress,kwargs...)
+  timers, kwargs
+end
+
+function default_timers(parts;kwargs...)
+  Dict(
+    "global" => PTimer(parts;kwargs...),
+    "coarse" => PTimer(parts;kwargs...),
+    "fine" => PTimer(parts;kwargs...))
+end
+
 """
     find_root_part(cuts::AbstractArray{<:AbstractEmbeddedDiscretization},cells)
 

src/Embedded.jl

@@ -39,6 +39,10 @@ function similar_geometry(a::STLGeometry,tree::Leaf)
   STLGeometry(tree)
 end
 
+function compute_cell_to_facets(a::DiscreteModel,geo::STLGeometry)
+  compute_cell_to_facets(get_grid(a),STL(get_stl(geo)))
+end
+
 function closest_point(x::AbstractVector,geo::STLGeometry,args...)
   closest_point(x,get_stl(geo),args...)
 end

src/SubTriangulations.jl

@@ -60,6 +60,7 @@ end
       - `:both`: retunrs boths `:interior` and `:exterior` `face_grid`. It must be filtered (only used for debugging purposes).
   - `showprogress`: (default `true`) show progress bar
   - `all_defined`: (default `true`) if `true` all cells are defined as `IN`, `OUT` or `CUT`. If `false` undefined cells are allowed (only used for distributed meshes)
+  - `cell_to_facets`: (default [`compute_cell_to_facets`](@ref)) precomputed cell to stl facets mapping.
 """
 function subtriangulate(
   bgmodel::DiscreteModel,
@@ -68,24 +69,23 @@ function subtriangulate(
   tolfactor=DEFAULT_TOL_FACTOR,
   surfacesource=:skin,
   showprogress=true,
-  all_defined=true)
+  all_defined=true,
+  cell_to_facets=compute_cell_to_facets(bgmodel,STL(stlmodel)))
   # cutfacets = false)
 
   grid = get_grid(bgmodel)
-  grid_topology = get_grid_topology(bgmodel)
   p = get_polytope(only(get_reffes(bgmodel)))
   node_to_coords = get_node_coordinates(grid)
   cell_to_nodes = get_cell_node_ids(grid)
   stl = STL(stlmodel)
-  D = num_dims(grid)
   atol = eps(grid)*tolfactor
 
   f_to_isempty = get_facet_to_isempty(stl;atol)
   Πf = get_facet_planes(stl)
   Πf = correct_small_facets_planes!(stl,Πf,f_to_isempty;atol)
   Πr = get_reflex_planes(stl,Πf)
 
-  c_to_stlf = compute_cell_to_facets(grid,stl)
+  c_to_stlf = cell_to_facets
 
   Γ0 = Polyhedron(stl)
 

test/DistributedTests/CutterTests.jl

@@ -24,6 +24,9 @@ function main(distribute;
   end
 
   ranks = distribute(LinearIndices((prod(np),)))
+
+  timers = STLCutters.default_timers(ranks)
+
   geo = STLGeometry(filename)
 
   pmin,pmax = get_bounding_box(geo)
@@ -35,9 +38,13 @@ function main(distribute;
     bgmodel = simplexify(bgmodel,positive=true)
   end
 
-  cutter = STLCutter(;tolfactor)
+  cutter = STLCutter(;tolfactor,timers)
   cutgeo = cut(cutter,bgmodel,geo)
 
+  display(timers["coarse"])
+  display(timers["fine"])
+  display(timers["global"])
+
   Ωbg = Triangulation(bgmodel)
   Ωin = Triangulation(cutgeo,PHYSICAL_IN)
   Ωout = Triangulation(cutgeo,PHYSICAL_OUT)

test/DistributedTests/mpi/all_tests.jl

@@ -0,0 +1,23 @@
+
+include("../CutterTests.jl")
+include("../Poisson.jl")
+
+function all_tests(distribute,parts)
+  ranks = distribute(LinearIndices((prod(parts),)))
+
+  t = PArrays.PTimer(ranks,verbose=true)
+  PArrays.tic!(t)
+
+  DistributedCutterTests.main(distribute,np=parts,nc=(4,4,4))
+  DistributedCutterTests.main(distribute,np=parts,nc=(8,8,8))
+  DistributedCutterTests.main(distribute,np=parts,nc=(4,4,4),simplex=true)
+  DistributedCutterTests.main(distribute,np=parts,nc=(8,8,8),simplex=true)
+  PArrays.toc!(t,"MPIDistributedCutter")
+
+  DistributedPoissonTests.main(distribute,np=parts,nc=(4,4,4))
+  DistributedPoissonTests.main(distribute,np=parts,nc=(8,8,8))
+  DistributedPoissonTests.main(distribute,np=parts,nc=(8,8,8),geoname="Bunny-LowPoly")
+  PArrays.toc!(t,"MPIDistributedPoisson")
+
+  display(t)
+end

test/DistributedTests/mpi/runtests_body.jl

@@ -4,33 +4,12 @@ using PartitionedArrays
 const PArrays = PartitionedArrays
 using MPI
 
-include("../CutterTests.jl")
-include("../Poisson.jl")
+include("all_tests.jl")
 
 if ! MPI.Initialized()
   MPI.Init()
 end
 
-function all_tests(distribute,parts)
-  ranks = distribute(LinearIndices((prod(parts),)))
-
-  t = PArrays.PTimer(ranks,verbose=true)
-  PArrays.tic!(t)
-
-  DistributedCutterTests.main(distribute,np=parts,nc=(4,4,4))
-  DistributedCutterTests.main(distribute,np=parts,nc=(8,8,8))
-  DistributedCutterTests.main(distribute,np=parts,nc=(4,4,4),simplex=true)
-  DistributedCutterTests.main(distribute,np=parts,nc=(8,8,8),simplex=true)
-  PArrays.toc!(t,"MPIDistributedCutter")
-
-  DistributedPoissonTests.main(distribute,np=parts,nc=(4,4,4))
-  DistributedPoissonTests.main(distribute,np=parts,nc=(8,8,8))
-  DistributedPoissonTests.main(distribute,np=parts,nc=(8,8,8),geoname="Bunny-LowPoly")
-  PArrays.toc!(t,"MPIDistributedPoisson")
-
-  display(t)
-end
-
 if MPI.Comm_size(MPI.COMM_WORLD) == 8
   with_mpi() do distribute
     all_tests(distribute,(2,2,2))