Openmpi bugs

.github/workflows/CI.yml

@@ -1,17 +1,22 @@
 name: CI
+
 on:
  push:
  branches:
  - main
  tags: ['*']
  pull_request:
+
 concurrency:
  # Skip intermediate builds: always.
  # Cancel intermediate builds: only if it is a pull request build.
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
 jobs:
- test:
+
+ # default test
+ test-MPICH-jll:
  name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
  runs-on: ${{ matrix.os }}
  strategy:
@@ -27,7 +32,7 @@ jobs:
  arch:
  - x64
  steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
  - uses: actions/setup-java@v3
  with:
  distribution: 'temurin'
@@ -43,13 +48,155 @@ jobs:
  - uses: codecov/codecov-action@v2
  with:
  files: lcov.info
+
+
+ # test OpenMPI by requesting it with MPIPreferences
+ # adapted from MPI.jl
+ test-OpenMPI-jll:
+ name: Julia OpenMPI_jll - ${{ github.event_name }}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ version:
+ - '1.8'
+ os:
+ - ubuntu-latest
+ arch:
+ - x64
+
+ fail-fast: false
+ env:
+ JULIA_MPI_TEST_BINARY: OpenMPI_jll
+ JULIA_MPI_TEST_ABI: OpenMPI
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+
+ - uses: julia-actions/setup-julia@latest
+ with:
+ version: ${{ matrix.version }}
+ arch: ${{ matrix.arch }}
+ - uses: julia-actions/cache@v1
+
+ - name: use OpenMPI_jll
+ shell: julia --color=yes --project=test {0}
+ run: |
+ using Pkg
+ Pkg.instantiate()
+ using MPIPreferences
+ MPIPreferences.use_jll_binary("OpenMPI_jll")
+ rm("test/Manifest.toml")
+
+ - uses: julia-actions/julia-runtest@latest
+
+ # test system MPI using Brew in macOS
+ # adapted from MPI.jl
+ test-system-MPI-brew:
+ name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.mpi }} - ${{ github.event_name }}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ version:
+ - '1.8'
+ os:
+ - macos-latest
+ mpi:
+ - mpich
+ - openmpi
+ env:
+ JULIA_MPI_TEST_BINARY: system
+ ZES_ENABLE_SYSMAN: 1 # https://github.com/open-mpi/ompi/issues/10142
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Install MPI via homebrew
+ run: brew install $MPI
+ env:
+ MPI: ${{ matrix.mpi }}
+
+ - uses: actions/setup-java@v3
+ with:
+ distribution: 'temurin'
+ java-version: '11'
+
+ - uses: julia-actions/setup-julia@v1
+ with:
+ version: ${{ matrix.version }}
+ arch: x64
+
+ - uses: julia-actions/cache@v1
+
+ - name: use system MPI
+ shell: julia --color=yes --project=test {0}
+ run: |
+ using Pkg
+ Pkg.instantiate()
+ using MPIPreferences
+ MPIPreferences.use_system_binary()
+ run(`sed -i.bu 's/unknown/MPICH/' test/LocalPreferences.toml`) # fix wrong abi detection for mpich
+ rm("test/Manifest.toml")
+
+ - uses: julia-actions/julia-runtest@v1
+
+ # # test system MPI using apt in ubuntu
+ # # adapted from MPI.jl
+ # # TODO: commented out because apt has older versions of MPICH and OMPI that
+ # # segfault with multithreading. Re-introduce them when apt pkgs are upgraded 
+ # test-system-MPI-apt:
+ # name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.mpi }} - ${{ github.event_name }}
+ # runs-on: ${{ matrix.os }}
+ # strategy:
+ # fail-fast: false
+ # matrix:
+ # version:
+ # - '1.8'
+ # os:
+ # - ubuntu-latest
+ # mpi:
+ # - libmpich-dev
+ # env:
+ # JULIA_MPI_TEST_BINARY: system
+ # steps:
+ # - uses: actions/checkout@v3
+
+ # - name: Install MPI via apt
+ # run: |
+ # sudo apt-get update
+ # sudo apt-get install $MPI
+ # env:
+ # MPI: ${{ matrix.mpi }}
+
+ # - uses: actions/setup-java@v3
+ # with:
+ # distribution: 'temurin'
+ # java-version: '11'
+
+ # - uses: julia-actions/setup-julia@v1
+ # with:
+ # version: ${{ matrix.version }}
+ # arch: x64
+
+ # - uses: julia-actions/cache@v1
+
+ # - name: use system MPI
+ # shell: julia --color=yes --project=test {0}
+ # run: |
+ # using Pkg
+ # Pkg.instantiate()
+ # using MPIPreferences
+ # MPIPreferences.use_system_binary()
+ # rm("test/Manifest.toml")
+
+ # - uses: julia-actions/julia-runtest@v1
+
  docs:
  name: Documentation
  runs-on: ubuntu-latest
  permissions:
  contents: write
  steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
  - uses: actions/setup-java@v3
  with:
  distribution: 'temurin'

.gitignore

@@ -7,7 +7,8 @@
 .vscode/settings.json
 build
 .interfaces.md
-
+*.log
+*.err
 machines.txt
 results
 .includes_bu.jl

Project.toml

@@ -52,9 +52,3 @@ SpecialFunctions = "2"
 SplittableRandoms = "0.1"
 StatsBase = "0.33"
 julia = "1.6"
-
-[extras]
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-
-[targets]
-test = ["Test"]

src/Pigeons.jl

@@ -8,7 +8,7 @@ import MPI: Comm, Allreduce, Comm_rank,
  Comm_dup, Request, Waitall,
  RequestSet, mpiexec, Allreduce, 
  Allgather, Comm_split, isend, recv,
- bcast, tag_ub
+ bcast, tag_ub 
 
 
 using Base: Forward

src/mpi_utils/Entangler.jl

@@ -73,7 +73,7 @@ mutable struct Entangler
  println("Entangler initialized 1 process (without MPI); $(Threads.nthreads())")
  end
  else
- Init(threadlevel = :funneled) 
+ init_mpi()
  comm = Comm_dup(parent_communicator)
  transmit_counter_bound = ceil(Int, tag_ub() / n_global_indices - 2)
  my_process_index = Comm_rank(comm) + 1
@@ -101,10 +101,12 @@ mpi_active() =
  if silence_mpi[] 
  false
  else 
- Init(threadlevel = :funneled)
+ init_mpi()
  Comm_size(COMM_WORLD) > 1
  end
 
+init_mpi() = Init(threadlevel = :funneled)
+
 """
 $SIGNATURES
 
@@ -155,6 +157,8 @@ function transmit!(e::Entangler, source_data::AbstractVector{T}, to_global_indic
  # indicators of whether each local index is to be received over MPI
  e.current_received_bits .= true 
  at_least_one_mpi = false
+
+ requests = RequestSet() # non-blocking requests that will be waited on
 
  # send (or copy if local)
  for local_index in 1:myload
@@ -172,13 +176,15 @@ function transmit!(e::Entangler, source_data::AbstractVector{T}, to_global_indic
  source_view = Ref{T}(source_datum)
  mpi_rank = process_index - 1
  # asynchronously (non-blocking) send over MPI:
- Isend(source_view, e.communicator, dest = mpi_rank, tag = tag(e, transmit_index, global_index))
+ # note: we wait for the Isend request to avoid the application 
+ # terminating in the last iteration without completing its request.
+ request = Isend(source_view, e.communicator, dest = mpi_rank, tag = tag(e, transmit_index, global_index))
+ push!(requests, request)
  end
  end
 
  # receive
  if at_least_one_mpi
- requests = RequestSet()
  my_globals = my_global_indices(e.load)
  for local_index in 1:myload
  if e.current_received_bits[local_index]
@@ -238,6 +244,8 @@ function reduce_deterministically(operation, source_data::AbstractVector{T}, e::
  # outer loop is over the levels of a binary tree over the global indices
  iteration = 1
 
+ requests = RequestSet()
+
  while n_remaining_to_reduce > 1
  transmit_index = next_transmit_index!(e)
  current_local = my_first_remaining_local
@@ -251,7 +259,8 @@ function reduce_deterministically(operation, source_data::AbstractVector{T}, e::
  dest_global_index = current_global - spacing 
  dest_process = find_process(e.load, dest_global_index)
  dest_rank = dest_process - 1
- isend(work_array[current_local], e.communicator; dest = dest_rank, tag = tag(e, transmit_index, iteration))
+ request = isend(work_array[current_local], e.communicator; dest = dest_rank, tag = tag(e, transmit_index, iteration))
+ push!(requests, request)
  current_local += spacing 
  did_send = true 
  elseif current_global + spacing ≤ e.load.n_global_indices
@@ -274,6 +283,7 @@ function reduce_deterministically(operation, source_data::AbstractVector{T}, e::
 
  if did_send 
  my_first_remaining_local += spacing
+ Waitall(requests)
  end
  n_global_indices_remaining_before = ceil(Int, n_global_indices_remaining_before/2)
  spacing = spacing * 2

src/pt/pigeons.jl

@@ -10,6 +10,8 @@ This will also call [`report()`](@ref), [`write_checkpoint()`](@ref),
 and [`run_checks()`](@ref) between rounds. 
 """
 function pigeons(pt::PT) 
+ @warn "TEMPORARY - GC DISABLED FOR TESTING"
+ GC.enable(false)
  preflight_checks(pt)
  while next_round!(pt) # NB: while-loop instead of for-loop to support resuming from checkpoint
  reduced_recorders = run_one_round!(pt)

src/submission/ChildProcess.jl

@@ -19,7 +19,7 @@ $FIELDS
  (if of type `String`) needed by the child 
  process. 
  """
- dependencies::Vector{Any} = []
+ dependencies::Vector = []
  # eventually, detect & save which 
  # modules should be loaded? E.g. could use 
  # https://stackoverflow.com/questions/25575406/list-of-loaded-imported-packages-in-julia
@@ -34,15 +34,22 @@ $FIELDS
  third-party target distribution which somehow 
  does not support multi-threading. 
  """
- n_local_mpi_processes = 1
+ n_local_mpi_processes::Int = 1
 
  """
  If wait is false, the process runs asynchronously.
  When wait is false, the process' I/O streams are directed to devnull.
  """
- wait = true
+ wait::Bool = true
+
+ """
+ Extra arguments passed to mpiexec.
+ """
+ mpiexec_args::Cmd = ``
 end 
 
+
+
 """
 $SIGNATURES 
 
@@ -62,21 +69,26 @@ function pigeons(pt_arguments, new_process::ChildProcess)
  run(julia_cmd, wait = new_process.wait)
  else
  mpiexec() do exe
- mpi_cmd = `$exe -n $(new_process.n_local_mpi_processes)`
- cmd = `$mpi_cmd $julia_cmd`
+ mpi_cmd = `$exe $(new_process.mpiexec_args) -n $(new_process.n_local_mpi_processes)`
+ cmd  = `$mpi_cmd $julia_cmd`
  run(cmd, wait = new_process.wait)
  end
  end
  return Result{PT}(exec_folder)
 end
 
 function launch_cmd(pt_arguments, exec_folder, dependencies, n_threads::Int, silence_mpi::Bool)
- julia_bin = Base.julia_cmd()
- script_path = launch_script(pt_arguments, exec_folder, dependencies, silence_mpi)
- return `$julia_bin 
- --project 
- --threads=$n_threads 
- $script_path`
+ script_path = launch_script(pt_arguments, exec_folder, dependencies, silence_mpi)
+ jl_cmd = Base.julia_cmd()
+ project_file = Base.current_project()
+ if !isnothing(project_file)
+ # forcing instantiate the project to make sure dependencies exist
+ # also, precompile to avoid issues with coordinating access to compile cache
+ project_dir = dirname(project_file)
+ jl_cmd = `$jl_cmd --project=$project_dir`
+ run(`$jl_cmd -e "using Pkg; Pkg.instantiate(); Pkg.precompile()"`)
+ end
+ return `$jl_cmd --threads=$n_threads $script_path`
 end
 
 function launch_script(pt_arguments, exec_folder, dependencies, silence_mpi)
@@ -129,7 +141,6 @@ function launch_code(
 
  Pigeons.deserialize_immutables(raw"$path_to_serialized_immutables")
  pt_arguments = deserialize(raw"$path_to_serialized_pt_arguments")
-
  pt = PT(pt_arguments, exec_folder = raw"$exec_folder")
  pigeons(pt)
  """