From a3287fd42f0829d5162a24bd3b5e88af3530e39b Mon Sep 17 00:00:00 2001
From: Carlos Segarra <carlos@carlossegarra.com>
Date: Wed, 16 Jun 2021 11:03:27 +0000
Subject: [PATCH] adding more tests

---
 .github/workflows/tests.yml                   |  2 +-
 .../test/scheduler/test_remote_mpi_worlds.cpp | 57 +++++++++++++++++++
 2 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 58b8536ba..bc80693a9 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -68,7 +68,7 @@ jobs:
         run: inv dev.cc faabric_tests
       # --- Tests ---
       - name: "Run tests"
-        run: ./bin/faabric_tests
+        run: LOG_LEVEL=trace ./bin/faabric_tests
         working-directory: /build/faabric/static
 
   dist-tests:
diff --git a/tests/test/scheduler/test_remote_mpi_worlds.cpp b/tests/test/scheduler/test_remote_mpi_worlds.cpp
index 5436f2eb9..2a1512cb2 100644
--- a/tests/test/scheduler/test_remote_mpi_worlds.cpp
+++ b/tests/test/scheduler/test_remote_mpi_worlds.cpp
@@ -94,6 +94,63 @@ TEST_CASE_METHOD(RemoteMpiTestFixture, "Test send across hosts", "[mpi]")
     localWorld.destroy();
 }
 
+TEST_CASE_METHOD(RemoteMpiTestFixture,
+                 "Test send and recv across hosts",
+                 "[mpi]")
+{
+    // Register two ranks (one on each host)
+    this->setWorldsSizes(2, 1, 1);
+    int rankA = 0;
+    int rankB = 1;
+    std::vector<int> messageData = { 0, 1, 2 };
+    std::vector<int> messageData2 = { 3, 4, 5 };
+
+    // Init worlds
+    MpiWorld& localWorld = getMpiWorldRegistry().createWorld(msg, worldId);
+    faabric::util::setMockMode(false);
+
+    std::thread senderThread([this, rankA, rankB, &messageData, &messageData2] {
+        remoteWorld.initialiseFromMsg(msg);
+
+        // Send a message that should get sent to this host
+        remoteWorld.send(
+          rankB, rankA, BYTES(messageData.data()), MPI_INT, messageData.size());
+
+        // Now recv
+        auto buffer = new int[messageData2.size()];
+        remoteWorld.recv(rankA,
+                         rankB,
+                         BYTES(buffer),
+                         MPI_INT,
+                         messageData2.size(),
+                         MPI_STATUS_IGNORE);
+        std::vector<int> actual(buffer, buffer + messageData2.size());
+        REQUIRE(actual == messageData2);
+
+        remoteWorld.destroy();
+    });
+
+    // Receive the message for the given rank
+    MPI_Status status{};
+    auto buffer = new int[messageData.size()];
+    localWorld.recv(
+      rankB, rankA, BYTES(buffer), MPI_INT, messageData.size(), &status);
+    std::vector<int> actual(buffer, buffer + messageData.size());
+    REQUIRE(actual == messageData);
+
+    // Now send a message
+    localWorld.send(
+      rankA, rankB, BYTES(messageData2.data()), MPI_INT, messageData2.size());
+
+    REQUIRE(status.MPI_SOURCE == rankB);
+    REQUIRE(status.MPI_ERROR == MPI_SUCCESS);
+    REQUIRE(status.bytesSize == messageData.size() * sizeof(int));
+
+    // Destroy worlds
+    senderThread.join();
+    localWorld.destroy();
+}
+
 TEST_CASE_METHOD(RemoteMpiTestFixture,
                  "Test sending many messages across host",
                  "[mpi]")