Merge pull request #3376 from pnorbert/ulimit

Raise the limit of open filesi to max in BP file engines
ornladios · Oct 25, 2022 · c9335fd · c9335fd
2 parents cda6a53 + 4b739ca
commit c9335fd
Show file tree

Hide file tree

Showing 5 changed files with 68 additions and 0 deletions.
diff --git a/source/adios2/engine/bp3/BP3Reader.cpp b/source/adios2/engine/bp3/BP3Reader.cpp
@@ -156,6 +156,7 @@ void BP3Reader::Init()
     // if IO was involved in reading before this flag may be true now
     m_IO.m_ReadStreaming = false;
 
+    helper::RaiseLimitNoFile();
     InitTransports();
     InitBuffer();
 }

diff --git a/source/adios2/engine/bp4/BP4Reader.cpp b/source/adios2/engine/bp4/BP4Reader.cpp
@@ -193,6 +193,8 @@ void BP4Reader::Init()
     m_BP4Deserializer.Init(m_IO.m_Parameters, "in call to BP4::Open to write");
     InitTransports();
 
+    helper::RaiseLimitNoFile();
+
     /* Do a collective wait for the file(s) to appear within timeout.
        Make sure every process comes to the same conclusion */
     const Seconds timeoutSeconds(

diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp
@@ -503,6 +503,12 @@ void BP5Reader::InitParameters()
             m_Threads = helper::SetWithinLimit(8U / NodeSize, 1U, 8U);
         }
     }
+
+    size_t limit = helper::RaiseLimitNoFile();
+    if (m_Parameters.MaxOpenFilesAtOnce > limit - 8)
+    {
+        m_Parameters.MaxOpenFilesAtOnce = limit - 8;
+    }
 }
 
 bool BP5Reader::SleepOrQuit(const TimePoint &timeoutInstant,

diff --git a/source/adios2/helper/adiosSystem.cpp b/source/adios2/helper/adiosSystem.cpp
@@ -14,6 +14,11 @@
 #include <system_error>
 #include <thread>
 
+#ifndef _WIN32
+#include <sys/resource.h> // getrlimits, setrlimits
+#include <sys/time.h>
+#endif
+
 #include <adios2sys/SystemTools.hxx>
 
 #include "adios2/common/ADIOSTypes.h"
@@ -187,5 +192,54 @@ unsigned int NumHardwareThreadsPerNode()
     return std::thread::hardware_concurrency();
 }
 
+size_t RaiseLimitNoFile()
+{
+#ifdef _WIN32
+    return _setmaxstdio(8192);
+#else
+    static size_t raisedLimit = 0;
+    static bool firstCallRaiseLimit = true;
+
+    if (firstCallRaiseLimit)
+    {
+        struct rlimit limit;
+        errno = 0;
+        int err = getrlimit(RLIMIT_NOFILE, &limit);
+        raisedLimit = limit.rlim_cur;
+        if (!err)
+        {
+            /*std::cout
+                << "adios2::helper::RaiseLimitNoFile() found limits soft = "
+                << limit.rlim_cur << " hard = " << limit.rlim_max <<
+               std::endl;*/
+            if (limit.rlim_cur < limit.rlim_max)
+            {
+                limit.rlim_cur = limit.rlim_max;
+                err = setrlimit(RLIMIT_NOFILE, &limit);
+                if (!err)
+                {
+                    getrlimit(RLIMIT_NOFILE, &limit);
+                    raisedLimit = limit.rlim_cur;
+                    /*std::cout << "adios2::helper::RaiseLimitNoFile() set "
+                                 "limits soft = "
+                              << limit.rlim_cur << " hard = " << limit.rlim_max
+                              << std::endl;*/
+                }
+            }
+        }
+
+        if (err)
+        {
+            std::cerr << "adios2::helper::RaiseLimitNoFile(soft="
+                      << limit.rlim_cur << ", hard=" << limit.rlim_max
+                      << ") failed with error code " << errno << ": "
+                      << strerror(errno) << std::endl;
+        }
+        firstCallRaiseLimit = false;
+    }
+    return raisedLimit;
+#endif
+}
+
 } // end namespace helper
 } // end namespace adios2
diff --git a/source/adios2/helper/adiosSystem.h b/source/adios2/helper/adiosSystem.h
@@ -85,6 +85,11 @@ char BPVersion(const std::string &name, helper::Comm &comm,
  */
 unsigned int NumHardwareThreadsPerNode();
 
+/** Attempt to raise the limit of number of files opened at once
+ *  Return: the limit on number of open files
+ */
+size_t RaiseLimitNoFile();
+
 } // end namespace helper
 } // end namespace adios2