Data Transformers and Data Loaders

CMakeLists.txt

@@ -156,7 +156,6 @@ else()
     message(FATAL_ERROR "Unknown RUNTIME_TYPE: ${RUNTIME_TYPE}. Supported values are 'STARPU' or 'PARSEC'.")
 endif()
 
-
 # ExaGeoStatCPP depends on LAPACK/BLASPP
 # -------------------------------
 include(ImportBLASPP)

cmake/FindHICMA-X.cmake

@@ -30,6 +30,7 @@ if(PKG_CONFIG_FOUND)
         endif()
         # TODO: This is not generalized for the case of hicma installed manually
         set(HICMA_X_SRC_DIR ${HICMA_X_ROOT}/hicma-x-src)
+        set(HICMA_X_BIN_DIR ${HICMA_X_ROOT}/bin)
         set(HICMA-X_FOUND TRUE)
         set(HICMA-X_LIBRARIES ${DPLASMA_PKG_LIBRARIES} ${PARSEC_PKG_LIBRARIES})
         set(HICMA-X_LIBRARY_DIRS "${HICMA_X_LIB_PATH}")

cmake/ImportHiCMAX.cmake

@@ -16,7 +16,9 @@ set(name "HICMA-X")
 set(tag "FIX-package-installation-MK")
 # Flags to configure the build for HiCMA-X, including precision settings for DPLASMA
 # and disabling GPU support for both CUDA and HIP.
-set(flags '-DDPLASMA_PRECISIONS="s;d"' \-DPARSEC_WITH_DEVEL_HEADERS=ON \-DCMAKE_Fortran_FLAGS="-Wno-main" \-DPARSEC_GPU_WITH_HIP=OFF \-DPARSEC_GPU_WITH_CUDA=OFF \-DPARSEC_HAVE_CUDA=OFF \-DPARSEC_DIST_SHORT_LIMIT=0 \-DPARSEC_DIST_COLLECTIVES=ON \-DPARSEC_HAVE_DEV_CUDA_SUPPORT=OFF \-DDPLASMA_HAVE_CUDA=OFF)
+set(flags '-DDPLASMA_PRECISIONS="s;d"' \-DPARSEC_WITH_DEVEL_HEADERS=ON \-DCMAKE_Fortran_FLAGS="-Wno-main"
+    \-DPARSEC_GPU_WITH_HIP=OFF \-DPARSEC_GPU_WITH_CUDA=OFF \-DPARSEC_HAVE_CUDA=OFF \-DPARSEC_DIST_SHORT_LIMIT=0
+    \-DPARSEC_DIST_COLLECTIVES=ON \-DPARSEC_HAVE_DEV_CUDA_SUPPORT=OFF \-DDPLASMA_HAVE_CUDA=OFF \-DBLA_VENDOR=${BLA_VENDOR})
 # Indicates that HiCMA-X uses CMake for its build system.
 set(is_cmake ON)
 # Indicates that HiCMA-X is hosted on a Git repository.

cmake/ImportStarsH.cmake

@@ -31,7 +31,7 @@ elseif(RUNTIME_TYPE STREQUAL "PARSEC")
 endif()
 
 # 'flag' is used for additional build configuration options, specifically disabling StarPU and optionally enabling MPI.
-set(flag \-DSTARPU=OFF \-DMPI=${USE_MPI})
+set(flag \-DSTARPU=OFF \-DMPI=${USE_MPI} \-DBLA_VENDOR=${BLA_VENDOR})
 # 'is_cmake' indicates that STARSH uses CMake as its build system, set to ON.
 set(is_cmake ON)
 # 'is_git' denotes that the source code for STARSH is hosted on a Git repository, set to ON.

examples/climate-emulator/ClimateEmulator.cpp

@@ -27,17 +27,12 @@ int main(int argc, char **argv) {
     configurations.InitializeArguments(argc, argv);
     // Initialize the ExaGeoStat Hardware
     auto hardware = ExaGeoStatHardware(configurations);
-
-//    // Create a unique pointer to hold the data.
-//    std::unique_ptr<ExaGeoStatData<double>> data;
-//    // Transform and prepare the data.
-//    ExaGeoStat<double>::ExaGeoStatTransformData(configurations, data);
-//    // Load the data, either by reading from a file or generating synthetic data.
-//    ExaGeoStat<double>::ExaGeoStatLoadData(configurations, data);
-//    // Perform data modeling.
-//    ExaGeoStat<double>::ExaGeoStatDataModeling(configurations, data);
-//    // Analyze the data.
-//    ExaGeoStat<double>::ExaGeoStatDataAnalyzer(configurations, data);
+    // Create a unique pointer to hold the data.
+    std::unique_ptr<ExaGeoStatData<double>> data;
+    // Load the data, either by reading from a file or generating synthetic data.
+    ExaGeoStat<double>::ExaGeoStatLoadData(configurations, data);
+    // Perform data modeling.
+    ExaGeoStat<double>::ExaGeoStatDataModeling(configurations, data);
 
     return 0;
 }

inst/include/api/ExaGeoStat.hpp

@@ -14,8 +14,6 @@
 #ifndef EXAGEOSTATCPP_EXAGEOSTAT_HPP
 #define EXAGEOSTATCPP_EXAGEOSTAT_HPP
 
-#include <nlopt.hpp>
-
 #include <configurations/Configurations.hpp>
 #include <data-units/ExaGeoStatData.hpp>
 
@@ -51,17 +49,6 @@ namespace exageostat::api {
                                         std::unique_ptr<ExaGeoStatData<T>> &aData,
                                         T *apMeasurementsMatrix = nullptr);
 
-
-        /**
-         * @brief Objective function used in optimization, and following the NLOPT objective function format.
-         * @param[in] aTheta An array of length n containing the current point in the parameter space.
-         * @param[in] aGrad  An array of length n where you can optionally return the gradient of the objective function.
-         * @param[in] apInfo pointer containing needed configurations and data.
-         * @return double MLE results.
-         *
-         */
-        static double ModelingAPI(const std::vector<double> &aTheta, std::vector<double> &aGrad, void *apInfo);
-
         /**
          * @brief Predict missing measurements values.
          * @param[in] aConfigurations Reference to Configurations object containing user input data.
@@ -76,26 +63,6 @@ namespace exageostat::api {
         ExaGeoStatPrediction(configurations::Configurations &aConfigurations, std::unique_ptr<ExaGeoStatData<T>> &aData,
                              T *apMeasurementsMatrix = nullptr, dataunits::Locations<T> *apTrainLocations = nullptr,
                              dataunits::Locations<T> *apTestLocations = nullptr);
-       /**
-        * @brief Transform data into the required format for ExaGeoStat operations.
-        * @param[in] aConfigurations Reference to Configurations object containing user input data.
-        * @param[in, out] aData Reference to an ExaGeoStatData<T> object containing the descriptors and locations to be transformed.
-        * @return void
-        *
-        */
-        static void
-        ExaGeoStatTransformData(configurations::Configurations &aConfigurations, std::unique_ptr<ExaGeoStatData<T>> &aData);
-
-        /**
-         * @brief Analyze the data to extract insights and patterns.
-         * @param[in] aConfigurations Reference to Configurations object containing user input data.
-         * @param[in, out] aData Reference to an ExaGeoStatData<T> object that contains the data to be analyzed.
-         * @return void
-         *
-         */
-         static void
-         ExaGeoStatDataAnalyzer(configurations::Configurations &aConfigurations, std::unique_ptr<ExaGeoStatData<T>> &aData);
-
     };
 
     /**

inst/include/common/Definitions.hpp

@@ -131,7 +131,8 @@ namespace exageostat::common {
      */
     enum DescriptorType {
         CHAMELEON_DESCRIPTOR = 0,
-        HICMA_DESCRIPTOR = 1
+        HICMA_DESCRIPTOR = 1,
+        PARSEC_DESCRIPTOR = 2
     };
 
     /**
@@ -141,7 +142,8 @@ namespace exageostat::common {
      */
     enum DataSourceType {
         SYNTHETIC = 0,
-        CSV_FILE = 1
+        CSV_FILE = 1,
+        PARSEC_FILE = 2
     };
 
     /**
@@ -205,6 +207,21 @@ namespace exageostat::common {
         DESCRIPTOR_SUM = 52,
         DESCRIPTOR_R = 53,
         DESCRIPTOR_R_COPY = 54,
+        DESCRIPTOR_F_DATA = 55,
+        DESCRIPTOR_ET1 = 56,
+        DESCRIPTOR_ET2 = 57,
+        DESCRIPTOR_EP = 58,
+        DESCRIPTOR_SLMN = 59,
+        DESCRIPTOR_IE = 60,
+        DESCRIPTOR_IO = 61,
+        DESCRIPTOR_P = 62,
+        DESCRIPTOR_D = 63,
+        DESCRIPTOR_FLMERA = 64,
+        DESCRIPTOR_ZLM = 65,
+        DESCRIPTOR_SC = 66,
+        DESCRIPTOR_F_SPATIAL = 67,
+        DESCRIPTOR_FLM = 68,
+        DESCRIPTOR_FLMT = 69
     };
 
     /**

inst/include/data-analyzer/DataAnalyzer.hpp

@@ -27,35 +27,39 @@ namespace exageostat::analyzer{
     class DataAnalyzer {
 
     public:
-        /**
-         * @brief Default constructor.
-         */
-        DataAnalyzer();
 
         /**
-         * @brief Default destructor.
+         * @brief Analyzes the given matrix data pre computation.
+         * @param[in, out] aData Reference to an ExaGeoStatData object that contains matrix to be analyzed.
+         * @return void
+         *
          */
-        ~DataAnalyzer();
+        static void PreAnalyzeMatrix(std::unique_ptr<ExaGeoStatData<T>> &aData);
 
         /**
-         * @brief Analyzes the given matrix data.
-         * @param[in] aConfigurations Reference to Configurations object containing needed parameters.
+         * @brief Analyzes the given matrix data post computation.
          * @param[in, out] aData Reference to an ExaGeoStatData object that contains matrix to be analyzed.
          * @return void
          *
          */
-        static void AnalyzeMatrix(configurations::Configurations &aConfigurations, std::unique_ptr<ExaGeoStatData<T>> &aData);
+        static void PostAnalyzeMatrix(std::unique_ptr<ExaGeoStatData<T>> &aData);
 
         /**
-         * @brief Compares betweent two matrices by getting the difference.
-         * @param[in] aConfigurations Reference to Configurations object containing needed parameters.
+         * @brief Compares between two matrices by getting the difference.
          * @param[in, out] aData Reference to an ExaGeoStatData object that contains matrix to be analyzed.
          * @return the calculated MSE.
          *
          */
-        static double CompareMatDifference(configurations::Configurations &aConfigurations, std::unique_ptr<ExaGeoStatData<T>> &aData);
+        static double CompareMatDifference(std::unique_ptr<ExaGeoStatData<T>> &aData);
     };
 
+    /**
+     * @brief Instantiates the ExaGeoStat class for float and double types.
+     * @tparam T Data Type: float or double
+     *
+     */
+     EXAGEOSTAT_INSTANTIATE_CLASS(DataAnalyzer)
+
 }//namespace exageostat
 
 #endif // EXAGEOSTATCPP_DATAANALYZER_HPP

inst/include/data-generators/DataGenerator.hpp

@@ -61,8 +61,8 @@ namespace exageostat::generators {
 
     protected:
 
-        /// Used enum for data generators types.
-        static common::DataSourceType aDataSourceType;
+        /// Used flag to determine if data generated is synthetic.
+        static bool aIsSynthetic;
     };
 
     /**

inst/include/data-loader/DataLoader.hpp

@@ -67,6 +67,35 @@ namespace exageostat::dataLoader {
         virtual void
         WriteData(const T &aMatrixPointer, const int &aProblemSize, const int &aP, std::string &aLoggerPath,
                   exageostat::dataunits::Locations<T> &aLocations) = 0;
+
+        /**
+         * @brief Abstract method for loading data based on provided configurations and kernel.
+         * @param[in] aConfigurations Reference to the configurations object that contains parameters for loading data.
+         * @param[in] aKernel Reference to the kernel object that defines the operations to be applied while loading the data.
+         * @return A unique pointer to the loaded ExaGeoStatData object.
+         *
+         */
+        virtual std::unique_ptr<ExaGeoStatData<T>>
+        LoadData(configurations::Configurations &aConfigurations, exageostat::kernels::Kernel<T> &aKernel)  = 0;
+
+        /**
+         * @brief Factory method for creating a DataLoader instance based on the given configurations.
+         * This method dynamically determines the type of data loader to instantiate based on compile-time conditions.
+         * @param[in] aConfigurations Reference to the configurations object that contains parameters for loading data.
+         * @return A unique pointer to a DataLoader instance configured as per the specified runtime conditions.
+         *
+         */
+        static std::unique_ptr<DataLoader<T>>
+        CreateDataLoader(exageostat::configurations::Configurations &apConfigurations);
+
+        /**
+         * @brief Releases the singleton instance of the currently active DataLoader.
+         * This method ensures proper deallocation of the singleton instance of the data loader,
+         * depending on the selected runtime.
+         *
+         */
+        static void ReleaseDataLoader();
+
     };
 
     /**

inst/include/data-loader/concrete/CSVLoader.hpp

@@ -5,7 +5,7 @@
 
 /**
  * @file  CSVLoader.hpp
- * @brief A class for generating synthetic data.
+ * @brief A class for loading csv format data.
  * @version 1.1.0
  * @author Mahmoud ElKarargy
  * @author Sameh Abdulah
@@ -54,6 +54,14 @@ namespace exageostat::dataLoader::csv {
                   exageostat::dataunits::Locations<T> &aLocations) override;
 
         /**
+        * @brief Loads data based on given configuration.
+        * @copydoc DataLoader::LoadData()
+        *
+        */
+       std::unique_ptr<ExaGeoStatData<T>>
+       LoadData(configurations::Configurations &aConfigurations, exageostat::kernels::Kernel<T> &aKernel) override;
+
+       /**
          * @brief Release the singleton instance of the  CSVLoader class.
          * @return void
          *