diff --git a/.docker/base.Dockerfile b/.docker/base.Dockerfile new file mode 100644 index 000000000..13af12510 --- /dev/null +++ b/.docker/base.Dockerfile @@ -0,0 +1,90 @@ +# Use Ubuntu Jammy (latest LTS) as the base image +FROM ubuntu:jammy + +# Install necessary tools, MPICH, UUID library and developer files +RUN apt-get update && apt-get install -y \ + build-essential \ + git \ + mpich \ + libmpich-dev \ + uuid \ + uuid-dev \ + autoconf \ + libtool \ + cmake \ + cmake-curses-gui \ + wget \ + axel \ + curl \ + vim \ + nano \ + gdb \ + cgdb \ + curl \ + valgrind + +# Set WORK_SPACE environment variable and create necessary directories +ENV WORK_SPACE=/home/codespace +RUN mkdir -p $WORK_SPACE + +# Clone the repositories +WORKDIR $WORK_SPACE/source +RUN git clone https://github.com/ofiwg/libfabric.git && \ + git clone https://github.com/mercury-hpc/mercury.git --recursive + +COPY ./ ${WORK_SPACE}/source/pdc + +ENV LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric +ENV MERCURY_SRC_DIR=$WORK_SPACE/source/mercury +ENV PDC_SRC_DIR=$WORK_SPACE/source/pdc +ENV LIBFABRIC_DIR=$WORK_SPACE/install/libfabric +ENV MERCURY_DIR=$WORK_SPACE/install/mercury +ENV PDC_DIR=$WORK_SPACE/install/pdc + +RUN mkdir -p $LIBFABRIC_SRC_DIR \ + mkdir -p $MERCURY_SRC_DIR \ + mkdir -p $PDC_SRC_DIR \ + mkdir -p $LIBFABRIC_DIR \ + mkdir -p $MERCURY_DIR \ + mkdir -p $PDC_DIR + + +# Save the environment variables to a file +RUN echo "export LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric" > $WORK_SPACE/pdc_env.sh && \ + echo "export MERCURY_SRC_DIR=$WORK_SPACE/source/mercury" >> $WORK_SPACE/pdc_env.sh && \ + echo "export PDC_SRC_DIR=$WORK_SPACE/source/pdc" >> $WORK_SPACE/pdc_env.sh && \ + echo "export LIBFABRIC_DIR=$WORK_SPACE/install/libfabric" >> $WORK_SPACE/pdc_env.sh && \ + echo "export MERCURY_DIR=$WORK_SPACE/install/mercury" >> $WORK_SPACE/pdc_env.sh && \ + echo "export PDC_DIR=$WORK_SPACE/install/pdc" >> $WORK_SPACE/pdc_env.sh + + +# Build and install libfabric +WORKDIR $LIBFABRIC_SRC_DIR +RUN git checkout v1.18.0 && \ + ./autogen.sh && \ + ./configure --prefix=$LIBFABRIC_DIR CC=mpicc CFLAG="-O2" && \ + make clean && \ + make -j && make install && \ + make check + +ENV LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH" +ENV PATH="$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH" +RUN echo 'export LD_LIBRARY_PATH=$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh && \ + echo 'export PATH=$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh + + +# Build and install Mercury +WORKDIR $MERCURY_SRC_DIR +ENV MERCURY_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DNA_OFI_TESTING_PROTOCOL=tcp " +RUN git checkout v2.2.0 \ + mkdir -p build +WORKDIR ${MERCURY_SRC_DIR}/build +RUN cmake $MERCURY_CMAKE_FLAGS ../ && \ + make -j && make install && \ + ctest + +# Set the environment variables +ENV LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" +ENV PATH="$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH" +RUN echo 'export LD_LIBRARY_PATH=$MERCURY_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh \ + echo 'export PATH=$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh \ No newline at end of file diff --git a/.docker/local.Dockerfile b/.docker/local.Dockerfile new file mode 100644 index 000000000..f5a0e0302 --- /dev/null +++ b/.docker/local.Dockerfile @@ -0,0 +1,29 @@ +# Note: Run `docker build -f .docker/Dockerfile -t pdc:latest .` from the root directory of the repository to build the docker image. + +# Use Ubuntu Jammy (latest LTS) as the base image +FROM zhangwei217245/pdc_dev_base:latest + +# Build and install PDC +ENV PDC_CMAKE_FLAGS="-DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DMPI_RUN_CMD=mpiexec " + + +WORKDIR $PDC_SRC_DIR +RUN rm -rf build && \ + mkdir -p build + +# COPY ../ ${PDC_SRC_DIR} +# RUN ls -l $PDC_SRC_DIR + +WORKDIR ${PDC_SRC_DIR}/build +RUN cmake $PDC_CMAKE_FLAGS ../ 2>&1 > ./cmake_config.log || echo "ignoring cmake config error and proceed" && \ + make -j && make install + +# Set the environment variables +ENV LD_LIBRARY_PATH="$PDC_DIR/lib:$LD_LIBRARY_PATH" +ENV PATH="$PDC_DIR/include:$PDC_DIR/lib:$PATH" +RUN echo 'export LD_LIBRARY_PATH=$PDC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh && \ + echo 'export PATH=$PDC_DIR/include:$PDC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh + + +# WORKDIR $PDC_SRC_DIR/build +# RUN ctest \ No newline at end of file diff --git a/.docker/local.Dockerfile.dockerignore b/.docker/local.Dockerfile.dockerignore new file mode 100644 index 000000000..c29c58f5a --- /dev/null +++ b/.docker/local.Dockerfile.dockerignore @@ -0,0 +1,2 @@ +# Exclude files and directories from the Docker build context +!/.git/ diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 5653602d9..c5c377bc8 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -1,72 +1,4 @@ -find_package(MERCURY REQUIRED) -if(PDC_ENABLE_FASTBIT) - add_definitions(-DENABLE_FASTBIT=1) - find_path(FASTBIT_INCLUDE_DIR fastbit/iapi.h) - find_library(FASTBIT_LIBRARY fastbit $ENV{HOME}/cori/fastbit-2.0.3/install) -endif() - -if(PDC_ENABLE_ROCKSDB) - add_definitions(-DENABLE_ROCKSDB=1) - find_path(ROCKSDB_INCLUDE_DIR include/db.h) - find_library(ROCKSDB_LIBRARY rocksdb 8.1.1< REQUIRED) -endif() - -if(PDC_ENABLE_SQLITE3) - add_definitions(-DENABLE_SQLITE3=1) - find_package(SQLite3 3.31.0 REQUIRED) -endif() - -include_directories( - ${PDC_COMMON_INCLUDE_DIRS} - ${PDC_INCLUDES_BUILD_TIME} - ${PROJECT_SOURCE_DIR} - ${PROJECT_BINARY_DIR} - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ${PDC_SOURCE_DIR}/src/server/include - ${PDC_SOURCE_DIR}/src/server/pdc_server_region - ${PDC_SOURCE_DIR}/src/server/pdc_server_region/include - ${PDC_SOURCE_DIR}/src/server/pdc_server_analysis/include - ${PDC_SOURCE_DIR}/src/server/dablooms - ${PDC_SOURCE_DIR}/src/api/include - ${PDC_SOURCE_DIR}/src/api/pdc_obj/include - ${PDC_SOURCE_DIR}/src/api/pdc_region/include - ${PDC_SOURCE_DIR}/src/api/pdc_query/include - ${PDC_SOURCE_DIR}/src/api/pdc_transform/include - ${PDC_SOURCE_DIR}/src/api/pdc_analysis/include - ${PDC_SOURCE_DIR}/src/api/profiling/include - ${PDC_SOURCE_DIR}/src/utils/include - ${MERCURY_INCLUDE_DIR} - ${FASTBIT_INCLUDE_DIR} - ${ROCKSDB_INCLUDE_DIR} -) - -add_definitions( -DIS_PDC_SERVER=1 ) -add_definitions( -DPDC_LUSTRE_TOTAL_OST=${PDC_LUSTRE_TOTAL_OST} ) -if(PDC_ENABLE_TIMING) - add_definitions(-DPDC_TIMING=1) -endif() - - -add_library(pdc_server_lib - pdc_server.c - pdc_server_metadata_index.c - pdc_server_metadata.c - pdc_client_server_common.c - dablooms/pdc_dablooms.c - dablooms/pdc_murmur.c - pdc_hash-table.c - ${PDC_SOURCE_DIR}/src/server/pdc_server_analysis/pdc_server_analysis.c - ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_data.c - ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_cache.c - ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer.c - ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c - ${PDC_SOURCE_DIR}/src/utils/pdc_region_utils.c - ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_analysis_common.c - ${PDC_SOURCE_DIR}/src/api/pdc_transform/pdc_transforms_common.c - ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_hist_pkg.c -) if(PDC_ENABLE_FASTBIT) message(STATUS "Enabled fastbit") target_link_libraries(pdc_server_lib ${MERCURY_LIBRARY} ${PDC_COMMONS_LIBRARIES} -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES} ${FASTBIT_LIBRARY}/libfastbit.so) @@ -80,27 +12,4 @@ elseif(PDC_ENABLE_SQLITE3) target_link_libraries(pdc_server_lib ${MERCURY_LIBRARY} ${PDC_COMMONS_LIBRARIES} -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES} SQLite::SQLite3) else() target_link_libraries(pdc_server_lib ${MERCURY_LIBRARY} ${PDC_COMMONS_LIBRARIES} -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES}) -endif() - -add_executable(pdc_server.exe - pdc_server_main.c -) -target_link_libraries(pdc_server.exe pdc_server_lib) - - -add_executable(pdc_server_metadata_index_test - pdc_server_metadata_index_test.c -) -target_link_libraries(pdc_server_metadata_index_test pdc_server_lib) - - -if(NOT ${PDC_INSTALL_BIN_DIR} MATCHES ${PROJECT_BINARY_DIR}/bin) -install( - TARGETS - pdc_server.exe - pdc_server_lib - LIBRARY DESTINATION ${PDC_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${PDC_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${PDC_INSTALL_BIN_DIR} -) -endif() +endif() \ No newline at end of file diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 95e90b146..d73fd9b9f 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -2384,4 +2384,4 @@ server_run(int argc, char *argv[]) MPI_Finalize(); #endif return 0; -} +} \ No newline at end of file diff --git a/src/server/pdc_server_metadata.c b/src/server/pdc_server_metadata.c index 1f5e58c3a..137516479 100644 --- a/src/server/pdc_server_metadata.c +++ b/src/server/pdc_server_metadata.c @@ -1845,7 +1845,7 @@ PDC_Server_query_kvtag_someta(pdc_kvtag_t *in, uint32_t *n_meta, uint64_t **obj_ *obj_ids = (void *)realloc(*obj_ids, alloc_size * sizeof(uint64_t)); } (*obj_ids)[iter++] = elt->obj_id; - break; + // break; // FIXME: shall we break here? or continue to check other kvtags? } } // End for each kvtag in list } // End for each metadata from hash table entry