Skip to content

Commit

Permalink
Monitoring changes from feature/dart to develop (#18)
Browse files Browse the repository at this point in the history 
Major changes: 
* IDIOMS -> affix-based query benchmark
* IDIOMS -> Simulation Test
* IDIOMS -> Multi data type supported for the same key
* IDIOMS -> Range Query and Exact Query for Numeric Values
* IDIOMS -> benchmark for numeric values (exact search and range query)
* IDIOMS -> Index Persistence
* BULKI -> A data serialization and deserialization mechanism.
  • Loading branch information
@zhangwei217245
zhangwei217245 authored Jun 24, 2024
1 parent 5575907 commit 45562e0
Show file tree
Hide file tree
Showing 106 changed files with 16,852 additions and 2,868 deletions.
170 changes: 76 additions & 94 deletions .docker/dev_base.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,55 @@
ARG ARCH
FROM ${ARCH}ubuntu:jammy

RUN echo "ARCH=${ARCH}" && sleep 3

ARG ARCH_CODE

RUN echo "ARCH_CODE=${ARCH_CODE}" && sleep 3
ENV JULIA_HOME=/opt/julia
ENV WORK_SPACE=/home/project
ENV LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric
ENV LIBFABRIC_DIR=$WORK_SPACE/install/libfabric
ENV MERCURY_SRC_DIR=$WORK_SPACE/source/mercury
ENV MERCURY_DIR=$WORK_SPACE/install/mercury

ENV PDC_SRC_DIR=$WORK_SPACE/source/pdc
ENV PDC_DIR=$WORK_SPACE/install/pdc

ENV LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH"
ENV PATH="$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH"
ENV LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
ENV PATH="$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH"


ENV MERCURY_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DNA_OFI_TESTING_PROTOCOL=tcp "
ENV CLANG_FORMAT_PATH=$WORK_SPACE/software/clang-format-lint-action/clang-format/clang-format10

# Install necessary tools, MPICH, UUID library and developer files
RUN apt-get update && apt-get install -y \
RUN echo "ARCH=${ARCH}" && echo "ARCH_CODE=${ARCH_CODE}" && sleep 3 && mkdir -p $WORK_SPACE && \
mkdir -p $WORK_SPACE/software && \
mkdir -p $WORK_SPACE/source && \
mkdir -p $WORK_SPACE/install && \
mkdir -p $LIBFABRIC_SRC_DIR && \
mkdir -p $MERCURY_SRC_DIR && \
mkdir -p $PDC_SRC_DIR && \
mkdir -p $LIBFABRIC_DIR && \
mkdir -p $MERCURY_DIR && \
mkdir -p $PDC_DIR && \
rm -rf $LIBFABRIC_SRC_DIR/* && \
rm -rf $MERCURY_SRC_DIR/* && \
rm -rf $PDC_SRC_DIR/* && \
rm -rf $LIBFABRIC_DIR/* && \
rm -rf $MERCURY_DIR/* && \
rm -rf $PDC_DIR/* && \
echo "export LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric" > $WORK_SPACE/pdc_env.sh && \
echo "export LIBFABRIC_DIR=$WORK_SPACE/install/libfabric" >> $WORK_SPACE/pdc_env.sh && \
echo "export MERCURY_SRC_DIR=$WORK_SPACE/source/mercury" >> $WORK_SPACE/pdc_env.sh && \
echo "export MERCURY_DIR=$WORK_SPACE/install/mercury" >> $WORK_SPACE/pdc_env.sh && \
echo "export PDC_SRC_DIR=$WORK_SPACE/source/pdc" >> $WORK_SPACE/pdc_env.sh && \
echo "export PDC_DIR=$WORK_SPACE/install/pdc" >> $WORK_SPACE/pdc_env.sh && \
echo 'export LD_LIBRARY_PATH=$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh && \
echo 'export PATH=$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh && \
echo 'export LD_LIBRARY_PATH=$MERCURY_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh \
echo 'export PATH=$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh && \
apt-get update && apt-get install -y \
build-essential \
git \
mpich \
Expand All @@ -28,107 +70,47 @@ RUN apt-get update && apt-get install -y \
wget \
axel \
curl \
bc \
vim \
nano \
gdb \
cgdb \
curl \
valgrind \
python3

# Install Oh My Bash
RUN bash -c "$(curl -fsSL https://raw.githubusercontent.com/ohmybash/oh-my-bash/master/tools/install.sh)" && \
sed -i 's/OSH_THEME="font"/OSH_THEME="powerline-multiline"/g' ~/.bashrc

# Install Julia

RUN echo "https://julialang-s3.julialang.org/bin/linux/aarch64/1.6/julia-1.6.7-linux-aarch64.tar.gz" > /julia_url_arm64v8.txt && \
echo "https://julialang-s3.julialang.org/bin/linux/x64/1.6/julia-1.6.7-linux-x86_64.tar.gz" > /julia_url_amd64.txt

RUN echo $(cat /julia_url_${ARCH_CODE}.txt) && sleep 3

RUN mkdir -p /opt/julia && wget -O - $(cat /julia_url_${ARCH_CODE}.txt) | tar -xz -C /opt/julia --strip-components=1 && \
ln -s /opt/julia/bin/julia /usr/local/bin/julia

RUN rm -rf /tmp/julia_url_*.txt

ENV JULIA_HOME=/opt/julia

# Install Rust
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
RUN echo 'source $HOME/.cargo/env' >> ~/.bashrc


# Set WORK_SPACE environment variable and create necessary directories
ENV WORK_SPACE=/home/project
RUN mkdir -p $WORK_SPACE

# Install clang-format repo
RUN mkdir -p $WORK_SPACE/software
RUN cd $WORK_SPACE/software && git clone https://github.com/DoozyX/clang-format-lint-action.git
ENV CLANG_FORMAT_PATH=$WORK_SPACE/software/clang-format-lint-action/clang-format/clang-format10

# Clone the repositories
WORKDIR $WORK_SPACE/source
RUN git clone https://github.com/ofiwg/libfabric.git && \
git clone https://github.com/mercury-hpc/mercury.git --recursive

ENV LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric
ENV LIBFABRIC_DIR=$WORK_SPACE/install/libfabric
ENV MERCURY_SRC_DIR=$WORK_SPACE/source/mercury
ENV MERCURY_DIR=$WORK_SPACE/install/mercury

ENV PDC_SRC_DIR=$WORK_SPACE/source/pdc
ENV PDC_DIR=$WORK_SPACE/install/pdc

RUN mkdir -p $LIBFABRIC_SRC_DIR && \
mkdir -p $MERCURY_SRC_DIR && \
mkdir -p $PDC_SRC_DIR && \
mkdir -p $LIBFABRIC_DIR && \
mkdir -p $MERCURY_DIR && \
mkdir -p $PDC_DIR


# Save the environment variables to a file
RUN echo "export LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric" > $WORK_SPACE/pdc_env.sh && \
echo "export LIBFABRIC_DIR=$WORK_SPACE/install/libfabric" >> $WORK_SPACE/pdc_env.sh && \
echo "export MERCURY_SRC_DIR=$WORK_SPACE/source/mercury" >> $WORK_SPACE/pdc_env.sh && \
echo "export MERCURY_DIR=$WORK_SPACE/install/mercury" >> $WORK_SPACE/pdc_env.sh && \
echo "export PDC_SRC_DIR=$WORK_SPACE/source/pdc" >> $WORK_SPACE/pdc_env.sh && \
echo "export PDC_DIR=$WORK_SPACE/install/pdc" >> $WORK_SPACE/pdc_env.sh


# Build and install libfabric
WORKDIR $LIBFABRIC_SRC_DIR
RUN git checkout v1.18.0 && \
python3 && \
cd $WORK_SPACE/software && \
bash -c "$(curl -fsSL https://raw.githubusercontent.com/ohmybash/oh-my-bash/master/tools/install.sh)" && \
sed -i 's/OSH_THEME="font"/OSH_THEME="powerline-multiline"/g' ~/.bashrc && \
echo "https://julialang-s3.julialang.org/bin/linux/aarch64/1.6/julia-1.6.7-linux-aarch64.tar.gz" > $WORK_SPACE/software/julia_url_arm64v8.txt && \
echo "https://julialang-s3.julialang.org/bin/linux/x64/1.6/julia-1.6.7-linux-x86_64.tar.gz" > $WORK_SPACE/software/julia_url_amd64.txt && \
echo $(cat $WORK_SPACE/software/julia_url_${ARCH_CODE}.txt) && sleep 3 && \
mkdir -p /opt/julia && wget -O - $(cat $WORK_SPACE/software/julia_url_${ARCH_CODE}.txt) | tar -xz -C /opt/julia --strip-components=1 && \
ln -s /opt/julia/bin/julia /usr/local/bin/julia && \
rm -rf $WORK_SPACE/software/julia_url_*.txt && \
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
echo 'source $HOME/.cargo/env' >> ~/.bashrc && \
git clone https://github.com/DoozyX/clang-format-lint-action.git && \
git clone https://github.com/ofiwg/libfabric.git ${LIBFABRIC_SRC_DIR} && \
git clone https://github.com/mercury-hpc/mercury.git --recursive ${MERCURY_SRC_DIR} && \
cd $LIBFABRIC_SRC_DIR && \
git checkout v1.18.0 && \
./autogen.sh && \
./configure --prefix=$LIBFABRIC_DIR CC=mpicc CFLAG="-O2" && \
make clean && \
make -j 8 && make install && \
make check

ENV LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH"
ENV PATH="$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH"
RUN echo 'export LD_LIBRARY_PATH=$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh && \
echo 'export PATH=$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh


# Build and install Mercury
WORKDIR $MERCURY_SRC_DIR
ENV MERCURY_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DNA_OFI_TESTING_PROTOCOL=tcp "
RUN git checkout v2.2.0 \
mkdir -p build
WORKDIR ${MERCURY_SRC_DIR}/build
RUN cmake $MERCURY_CMAKE_FLAGS ../ && \
make check && \
cd $MERCURY_SRC_DIR && \
git checkout v2.2.0 && \
mkdir -p build && \
cd ${MERCURY_SRC_DIR}/build && \
cmake $MERCURY_CMAKE_FLAGS ../ && \
make -j 16 && make install && \
ctest

# Set the environment variables
ENV LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
ENV PATH="$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH"
RUN echo 'export LD_LIBRARY_PATH=$MERCURY_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh \
echo 'export PATH=$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh

ctest && \
apt-get autoremove -y && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
rm -rf /tmp/* && \
rm -rf /var/tmp/*

ENV PDC_CMAKE_FLAGS="-DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DMPI_RUN_CMD=mpiexec "

Expand Down
6 changes: 6 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,9 @@ build

# ignore docs build dir
docs/build

# ignore venv
**/*venv

# ignore .zed
.zed
5 changes: 5 additions & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,11 @@ if(SUPPRESS_IGNORABLE_WARNINGS)
set(SUPPRESSED_LIST "-Wno-unused-function" ${SUPPRESSED_LIST})
set(SUPPRESSED_LIST "-Wno-unused-result" ${SUPPRESSED_LIST})
set(SUPPRESSED_LIST "-Wno-unused-but-set-variable" ${SUPPRESSED_LIST})
set(SUPPRESSED_LIST "-Wno-inline" ${SUPPRESSED_LIST})
set(SUPPRESSED_LIST "-Wno-alloc-size-larger-than" ${SUPPRESSED_LIST})
set(SUPPRESSED_LIST "-Wno-array-bounds" ${SUPPRESSED_LIST})
set(SUPPRESSED_LIST "-Wno-stringop-truncation" ${SUPPRESSED_LIST})
set(SUPPRESSED_LIST "-Wno-use-after-free" ${SUPPRESSED_LIST})

if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang")
add_compile_options(${SUPPRESSED_LIST})
Expand Down
68 changes: 68 additions & 0 deletions benchmark/idioms/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
cmake_minimum_required (VERSION 2.8.12)

# Setup cmake policies.
foreach(p
CMP0012
CMP0013
CMP0014
CMP0022 # CMake 2.8.12
CMP0025 # CMake 3.0
CMP0053 # CMake 3.1
CMP0054 # CMake 3.1
CMP0074 # CMake 3.12
CMP0075 # CMake 3.12
CMP0083 # CMake 3.14
CMP0093 # CMake 3.15
)
if(POLICY ${p})
cmake_policy(SET ${p} NEW)
endif()
endforeach()

project(METADATA_JSON_LOADER C)

set(JMD_LDR_EXT_INCLUDE_DIRS "")
set(JMD_LDR_EXT_LIBRARIES "")

set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Choose the type of build." FORCE)
# set(JMD_DEBUG ON CACHE BOOL "Enable debug mode.")
option(JMD_DEBUG_MODE "Enable debug mode." ON)
if(JMD_DEBUG_MODE)
add_definitions(-DJMD_DEBUG=1)
endif()

option(USE_SYSTEM_MPI "Use system-installed MPI." ON)
if(USE_SYSTEM_MPI)
find_package(MPI)
if(MPI_FOUND)
add_definitions(-DJMD_LDR_ENABLE_MPI=1)
SET(CMAKE_C_COMPILER ${MPI_C_COMPILER})
SET(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER})
set(JMD_LDR_EXT_INCLUDE_DIRS ${MPI_C_INCLUDE_PATH}
${JMD_LDR_EXT_INCLUDE_DIRS}
)
set(JMD_LDR_EXT_LIBRARIES ${MPI_C_LIBRARIES} ${JMD_LDR_EXT_LIBRARIES})
endif()
endif()

find_package(PDC REQUIRED)
if(PDC_FOUND)
#message(STATUS "PDC include directory: ${PDC_INCLUDE_DIR}")
set(JMD_LDR_EXT_INCLUDE_DIRS ${PDC_INCLUDE_DIR}
${JMD_LDR_EXT_INCLUDE_DIRS}
)
set(JMD_LDR_EXT_LIBRARIES pdc ${JMD_LDR_EXT_LIBRARIES})
endif()


add_library(cjson cjson/cJSON.c)

add_executable(metadata_json_loader
metadata_json_loader.c
metadata_json_processor.h
metadata_json_printer.c
metadata_json_printer.h
metadata_json_importer.c
metadata_json_importer.h)
target_link_libraries(metadata_json_loader ${PDC_EXT_LIB_DEPENDENCIES} pdc cjson ${JMD_LDR_EXT_LIBRARIES})
target_include_directories(metadata_json_loader PUBLIC ${PDC_EXT_INCLUDE_DEPENDENCIES} ${JMD_LDR_EXT_INCLUDE_DIRS})
Loading

0 comments on commit 45562e0

Please sign in to comment.