-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
advection move direction optimization - untested
- Loading branch information
1 parent
0a64913
commit 2ce4584
Showing
37 changed files
with
4,724 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,178 @@ | ||
|
|
||
| SRC = . | ||
| INC = . | ||
| LIB = lib | ||
| OBJ = obj | ||
| BIN = bin | ||
|
|
||
| CPP = $(CC_COMPILER) | ||
| MPICPP = $(MPI_COMPILER) | ||
| NVCC = nvcc | ||
| HIPCC = $(HIP_COMPILER) | ||
| SYCLCC = icpx | ||
|
|
||
| CPPFLAGS = -std=c++14 -Wall -fPIC $(CPPFLAGS_ADD) | ||
| NVCCFLAGS = -std=c++14 -m64 -Xptxas -dlcm=ca $(NVCCFLAGS_ADD) | ||
| HIPCCFLAGS = $(HIPCCFLAGS_ADD) -std=c++17 -Wall -isystem -D__HIP_PLATFORM_AMD__ | ||
| SYCLFLAGS = -w -fsycl -std=c++17 $(SYCLFLAGS_ADD) | ||
|
|
||
| DEBUG ?= 0 | ||
| ifeq ($(DEBUG), 1) | ||
| CPPFLAGS += -O0 -g | ||
| NVCCFLAGS += -O0 -G -g -lineinfo | ||
| HIPCCFLAGS += -O0 -g | ||
| SYCLFLAGS += -O0 -g | ||
| DEBUG_LOG = 1 | ||
| else | ||
| CPPFLAGS += -O3 -g -DLOG_HOPS | ||
| NVCCFLAGS += -O3 -g -DLOG_HOPS | ||
| HIPCCFLAGS += -O3 -g -DLOG_HOPS | ||
| SYCLFLAGS += -O3 -g -DLOG_HOPS | ||
| endif | ||
|
|
||
| DEBUG_LOG ?= 0 | ||
| ifeq ($(DEBUG_LOG), 1) | ||
| CPPFLAGS += -DDEBUG_LOG | ||
| NVCCFLAGS += -DDEBUG_LOG | ||
| HIPCCFLAGS += -DDEBUG_LOG | ||
| SYCLFLAGS += -DDEBUG_LOG | ||
| endif | ||
|
|
||
| IEEE ?= 0 | ||
| ifeq ($(IEEE), 1) | ||
| CPPFLAGS += -fp-model strict -fp-model source -prec-div -prec-sqrt | ||
| NVCCFLAGS += -fmad=false -ftz=false -prec-div=true -prec-sqrt=true | ||
| HIPCCFLAGS += -fmad=false -ftz=false -prec-div=true -prec-sqrt=true | ||
| endif | ||
|
|
||
| CUDA_INC = -I$(CUDA_INSTALL_PATH)/include -I$(MPI_INSTALL_PATH)/include | ||
| CUDA_LIB = -L$(CUDA_INSTALL_PATH)/lib64 -lcudart -lcuda -I$(MPI_INSTALL_PATH)/lib -lmpi | ||
|
|
||
| HIP_INC = -I$(ROCM_INSTALL_DIR)/include -I$(ROCM_THRUST_DIR)/include -I$(ROCM_PRIM_DIR)/include | ||
| HIP_LIB = -L$(ROCM_INSTALL_DIR)/lib -lamdhip64 | ||
|
|
||
| ALL_INC += -I$(OPP_PATH)/include | ||
| ALL_LIBS += -L$(OPP_PATH)/lib | ||
|
|
||
| .PHONY: clean mklib | ||
|
|
||
| mklib: | ||
| @mkdir -p $(OBJ) $(BIN) | ||
|
|
||
| # ------------------------------------------------------------------------------------------ | ||
| advec_opp+seq: | ||
| $(CPP) $(CPPFLAGS) -c advec_opp.cpp -o $(OBJ)/advec_opp_seq.o $(ALL_INC) | ||
| opp_kernels+seq: | ||
| $(CPP) $(CPPFLAGS) -c seq/opp_kernels.cpp -o $(OBJ)/opp_kernels_seq.o $(ALL_INC) | ||
| seq: mklib advec_opp+seq opp_kernels+seq | ||
| $(CPP) $(CPPFLAGS) -o $(BIN)/seq \ | ||
| $(OBJ)/advec_opp_seq.o \ | ||
| $(OBJ)/opp_kernels_seq.o \ | ||
| $(ALL_INC) $(ALL_LIBS) -lopp_seq -lz | ||
|
|
||
| # ------------------------------------------------------------------------------------------ | ||
| advec_opp+mpi: | ||
| $(MPICPP) $(CPPFLAGS) -DUSE_MPI -c advec_opp.cpp -o $(OBJ)/advec_opp_mpi.o $(ALL_INC) | ||
| opp_kernels+mpi: | ||
| $(MPICPP) $(CPPFLAGS) -DUSE_MPI -c mpi/opp_kernels.cpp -o $(OBJ)/opp_kernels_mpi.o $(ALL_INC) | ||
| mpi: mklib advec_opp+mpi opp_kernels+mpi | ||
| $(MPICPP) $(CPPFLAGS) -DUSE_MPI -o $(BIN)/mpi \ | ||
| $(OBJ)/advec_opp_mpi.o \ | ||
| $(OBJ)/opp_kernels_mpi.o \ | ||
| $(ALL_INC) $(ALL_LIBS) -lopp_mpi -lz | ||
|
|
||
| # ------------------------------------------------------------------------------------------ | ||
| advec_opp+omp: | ||
| $(CPP) $(CPPFLAGS) -fopenmp -DUSE_OMP -c advec_opp.cpp -o $(OBJ)/advec_opp_omp.o $(ALL_INC) | ||
| opp_kernels+omp: | ||
| $(CPP) $(CPPFLAGS) -fopenmp -DUSE_OMP -c omp/opp_kernels.cpp -o $(OBJ)/opp_kernels_omp.o $(ALL_INC) | ||
| omp: mklib advec_opp+omp opp_kernels+omp | ||
| $(CPP) $(CPPFLAGS) -fopenmp -o $(BIN)/omp \ | ||
| $(OBJ)/advec_opp_omp.o \ | ||
| $(OBJ)/opp_kernels_omp.o \ | ||
| $(ALL_INC) $(ALL_LIBS) -lopp_omp -lz | ||
|
|
||
| # ------------------------------------------------------------------------------------------ | ||
| advec_opp+omp_mpi: | ||
| $(MPICPP) $(CPPFLAGS) -fopenmp -DUSE_MPI -DUSE_OMP -c advec_opp.cpp -o $(OBJ)/advec_opp_omp_mpi.o $(ALL_INC) | ||
| opp_kernels+omp_mpi: | ||
| $(MPICPP) $(CPPFLAGS) -fopenmp -DUSE_MPI -DUSE_OMP -c omp/opp_kernels.cpp -o $(OBJ)/opp_kernels_omp_mpi.o $(ALL_INC) | ||
| omp_mpi: mklib advec_opp+omp_mpi opp_kernels+omp_mpi | ||
| $(MPICPP) $(CPPFLAGS) -fopenmp -o $(BIN)/omp_mpi \ | ||
| $(OBJ)/advec_opp_omp_mpi.o \ | ||
| $(OBJ)/opp_kernels_omp_mpi.o \ | ||
| $(ALL_INC) $(ALL_LIBS) -lopp_omp_mpi -lz | ||
|
|
||
| # ------------------------------------------------------------------------------------------ | ||
| advec_opp+cuda: | ||
| $(CPP) $(CPPFLAGS) -DUSE_CUDA -c advec_opp.cpp -o $(OBJ)/advec_opp_cuda.o $(ALL_INC) | ||
| opp_kernels+cuda: | ||
| $(NVCC) $(NVCCFLAGS) -DUSE_CUDA -c cuda/opp_kernels.cu -o $(OBJ)/opp_kernels_cuda.o $(ALL_INC) $(CUDA_INC) | ||
| cuda: mklib advec_opp+cuda opp_kernels+cuda | ||
| $(CPP) $(CPPFLAGS) -o $(BIN)/cuda \ | ||
| $(OBJ)/advec_opp_cuda.o \ | ||
| $(OBJ)/opp_kernels_cuda.o \ | ||
| $(ALL_INC) $(CUDA_INC) $(ALL_LIBS) $(CUDA_LIB) -lopp_cuda -lz | ||
|
|
||
| # ------------------------------------------------------------------------------------------ | ||
| advec_opp+cuda_mpi: | ||
| $(MPICPP) $(CPPFLAGS) -DUSE_CUDA -DUSE_MPI -c advec_opp.cpp -o $(OBJ)/advec_opp_cuda_mpi.o $(ALL_INC) $(CUDA_INC) | ||
| opp_kernels+cuda_mpi: | ||
| $(NVCC) $(NVCCFLAGS) -DUSE_CUDA -DUSE_MPI -c cuda/opp_kernels.cu -o $(OBJ)/opp_kernels_cuda_mpi.o $(ALL_INC) $(CUDA_INC) | ||
| cuda_mpi: mklib advec_opp+cuda_mpi opp_kernels+cuda_mpi | ||
| $(MPICPP) $(CPPFLAGS) -DUSE_MPI -o $(BIN)/cuda_mpi \ | ||
| $(OBJ)/advec_opp_cuda_mpi.o \ | ||
| $(OBJ)/opp_kernels_cuda_mpi.o \ | ||
| $(ALL_INC) $(CUDA_INC) $(ALL_LIBS) $(CUDA_LIB) -lopp_cuda_mpi -lz | ||
|
|
||
| # ------------------------------------------------------------------------------------------ | ||
| advec_opp+sycl: | ||
| $(SYCLCC) $(SYCLFLAGS) -DUSE_SYCL -c advec_opp.cpp -o $(OBJ)/advec_opp_sycl.o $(ALL_INC) | ||
| opp_kernels+sycl: | ||
| $(SYCLCC) $(SYCLFLAGS) -DUSE_SYCL -c sycl/opp_kernels.cpp -o $(OBJ)/opp_kernels_sycl.o $(ALL_INC) | ||
| sycl: mklib advec_opp+sycl opp_kernels+sycl | ||
| $(SYCLCC) $(SYCLFLAGS) -o $(BIN)/sycl \ | ||
| $(OBJ)/advec_opp_sycl.o \ | ||
| $(OBJ)/opp_kernels_sycl.o \ | ||
| $(ALL_INC) $(ALL_LIBS) -lopp_sycl -lz | ||
|
|
||
| # ------------------------------------------------------------------------------------------ | ||
| advec_opp+sycl_mpi: | ||
| $(MPICPP) $(SYCLFLAGS) -DUSE_SYCL -DUSE_MPI -c advec_opp.cpp -o $(OBJ)/advec_opp_sycl_mpi.o $(ALL_INC) | ||
| opp_kernels+sycl_mpi: | ||
| $(MPICPP) $(SYCLFLAGS) -DUSE_SYCL -DUSE_MPI -c sycl/opp_kernels.cpp -o $(OBJ)/opp_kernels_sycl_mpi.o $(ALL_INC) | ||
| sycl_mpi: mklib advec_opp+sycl_mpi opp_kernels+sycl_mpi | ||
| $(MPICPP) $(SYCLFLAGS) -o $(BIN)/sycl_mpi \ | ||
| $(OBJ)/advec_opp_sycl_mpi.o \ | ||
| $(OBJ)/opp_kernels_sycl_mpi.o \ | ||
| $(ALL_INC) $(ALL_LIBS) -lopp_sycl_mpi -lz | ||
|
|
||
| # ------------------------------------------------------------------------------------------ | ||
| advec_opp+hip: | ||
| $(MPICPP) $(CPPFLAGS) -DUSE_HIP -c advec_opp.cpp -o $(OBJ)/advec_opp_hip.o $(ALL_INC) $(HIP_INC) | ||
| opp_kernels+hip: | ||
| $(HIPCC) $(HIPCCFLAGS) -DUSE_HIP -c hip/opp_kernels.cpp -o $(OBJ)/opp_kernels_hip.o $(ALL_INC) $(HIP_INC) | ||
| hip: mklib advec_opp+hip opp_kernels+hip | ||
| $(MPICPP) $(CPPFLAGS) -o $(BIN)/hip \ | ||
| $(OBJ)/advec_opp_hip.o \ | ||
| $(OBJ)/opp_kernels_hip.o \ | ||
| $(ALL_INC) $(HIP_INC) $(ALL_LIBS) -lopp_hip -lz $(HIP_LIB) | ||
|
|
||
| # ------------------------------------------------------------------------------------------ | ||
| advec_opp+hip_mpi: | ||
| $(MPICPP) $(CPPFLAGS) -DUSE_HIP -DUSE_MPI -c advec_opp.cpp -o $(OBJ)/advec_opp_hip_mpi.o $(ALL_INC) $(HIP_INC) | ||
| opp_kernels+hip_mpi: | ||
| $(HIPCC) $(HIPCCFLAGS) -DUSE_HIP -DUSE_MPI -c hip/opp_kernels.cpp -o $(OBJ)/opp_kernels_hip_mpi.o $(ALL_INC) $(HIP_INC) | ||
| hip_mpi: mklib advec_opp+hip_mpi opp_kernels+hip_mpi | ||
| $(MPICPP) $(CPPFLAGS) -o $(BIN)/hip_mpi \ | ||
| $(OBJ)/advec_opp_hip_mpi.o \ | ||
| $(OBJ)/opp_kernels_hip_mpi.o \ | ||
| $(ALL_INC) $(HIP_INC) $(ALL_LIBS) -lopp_hip_mpi -lmpi -lz $(HIP_LIB) | ||
|
|
||
| # ------------------------------------------------------------------------------------------ | ||
| clean: | ||
| rm -f *.o *.d *.a | ||
| rm -f $(OBJ)/* | ||
| rm -f $(BIN)/* | ||
| rm -f $(LIB)/* | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| # OP-PIC NESO-Advection (Already code generated for reference) | ||
|
|
||
| **This folder contain both user written code and OP-PIC code generated code.** | ||
|
|
||
| All the user written code will have the below comment; | ||
|
|
||
| `// *********************************************`<br> | ||
| `// USER WRITTEN CODE `<br> | ||
| `// *********************************************` | ||
|
|
||
| The code-generator has added the below comment to all the generated code; | ||
|
|
||
| `// *********************************************`<br> | ||
| `// AUTO GENERATED CODE `<br> | ||
| `// *********************************************` | ||
|
|
||
| If code-generator is invoked in this folder, the available generated code will be replaced with the newly generated code (May generate the same if `advec.cpp` or `kernels.h` is not changed) | ||
|
|
||
| ## | ||
| NESO Advection is a simple 2D demonstrator unstructured mesh advection benchmark code that can be used as a particle mover of a PIC application (https://github.com/will-saunders-ukaea/hybrid_move_benchmark/). | ||
|
|
||
| It is based on quadrilateral mesh cells and particles are periodically move through the mesh by updating its position using with `s=ut`, without any field influence. | ||
| Overall NESO_Advection has 3 DOFs per particle. | ||
|
|
||
| Here, we have implemented the application with OP-PIC, solving the same physics as the original. | ||
|
|
||
| The main function is included in `advec.cpp` containing OP-PIC API calls. | ||
| It contains one mesh cell set, and a particle set. | ||
| NESO-Advection also contain two maps. | ||
| They are, `cell to cell map` and `particle to cell map`. | ||
|
|
||
| ## Structure | ||
| * `advec.cpp` : The main file containing OP-PIC API calls. | ||
| * `kernels.h` : The user written elemental kernel functions. | ||
| * `advec_defs.h` : The defines used in the simulation | ||
| * `advec_misc.h` : Miscellaneous functions including particle distribution initialization and the structure used to hold mesh data till OP-PIC is initialized.. | ||
|
|
||
| ## Code Generation | ||
| Code generation can be done by invoking the below command. | ||
|
|
||
| `python3 $OPP_TRANSLATOR -v -I$OPP_PATH/include/ --file_paths advec.cpp` | ||
|
|
||
| Once the code-generator is invoked, a `advec_opp.cpp` file and `seq`, `omp`, `mpi`, `cuda`, `hip` and `sycl` folders, including `opp_kernels.<cpp|cu>` and a loop kernel header file per unique `opp_par_loop` or `opp_particle_move` loop will get generated. | ||
|
|
||
| ## Compile | ||
| Once the platform specific target files are generated, use the provided `MakeFile` to compile the application. | ||
| * `make seq` | ||
| * `make mpi` | ||
| * `make omp` | ||
| * `make omp_mpi` | ||
| * `make cuda` | ||
| * `make cuda_mpi` | ||
| * `make hip` | ||
| * `make hip_mpi` | ||
| * `make sycl` | ||
| * `make sycl_mpi` | ||
|
|
||
| ### Configuration | ||
| An example configuration file is provided in `OP-PIC/app_neso_advection_cg/configs` folder. | ||
|
|
||
| This file can be used to change the application configurations such as number of steps in the main iterating loop (`num_steps`), domain mesh size (`nx, ny`) and other parameters included in the original neso_advection application. | ||
|
|
||
| In addition, the config file include OP-PIC DSL simulation parameters, such as gpu threads per block (`opp_threads_per_block`), particle move finalizing mechanism (`opp_fill`=[`HoleFill_All`, `Sort_All`, `Shuffle_All`, `Sort_Periodic`, `Shuffle_Periodic`]). | ||
|
|
||
| # Run | ||
| To run the application, below commands can be used. | ||
| * `bin/seq configs/coarse.param` | ||
| * `bin/omp configs/coarse.param` | ||
| * `bin/cuda configs/coarse.param` | ||
| * `bin/hip configs/coarse.param` | ||
| * `mpirun -np <num_ranks> bin/mpi configs/coarse.param` | ||
| * `mpirun -np <num_ranks> bin/cuda_mpi configs/coarse.param` | ||
| * `mpirun -np <num_ranks> bin/hip_mpi configs/coarse.param` | ||
|
|
||
| In addition, `srun` can be used for execution. |
Oops, something went wrong.