Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Halldm2000/homme/pese2 #1237

Closed
wants to merge 9 commits into from
Closed

Halldm2000/homme/pese2 #1237

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
34a6f44
create pese target by duplicating preqx target
david-m-hall Jan 12, 2017
3be8310
create one sub-directory per target for each dcmip test
david-m-hall Jan 12, 2017
7c84d6b
import source files needed by pese.
david-m-hall Jan 18, 2017
31246bd
enable vertical se dynamics and advection for pese target
david-m-hall Jan 19, 2017
ba98900
added dcmip job scripts for cori machine
david-m-hall Jan 20, 2017
52f1abe
pese: restore performance to previous levels by precomputing LU decom…
david-m-hall Jan 23, 2017
3c9039e
pese: futher performance improvements
david-m-hall Jan 24, 2017
0134fab
made changes to enable CAM build of preqx and pese targets
david-m-hall Jan 24, 2017
489458d
added sb81 directory
david-m-hall Jan 24, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions components/cam/bld/configure
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2782,9 +2782,14 @@ sub write_filepath
print $fh "$camsrcdir/homme/src/share\n";
if ($opts{'dyn_target'} eq 'preqx') {
print $fh "$camsrcdir/homme/src/preqx\n";
print $fh "$camsrcdir/homme/src/share/sb81\n";
}
elsif($opts{'dyn_target'} eq 'preqx_acc') {
print $fh "$camsrcdir/homme/src/preqx_acc\n";
print $fh "$camsrcdir/homme/src/share/sb81\n";
}
elsif($opts{'dyn_target'} eq 'pese') {
print $fh "$camsrcdir/homme/src/pese\n";
}
}

Expand Down
2 changes: 1 addition & 1 deletion components/cam/cime_config/config_component.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@

<entry id="CAM_TARGET">
<type>char</type>
<valid_values>preqx,preqx_acc</valid_values>
<valid_values>preqx,preqx_acc,pese</valid_values>
<default_value>preqx</default_value>
<group>build_component_cam</group>
<file>env_build.xml</file>
Expand Down
6 changes: 3 additions & 3 deletions components/cam/src/dynamics/se/dp_coupling.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -440,11 +440,11 @@ subroutine p_d_coupling(phys_state, phys_tend, dyn_in)
call t_startf('putUniquePoints')
do ie=1,nelemd
ncols = elem(ie)%idxP%NumUniquePts
call putUniquePoints(elem(ie)%idxP, nlev, T_tmp(1:ncols,:,ie), elem(ie)%derived%fT(:,:,:,1))
call putUniquePoints(elem(ie)%idxP, nlev, T_tmp(1:ncols,:,ie), elem(ie)%derived%fT(:,:,:))
call putUniquePoints(elem(ie)%idxP, 2, nlev, uv_tmp(1:ncols,:,:,ie), &
elem(ie)%derived%fM(:,:,:,:,1))
elem(ie)%derived%fM(:,:,:,:))
call putUniquePoints(elem(ie)%idxP, nlev,pcnst, q_tmp(1:ncols,:,:,ie), &
elem(ie)%derived%fQ(:,:,:,:,1))
elem(ie)%derived%fQ(:,:,:,:))
end do
call t_stopf('putUniquePoints')
end if
Expand Down
2 changes: 1 addition & 1 deletion components/cam/src/dynamics/se/dyn_comp.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ subroutine dyn_init2(dyn_in)
if(runtype == 0) then
do ie=nets,nete
elem(ie)%state%q(:,:,:,:)=0.0_r8
elem(ie)%derived%fq(:,:,:,:,:)=0.0_r8
elem(ie)%derived%fq(:,:,:,:)=0.0_r8
end do
end if
else if(ideal_phys) then
Expand Down
30 changes: 15 additions & 15 deletions components/cam/src/dynamics/se/stepon.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,12 +231,12 @@ subroutine stepon_run2(phys_state, phys_tend, dyn_in, dyn_out )
! do boundary exchange
do ie=1,nelemd
kptr=0
call edgeVpack(edgebuf,dyn_in%elem(ie)%derived%FM(:,:,:,:,1),2*nlev,kptr,ie)
call edgeVpack(edgebuf,dyn_in%elem(ie)%derived%FM(:,:,:,:),2*nlev,kptr,ie)
kptr=kptr+2*nlev

call edgeVpack(edgebuf,dyn_in%elem(ie)%derived%FT(:,:,:,1),nlev,kptr,ie)
call edgeVpack(edgebuf,dyn_in%elem(ie)%derived%FT(:,:,:),nlev,kptr,ie)
kptr=kptr+nlev
call edgeVpack(edgebuf,dyn_in%elem(ie)%derived%FQ(:,:,:,:,1),nlev*pcnst,kptr,ie)
call edgeVpack(edgebuf,dyn_in%elem(ie)%derived%FQ(:,:,:,:),nlev*pcnst,kptr,ie)
end do

call bndry_exchangeV(par, edgebuf)
Expand All @@ -252,13 +252,13 @@ subroutine stepon_run2(phys_state, phys_tend, dyn_in, dyn_out )
do ie=1,nelemd
kptr=0

call edgeVunpack(edgebuf,dyn_in%elem(ie)%derived%FM(:,:,:,:,1),2*nlev,kptr,ie)
call edgeVunpack(edgebuf,dyn_in%elem(ie)%derived%FM(:,:,:,:),2*nlev,kptr,ie)
kptr=kptr+2*nlev

call edgeVunpack(edgebuf,dyn_in%elem(ie)%derived%FT(:,:,:,1),nlev,kptr,ie)
call edgeVunpack(edgebuf,dyn_in%elem(ie)%derived%FT(:,:,:),nlev,kptr,ie)
kptr=kptr+nlev

call edgeVunpack(edgebuf,dyn_in%elem(ie)%derived%FQ(:,:,:,:,1),nlev*pcnst,kptr,ie)
call edgeVunpack(edgebuf,dyn_in%elem(ie)%derived%FQ(:,:,:,:),nlev*pcnst,kptr,ie)

tl_f = TimeLevel%n0 ! timelevel which was adjusted by physics

Expand All @@ -283,14 +283,14 @@ subroutine stepon_run2(phys_state, phys_tend, dyn_in, dyn_out )

do ic=1,pcnst
! back out tendency: Qdp*dtime
fq = dp(i,j,k)*( dyn_in%elem(ie)%derived%FQ(i,j,k,ic,1) - &
fq = dp(i,j,k)*( dyn_in%elem(ie)%derived%FQ(i,j,k,ic) - &
dyn_in%elem(ie)%state%Q(i,j,k,ic))

! apply forcing to Qdp
! dyn_in%elem(ie)%state%Qdp(i,j,k,ic,tl_fQdp) = &
! dyn_in%elem(ie)%state%Qdp(i,j,k,ic,tl_fQdp) + fq
dyn_in%elem(ie)%state%Qdp(i,j,k,ic,tl_fQdp) = &
dp(i,j,k)*dyn_in%elem(ie)%derived%FQ(i,j,k,ic,1)
dp(i,j,k)*dyn_in%elem(ie)%derived%FQ(i,j,k,ic)

! BEWARE critical region if using OpenMP over k (AAM)
if (ic==1) then
Expand Down Expand Up @@ -338,7 +338,7 @@ subroutine stepon_run2(phys_state, phys_tend, dyn_in, dyn_out )

do ic=1,pcnst
! back out tendency: Qdp*dtime
fq = dp(i,j,k)*( dyn_in%elem(ie)%derived%FQ(i,j,k,ic,1) - &
fq = dp(i,j,k)*( dyn_in%elem(ie)%derived%FQ(i,j,k,ic) - &
dyn_in%elem(ie)%state%Q(i,j,k,ic))

! apply forcing to Qdp
Expand All @@ -356,11 +356,11 @@ subroutine stepon_run2(phys_state, phys_tend, dyn_in, dyn_out )
! force V, T, both timelevels
dyn_in%elem(ie)%state%v(i,j,:,k,tl_f)= &
dyn_in%elem(ie)%state%v(i,j,:,k,tl_f) + &
dtime*dyn_in%elem(ie)%derived%FM(i,j,:,k,1)
dtime*dyn_in%elem(ie)%derived%FM(i,j,:,k)

dyn_in%elem(ie)%state%T(i,j,k,tl_f)= &
dyn_in%elem(ie)%state%T(i,j,k,tl_f) + &
dtime*dyn_in%elem(ie)%derived%FT(i,j,k,1)
dtime*dyn_in%elem(ie)%derived%FT(i,j,k)

end do
end do
Expand Down Expand Up @@ -402,8 +402,8 @@ subroutine stepon_run2(phys_state, phys_tend, dyn_in, dyn_out )
dp_tmp = ( hyai(k+1) - hyai(k) )*dyn_ps0 + &
( hybi(k+1) - hybi(k) )*dyn_in%elem(ie)%state%ps_v(i,j,tl_f)

dyn_in%elem(ie)%derived%FQ(i,j,k,ic,1)=&
( dyn_in%elem(ie)%derived%FQ(i,j,k,ic,1) - &
dyn_in%elem(ie)%derived%FQ(i,j,k,ic)=&
( dyn_in%elem(ie)%derived%FQ(i,j,k,ic) - &
dyn_in%elem(ie)%state%Q(i,j,k,ic))*rec2dt*dp_tmp
end do
end do
Expand Down Expand Up @@ -481,8 +481,8 @@ subroutine stepon_run2(phys_state, phys_tend, dyn_in, dyn_out )
do k=1,nlev
do j=1,np
do i=1,np
ftmp(i+(j-1)*np,k,1) = dyn_in%elem(ie)%derived%FM(i,j,1,k,1)
ftmp(i+(j-1)*np,k,2) = dyn_in%elem(ie)%derived%FM(i,j,2,k,1)
ftmp(i+(j-1)*np,k,1) = dyn_in%elem(ie)%derived%FM(i,j,1,k)
ftmp(i+(j-1)*np,k,2) = dyn_in%elem(ie)%derived%FM(i,j,2,k)
end do
end do
end do
Expand Down
5 changes: 5 additions & 0 deletions components/homme/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ ENDIF ()
OPTION(BUILD_HOMME_SWEQX "Shallow water equations FEM" ON)
OPTION(BUILD_HOMME_PREQX "Primitive equations FEM" ON)
OPTION(BUILD_HOMME_PREQX_ACC "Primitive equations FEM with OpenACC" ON)
OPTION(BUILD_HOMME_PESE "Primitive equations with vertical spectral elements" ON)
OPTION(BUILD_HOMME_SWIM "Shallow water equations implicit" OFF)
OPTION(BUILD_HOMME_PRIM "Primitive equations implicit" OFF)

Expand Down Expand Up @@ -179,6 +180,10 @@ IF(${BUILD_HOMME_PREQX_ACC})
ADD_SUBDIRECTORY(src/preqx_acc)
ENDIF()

IF(${BUILD_HOMME_PESE})
ADD_SUBDIRECTORY(src/pese)
ENDIF()

IF(${BUILD_HOMME_FVM})
ADD_SUBDIRECTORY(src/fvm)
ENDIF()
Expand Down
2 changes: 1 addition & 1 deletion components/homme/README.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ Please see the HOMME wiki for information on how to build HOMME using the CMake
https://wiki.ucar.edu/display/homme/The+HOMME+CMake+build+and+testing+system

The CMAKE build system supports a number of user-configurable targets:
sweqx, preqx, preqx_acc, swim, prim
sweqx, preqx, preqx_acc, pese, swim, prim

Scripts which will CMake configure, build, construct namelists and run a simulation using these
targets, see:
Expand Down
1 change: 0 additions & 1 deletion components/homme/dcmip_tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
file(GLOB FILES "*") # get list of all files
foreach(file ${FILES}) # for each file
if(IS_DIRECTORY ${file}) # if the file is a dir
message(STATUS add_subdirectory ${file}) # display name of dir
add_subdirectory(${file}) # add subdir to cmake
endif()
endforeach(file)
Binary file added components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/.DS_Store
View file
Open in desktop
Binary file not shown.
18 changes: 7 additions & 11 deletions components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,10 @@
file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts
file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts
file(GLOB NAMELISTS "*.nl") # get namelists
file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts
file(GLOB M_FILES "*.m") # get matlab scripts

# Copy test files to build
install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR})

# Create a symbolic link to the vertical coordinate directory
execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/test/vcoord ${CMAKE_CURRENT_BINARY_DIR}/vcoord )

# Add subdirectory for results
add_subdirectory(movies)
# Add all subdirectories to cmake build
file(GLOB FILES "*") # get list of all files
foreach(file ${FILES}) # for each file
if(IS_DIRECTORY ${file}) # if the file is a dir
add_subdirectory(${file}) # add subdir to cmake
endif()
endforeach(file)
8 changes: 0 additions & 8 deletions ...nents/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/jobscript-darwin-preqx.sh
View file
Open in desktop

This file was deleted.

10 changes: 10 additions & 0 deletions components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts
file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts
file(GLOB NAMELISTS "*.nl") # get namelists
file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts
file(GLOB M_FILES "*.m") # get matlab scripts

# Copy test files to build
install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR})

add_subdirectory(movies)
6 changes: 6 additions & 0 deletions components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/bin/bash
cwd=`pwd`
cd ../../..
echo "make pese-nlev60"
make -j4 pese-nlev60
cd $cwd
17 changes: 17 additions & 0 deletions components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/jobscript-cori.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/bash
#
# Jobscript for launching dcmip2012 test1-1 on the NERSC Cori machine
#
# usage: sbatch jobscript-...

#SBATCH -J dcmip1-1 # job name
#SBATCH -o out_dcmip1-1.o%j # output and error file name (%j expands to jobID)
#SBATCH -n 768 # total number of mpi tasks requested
#SBATCH -p debug # queue (partition) -- normal, development, etc.
#SBATCH -t 00:15:00 # run time (hh:mm:ss)
#SBATCH -A acme # charge hours to account 1
#SBATCH -C haswell # use Haswell nodes

EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable
srun -n 768 $EXEC < ./namelist-default.nl # launch simulation

8 changes: 8 additions & 0 deletions ...onents/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/jobscript-darwin.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/bin/bash
#
# Jobscript for launching dcmip2012 test1-1 on a mac running Darwin
#
# usage: ./jobscript-...

EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable
openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation
2 changes: 1 addition & 1 deletion ...ormational_flow/jobscript-edison-preqx.sh → ...formational_flow/pese/jobscript-edison.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,6 @@
#SBATCH -t 00:15:00 # run time (hh:mm:ss)
#SBATCH -A acme # charge hours to account 1

EXEC=../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable
EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable
srun -n 768 $EXEC < ./namelist-default.nl # launch simulation

0 ..._deformational_flow/movies/CMakeLists.txt → ...rmational_flow/pese/movies/CMakeLists.txt
View file
Open in desktop
File renamed without changes.
2 changes: 1 addition & 1 deletion ...3d_deformational_flow/namelist-default.nl → ...formational_flow/pese/namelist-default.nl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
vtop = 0.2549944 ! vertical coordinate at top of atm 254.9 hPa (12km)
/
&analysis_nl
output_dir = "./movies/" ! destination dir for netcdf file
output_dir = "./movies/" ! destination dir for netcdf file
output_timeunits = 1, ! 1=days, 2=hours, 0=timesteps
output_frequency = 1, ! output every 4 hours
output_varnames1 ='u','geo','Q','Q2','Q3','Q4' ! variables to write to file
Expand Down
50 changes: 50 additions & 0 deletions components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/namelist-lowres.nl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
!
! namelist for dcmip2012 test1-1: 3d deformational flow
!_______________________________________________________________________
&ctl_nl
nthreads = 1
partmethod = 4 ! mesh parition method: 4 = space filling curve
topology = "cube" ! mesh type: cubed sphere
test_case = "dcmip2012_test1_1" ! test identifier
ne = 7 ! number of elements per cube face
qsize = 4 ! num tracer fields
ndays = 12 ! num simulation days: 0 = use nmax steps
statefreq = 10 !50 ! number of steps between screen dumps
restartfreq = -1 ! don't write restart files if < 0
runtype = 0 ! 0 = new run
tstep = 100 ! largest timestep in seconds
integration = 'explicit' ! explicit time integration
tstep_type = 1 ! 1 => default method
smooth = 0 ! timestep smooting
nu = 1.585e13 ! hyperviscosity minimum L2 error at 10^13.2
nu_s = 1.585e13
hypervis_order = 2 ! 2 = hyperviscosity
hypervis_subcycle = 1 ! 1 = no hyperviz subcycling
prescribed_wind = 1
/
&filter_nl/
&solver_nl
precon_method = "identity"
maxits = 50
tol = 1.e-7
/
&vert_nl
vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain
vanalytic = 1 ! set vcoords in initialization routine
vtop = 0.2549944 ! vertical coordinate at top of atm 254.9 hPa (12km)
/
&analysis_nl
output_dir = "./movies/" ! destination dir for netcdf file
output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps
output_frequency = 4, ! output every 4 hours
output_varnames1 ='u','geo','Q','Q2','Q3','Q4' ! variables to write to file
interp_type = 0 ! 0=native grid, 1=bilinear
output_type ='netcdf' ! netcdf or pnetcdf
num_io_procs = 16
interp_nlat = 256
interp_nlon = 512
/
&prof_inparm
profile_outpe_num = 100
profile_single_file = .true.
/
Loading