diff --git a/components/cam/bld/configure b/components/cam/bld/configure index 7508e2331c15..fb3bf66b5a0d 100755 --- a/components/cam/bld/configure +++ b/components/cam/bld/configure @@ -2782,9 +2782,14 @@ sub write_filepath print $fh "$camsrcdir/homme/src/share\n"; if ($opts{'dyn_target'} eq 'preqx') { print $fh "$camsrcdir/homme/src/preqx\n"; + print $fh "$camsrcdir/homme/src/share/sb81\n"; } elsif($opts{'dyn_target'} eq 'preqx_acc') { print $fh "$camsrcdir/homme/src/preqx_acc\n"; + print $fh "$camsrcdir/homme/src/share/sb81\n"; + } + elsif($opts{'dyn_target'} eq 'pese') { + print $fh "$camsrcdir/homme/src/pese\n"; } } diff --git a/components/cam/cime_config/config_component.xml b/components/cam/cime_config/config_component.xml index e8f7234c1c77..3f346b54f2f3 100644 --- a/components/cam/cime_config/config_component.xml +++ b/components/cam/cime_config/config_component.xml @@ -28,7 +28,7 @@ char - preqx,preqx_acc + preqx,preqx_acc,pese preqx build_component_cam env_build.xml diff --git a/components/cam/src/dynamics/se/dp_coupling.F90 b/components/cam/src/dynamics/se/dp_coupling.F90 index f074c819104d..a843c1e950f5 100644 --- a/components/cam/src/dynamics/se/dp_coupling.F90 +++ b/components/cam/src/dynamics/se/dp_coupling.F90 @@ -440,11 +440,11 @@ subroutine p_d_coupling(phys_state, phys_tend, dyn_in) call t_startf('putUniquePoints') do ie=1,nelemd ncols = elem(ie)%idxP%NumUniquePts - call putUniquePoints(elem(ie)%idxP, nlev, T_tmp(1:ncols,:,ie), elem(ie)%derived%fT(:,:,:,1)) + call putUniquePoints(elem(ie)%idxP, nlev, T_tmp(1:ncols,:,ie), elem(ie)%derived%fT(:,:,:)) call putUniquePoints(elem(ie)%idxP, 2, nlev, uv_tmp(1:ncols,:,:,ie), & - elem(ie)%derived%fM(:,:,:,:,1)) + elem(ie)%derived%fM(:,:,:,:)) call putUniquePoints(elem(ie)%idxP, nlev,pcnst, q_tmp(1:ncols,:,:,ie), & - elem(ie)%derived%fQ(:,:,:,:,1)) + elem(ie)%derived%fQ(:,:,:,:)) end do call t_stopf('putUniquePoints') end if diff --git a/components/cam/src/dynamics/se/dyn_comp.F90 b/components/cam/src/dynamics/se/dyn_comp.F90 index 0a3ef47777d0..08dc0246812d 100644 --- a/components/cam/src/dynamics/se/dyn_comp.F90 +++ b/components/cam/src/dynamics/se/dyn_comp.F90 @@ -292,7 +292,7 @@ subroutine dyn_init2(dyn_in) if(runtype == 0) then do ie=nets,nete elem(ie)%state%q(:,:,:,:)=0.0_r8 - elem(ie)%derived%fq(:,:,:,:,:)=0.0_r8 + elem(ie)%derived%fq(:,:,:,:)=0.0_r8 end do end if else if(ideal_phys) then diff --git a/components/cam/src/dynamics/se/stepon.F90 b/components/cam/src/dynamics/se/stepon.F90 index fbd032c52acd..74d252f456c7 100644 --- a/components/cam/src/dynamics/se/stepon.F90 +++ b/components/cam/src/dynamics/se/stepon.F90 @@ -231,12 +231,12 @@ subroutine stepon_run2(phys_state, phys_tend, dyn_in, dyn_out ) ! do boundary exchange do ie=1,nelemd kptr=0 - call edgeVpack(edgebuf,dyn_in%elem(ie)%derived%FM(:,:,:,:,1),2*nlev,kptr,ie) + call edgeVpack(edgebuf,dyn_in%elem(ie)%derived%FM(:,:,:,:),2*nlev,kptr,ie) kptr=kptr+2*nlev - call edgeVpack(edgebuf,dyn_in%elem(ie)%derived%FT(:,:,:,1),nlev,kptr,ie) + call edgeVpack(edgebuf,dyn_in%elem(ie)%derived%FT(:,:,:),nlev,kptr,ie) kptr=kptr+nlev - call edgeVpack(edgebuf,dyn_in%elem(ie)%derived%FQ(:,:,:,:,1),nlev*pcnst,kptr,ie) + call edgeVpack(edgebuf,dyn_in%elem(ie)%derived%FQ(:,:,:,:),nlev*pcnst,kptr,ie) end do call bndry_exchangeV(par, edgebuf) @@ -252,13 +252,13 @@ subroutine stepon_run2(phys_state, phys_tend, dyn_in, dyn_out ) do ie=1,nelemd kptr=0 - call edgeVunpack(edgebuf,dyn_in%elem(ie)%derived%FM(:,:,:,:,1),2*nlev,kptr,ie) + call edgeVunpack(edgebuf,dyn_in%elem(ie)%derived%FM(:,:,:,:),2*nlev,kptr,ie) kptr=kptr+2*nlev - call edgeVunpack(edgebuf,dyn_in%elem(ie)%derived%FT(:,:,:,1),nlev,kptr,ie) + call edgeVunpack(edgebuf,dyn_in%elem(ie)%derived%FT(:,:,:),nlev,kptr,ie) kptr=kptr+nlev - call edgeVunpack(edgebuf,dyn_in%elem(ie)%derived%FQ(:,:,:,:,1),nlev*pcnst,kptr,ie) + call edgeVunpack(edgebuf,dyn_in%elem(ie)%derived%FQ(:,:,:,:),nlev*pcnst,kptr,ie) tl_f = TimeLevel%n0 ! timelevel which was adjusted by physics @@ -283,14 +283,14 @@ subroutine stepon_run2(phys_state, phys_tend, dyn_in, dyn_out ) do ic=1,pcnst ! back out tendency: Qdp*dtime - fq = dp(i,j,k)*( dyn_in%elem(ie)%derived%FQ(i,j,k,ic,1) - & + fq = dp(i,j,k)*( dyn_in%elem(ie)%derived%FQ(i,j,k,ic) - & dyn_in%elem(ie)%state%Q(i,j,k,ic)) ! apply forcing to Qdp ! dyn_in%elem(ie)%state%Qdp(i,j,k,ic,tl_fQdp) = & ! dyn_in%elem(ie)%state%Qdp(i,j,k,ic,tl_fQdp) + fq dyn_in%elem(ie)%state%Qdp(i,j,k,ic,tl_fQdp) = & - dp(i,j,k)*dyn_in%elem(ie)%derived%FQ(i,j,k,ic,1) + dp(i,j,k)*dyn_in%elem(ie)%derived%FQ(i,j,k,ic) ! BEWARE critical region if using OpenMP over k (AAM) if (ic==1) then @@ -338,7 +338,7 @@ subroutine stepon_run2(phys_state, phys_tend, dyn_in, dyn_out ) do ic=1,pcnst ! back out tendency: Qdp*dtime - fq = dp(i,j,k)*( dyn_in%elem(ie)%derived%FQ(i,j,k,ic,1) - & + fq = dp(i,j,k)*( dyn_in%elem(ie)%derived%FQ(i,j,k,ic) - & dyn_in%elem(ie)%state%Q(i,j,k,ic)) ! apply forcing to Qdp @@ -356,11 +356,11 @@ subroutine stepon_run2(phys_state, phys_tend, dyn_in, dyn_out ) ! force V, T, both timelevels dyn_in%elem(ie)%state%v(i,j,:,k,tl_f)= & dyn_in%elem(ie)%state%v(i,j,:,k,tl_f) + & - dtime*dyn_in%elem(ie)%derived%FM(i,j,:,k,1) + dtime*dyn_in%elem(ie)%derived%FM(i,j,:,k) dyn_in%elem(ie)%state%T(i,j,k,tl_f)= & dyn_in%elem(ie)%state%T(i,j,k,tl_f) + & - dtime*dyn_in%elem(ie)%derived%FT(i,j,k,1) + dtime*dyn_in%elem(ie)%derived%FT(i,j,k) end do end do @@ -402,8 +402,8 @@ subroutine stepon_run2(phys_state, phys_tend, dyn_in, dyn_out ) dp_tmp = ( hyai(k+1) - hyai(k) )*dyn_ps0 + & ( hybi(k+1) - hybi(k) )*dyn_in%elem(ie)%state%ps_v(i,j,tl_f) - dyn_in%elem(ie)%derived%FQ(i,j,k,ic,1)=& - ( dyn_in%elem(ie)%derived%FQ(i,j,k,ic,1) - & + dyn_in%elem(ie)%derived%FQ(i,j,k,ic)=& + ( dyn_in%elem(ie)%derived%FQ(i,j,k,ic) - & dyn_in%elem(ie)%state%Q(i,j,k,ic))*rec2dt*dp_tmp end do end do @@ -481,8 +481,8 @@ subroutine stepon_run2(phys_state, phys_tend, dyn_in, dyn_out ) do k=1,nlev do j=1,np do i=1,np - ftmp(i+(j-1)*np,k,1) = dyn_in%elem(ie)%derived%FM(i,j,1,k,1) - ftmp(i+(j-1)*np,k,2) = dyn_in%elem(ie)%derived%FM(i,j,2,k,1) + ftmp(i+(j-1)*np,k,1) = dyn_in%elem(ie)%derived%FM(i,j,1,k) + ftmp(i+(j-1)*np,k,2) = dyn_in%elem(ie)%derived%FM(i,j,2,k) end do end do end do diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index 5362b074e752..d21033ede1ce 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -57,6 +57,7 @@ ENDIF () OPTION(BUILD_HOMME_SWEQX "Shallow water equations FEM" ON) OPTION(BUILD_HOMME_PREQX "Primitive equations FEM" ON) OPTION(BUILD_HOMME_PREQX_ACC "Primitive equations FEM with OpenACC" ON) +OPTION(BUILD_HOMME_PESE "Primitive equations with vertical spectral elements" ON) OPTION(BUILD_HOMME_SWIM "Shallow water equations implicit" OFF) OPTION(BUILD_HOMME_PRIM "Primitive equations implicit" OFF) @@ -179,6 +180,10 @@ IF(${BUILD_HOMME_PREQX_ACC}) ADD_SUBDIRECTORY(src/preqx_acc) ENDIF() +IF(${BUILD_HOMME_PESE}) + ADD_SUBDIRECTORY(src/pese) +ENDIF() + IF(${BUILD_HOMME_FVM}) ADD_SUBDIRECTORY(src/fvm) ENDIF() diff --git a/components/homme/README.cmake b/components/homme/README.cmake index 7240395cb313..f27ac7e082c6 100644 --- a/components/homme/README.cmake +++ b/components/homme/README.cmake @@ -6,7 +6,7 @@ Please see the HOMME wiki for information on how to build HOMME using the CMake https://wiki.ucar.edu/display/homme/The+HOMME+CMake+build+and+testing+system The CMAKE build system supports a number of user-configurable targets: -sweqx, preqx, preqx_acc, swim, prim +sweqx, preqx, preqx_acc, pese, swim, prim Scripts which will CMake configure, build, construct namelists and run a simulation using these targets, see: diff --git a/components/homme/dcmip_tests/CMakeLists.txt b/components/homme/dcmip_tests/CMakeLists.txt index 96e3d9f1e821..788c75d29a6d 100644 --- a/components/homme/dcmip_tests/CMakeLists.txt +++ b/components/homme/dcmip_tests/CMakeLists.txt @@ -3,7 +3,6 @@ file(GLOB FILES "*") # get list of all files foreach(file ${FILES}) # for each file if(IS_DIRECTORY ${file}) # if the file is a dir - message(STATUS add_subdirectory ${file}) # display name of dir add_subdirectory(${file}) # add subdir to cmake endif() endforeach(file) \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/.DS_Store b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/.DS_Store new file mode 100644 index 000000000000..67aa29031391 Binary files /dev/null and b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/.DS_Store differ diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/CMakeLists.txt index 23dc25d6dbac..90f557418413 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/CMakeLists.txt +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/CMakeLists.txt @@ -1,14 +1,10 @@ -file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts -file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts -file(GLOB NAMELISTS "*.nl") # get namelists -file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts -file(GLOB M_FILES "*.m") # get matlab scripts - -# Copy test files to build -install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - # Create a symbolic link to the vertical coordinate directory execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/test/vcoord ${CMAKE_CURRENT_BINARY_DIR}/vcoord ) -# Add subdirectory for results -add_subdirectory(movies) +# Add all subdirectories to cmake build +file(GLOB FILES "*") # get list of all files +foreach(file ${FILES}) # for each file + if(IS_DIRECTORY ${file}) # if the file is a dir + add_subdirectory(${file}) # add subdir to cmake + endif() +endforeach(file) \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/jobscript-darwin-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/jobscript-darwin-preqx.sh deleted file mode 100644 index 8b681f3f8947..000000000000 --- a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/jobscript-darwin-preqx.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# -# Jobscript for launching dcmip2012 test1-1 on a mac running Darwin -# -# usage: ./jobscript-... - -EXEC=../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable -openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/CMakeLists.txt new file mode 100644 index 000000000000..ecfb00937223 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/build.sh b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/build.sh new file mode 100644 index 000000000000..c102db1846b2 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cwd=`pwd` +cd ../../.. + echo "make pese-nlev60" + make -j4 pese-nlev60 +cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/jobscript-cori.sh new file mode 100644 index 000000000000..3bf719be2685 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/jobscript-cori.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-1 on the NERSC Cori machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip1-1 # job name +#SBATCH -o out_dcmip1-1.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 768 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:15:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 +#SBATCH -C haswell # use Haswell nodes + +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable +srun -n 768 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/jobscript-darwin.sh b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/jobscript-darwin.sh new file mode 100644 index 000000000000..585304dc3a32 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/jobscript-darwin.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-1 on a mac running Darwin +# +# usage: ./jobscript-... + +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable +openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/jobscript-edison-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/jobscript-edison.sh similarity index 90% rename from components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/jobscript-edison-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/jobscript-edison.sh index 966f55593f89..00546f2d406b 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/jobscript-edison-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/jobscript-edison.sh @@ -11,6 +11,6 @@ #SBATCH -t 00:15:00 # run time (hh:mm:ss) #SBATCH -A acme # charge hours to account 1 -EXEC=../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable srun -n 768 $EXEC < ./namelist-default.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/movies/CMakeLists.txt similarity index 100% rename from components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/movies/CMakeLists.txt rename to components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/movies/CMakeLists.txt diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/namelist-default.nl b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/namelist-default.nl similarity index 96% rename from components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/namelist-default.nl rename to components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/namelist-default.nl index 82bf32365c36..8d6281d15ea1 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/namelist-default.nl +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/namelist-default.nl @@ -34,7 +34,7 @@ vtop = 0.2549944 ! vertical coordinate at top of atm 254.9 hPa (12km) / &analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file + output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 1, ! 1=days, 2=hours, 0=timesteps output_frequency = 1, ! output every 4 hours output_varnames1 ='u','geo','Q','Q2','Q3','Q4' ! variables to write to file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/namelist-lowres.nl new file mode 100644 index 000000000000..0d1946c8f46e --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/namelist-lowres.nl @@ -0,0 +1,50 @@ +! +! namelist for dcmip2012 test1-1: 3d deformational flow +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test1_1" ! test identifier + ne = 7 ! number of elements per cube face + qsize = 4 ! num tracer fields + ndays = 12 ! num simulation days: 0 = use nmax steps + statefreq = 10 !50 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 = new run + tstep = 100 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 1 ! 1 => default method + smooth = 0 ! timestep smooting + nu = 1.585e13 ! hyperviscosity minimum L2 error at 10^13.2 + nu_s = 1.585e13 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + prescribed_wind = 1 +/ +&filter_nl/ +&solver_nl + precon_method = "identity" + maxits = 50 + tol = 1.e-7 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 0.2549944 ! vertical coordinate at top of atm 254.9 hPa (12km) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps + output_frequency = 4, ! output every 4 hours + output_varnames1 ='u','geo','Q','Q2','Q3','Q4' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlat = 256 + interp_nlon = 512 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/plot_q_lat_lon.ncl b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/plot_q_lat_lon.ncl new file mode 100644 index 000000000000..7a75e59194c6 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/plot_q_lat_lon.ncl @@ -0,0 +1,189 @@ +;================================================; +; plot-q-lat-lon +; plot lattitude-longitude cross section of tracer q +; at time_index t +;================================================; +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" +; ================================================; + +begin + + f = addfile("./movies/dcmip2012_test1_11.nc","r") + lat = f->lat + lon = f->lon + lev = f->lev + time = f->time + nlat = getfilevardimsizes(f, "lat" ) + nlon = getfilevardimsizes(f, "lon" ) + nlev = getfilevardimsizes(f, "lev" ) + nt = dimsizes(time) + + ; get time_index t from command line + + t_index = nt-1 + if(isvar("t")) then + t_index = t + end if + + ; get tracer_id q from command line + + tracer_id=1 + if(isvar("q")) then + tracer_id = q ; use q from command line if possible + end if + + ; get zoom factor from command line + + zoom_factor = 1.0 + if(isvar("zoom")) + zoom_factor = zoom + end if + + ; find level closest to 4900 m + + g = 9.80616d0; Gravity (m s^2) + heights = f->geo(0,:,0,0)/g + print ("heights = "+heights+" eta="+lev) + +; level = (nlev-1) - closest_val(4900,heights(::-1)) +; level = (nlev-1) - closest_val(5100,heights(::-1)) + level = closest_val(0.56,lev) + + ; use level index from command line if specified + if(isvar("l")) + level = l + end if + + print("height = "+heights(level)) + z = round( heights(level), 3) + + print ("tracer_id: q = " + tracer_id) + print ("t_index: t = " + t_index) + print ("zoom_factor: zoom = " + zoom_factor) + print ("level: l = " + level) + print ("eta: = " + lev(level) ) + print ("z: = " + z ) + + ; load chosen field + + if(tracer_id .eq. 1) then + q_i = f->Q (t_index,level,:,:) + + else if (tracer_id .eq. 2) then + q_i = f->Q2(t_index,level,:,:) + + else if (tracer_id .eq. 3) then + q_i = f->Q3(t_index,level,:,:) + + else if (tracer_id .eq. 4) then + q_i = f->Q4(t_index,level,:,:) + + end if + end if + end if + end if + + ; delete units and long names + + q_i@units = " " + q_i@long_name = " " + + ; display min and max values + minval = min(q_i) + maxval = max(q_i) + print("minval="+minval+" maxval="+maxval) + + ; round to two decimal places? + ;q_i = decimalPlaces(q_i,2,True) + ;printMinMax(q_i,0) + + ; Setup the plot + + res1 = True + res1@gsnDraw = False ; panel plot + res1@gsnFrame = False ; don't draw yet + res1@gsnCenterString = "" + res1@vpWidthF = 2 + res1@vpHeightF = 1 + res1@sfXArray = lon ; uses lon as plot x-axis + res1@sfYArray = lat ; uses lat for y axis + res1@tiYAxisString = "latitiude" ; y-axis title + res1@tiXAxisString = "longitude" ; x-axis title + res1@lbLabelBarOn = False ; label bar + + res1@cnLinesOn = False ;True + res1@cnFillOn = True + res1@cnInfoLabelOn = False ; don't give each + res1@cnLineLabelsOn = False ; panel its own + res1@cnMonoLineColor = True ; single color lines + res1@cnLineColor = "black" ; black lines + res1@cnLineDrawOrder = "PreDraw" + res1@cnLineThicknessF = 0.1 + res1@cnFillOpacityF = 1.0 + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = 1.00 ; max contour color label + res1@cnMinLevelValF = 0.00 ; min contour color label + res1@cnLevelSpacingF = 0.10 ; contour color spacing + + ; text properties + + res1@tmXBMajorLengthF = 0.03 + res1@txFont = 22 ; helvetica-bold + res1@tiMainFont = 22 ; helvetica-bold + res1@tiMainFontHeightF = 0.045 ; set font sizes + res1@tiXAxisFontHeightF = 0.05 + res1@tiYAxisFontHeightF = 0.05 + res1@tmXBLabelFontHeightF = 0.04 + res1@tiXAxisOffsetYF = 0.010 ; shift x label + res1@tiYAxisOffsetXF = 0.010 ; shift y label + res1@tiMainString = "DCMIP 1-1 q"+tracer_id+" t="+sprintf("%3.1f", time(t_index))+" days"+ " z="+z+"m range=("+sprintf("%5.4f", minval)+","+sprintf("%5.4f", maxval)+")" + + ; zoom in on the center of the plot + + res1@trXMinF = 180.0 - 180.0/zoom_factor + res1@trXMaxF = 180.0 + 180.0/zoom_factor + res1@trYMinF = 0.0 - 90.0/zoom_factor + res1@trYMaxF = 0.0 + 90.0/zoom_factor + + ; panel properties + + pres = True ; Panel resource + pres@gsnMaximize = True ; maximize panel size + pres@gsnFrame = False ; don't advance yet + pres@gsnPaperOrientation = "portrait" ; paper orientation + pres@gsnPanelLabelBar = True ; add common label bar + pres@lbLabelFontHeightF = 0.020 ; set labelbar font height + + ; set output type and name + + wks_type = "pdf" + wks_type@wkPaperHeightF = 8 + wks_type@wkPaperWidthF = 11 + filename ="dcmip2012_test1-1_q"+tracer_id+"_t"+t_index + wks = gsn_open_wks(wks_type,filename) + print("writing "+filename) + + ; use rainbow colormap with a few of color substitutions + + gsn_define_colormap(wks,"WhiteBlueGreenYellowRed") ; choose default colormap + colors = gsn_retrieve_colormap(wks) ; retrieve color map for editing + colors(25 ,:)= (/ 1, 1, 1/) ; white for tracer-free region + colors(2 ,:)= (/ 0.95, 0.95, 0.95 /) ; gray for undershoots + colors(255,:)= (/ 0.4, 0.0, 0.0 /) ; brick-red for overshoots + gsn_define_colormap(wks,colors) + + ; plot countours and label bar + + plot = new (1, graphic) ; define plot - need 4 panels + plot(0) = gsn_csm_contour(wks,q_i(:,:),res1) ; plot Q1 + gsn_panel(wks,plot,(/1,1/),pres) ; 2x2 plot + frame(wks) + +end + + + + diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/plot_tracers_height_lon.ncl b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/plot_tracers_height_lon.ncl similarity index 99% rename from components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/plot_tracers_height_lon.ncl rename to components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/plot_tracers_height_lon.ncl index 835fa3c49b5a..1f6d50c5cc98 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/plot_tracers_height_lon.ncl +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/plot_tracers_height_lon.ncl @@ -14,7 +14,7 @@ begin ; replace your models file name below - f = addfile("movies/dcmip2012_test1_11.nc","r") + f = addfile("./movies/dcmip2012_test1_11.nc","r") lat = f->lat lon = f->lon diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/plot_tracers_lat_lon.ncl b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/plot_tracers_lat_lon.ncl similarity index 98% rename from components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/plot_tracers_lat_lon.ncl rename to components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/plot_tracers_lat_lon.ncl index c5f498665cee..550ed0920842 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/plot_tracers_lat_lon.ncl +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/pese/plot_tracers_lat_lon.ncl @@ -11,7 +11,7 @@ load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" begin ; open file and read in data(time,lev,lat,long) from 0 to n-1 - f = addfile("movies/dcmip2012_test1_11.nc","r") + f = addfile("./movies/dcmip2012_test1_11.nc","r") lat = f->lat lon = f->lon diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/CMakeLists.txt new file mode 100644 index 000000000000..ecfb00937223 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/build-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/build.sh similarity index 88% rename from components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/build-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/build.sh index 74fd300c1513..217a14d27761 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/build-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/build.sh @@ -1,6 +1,6 @@ #!/bin/bash cwd=`pwd` -cd ../.. +cd ../../.. echo "make preqx-nlev60-interp" make -j4 preqx-nlev60-interp cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/jobscript-cori.sh new file mode 100644 index 000000000000..d66940b3945d --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/jobscript-cori.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-1 on the NERSC Edison machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip1-1 # job name +#SBATCH -o out_dcmip1-1.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 768 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:15:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 +#SBATCH -C haswell # use Haswell nodes + +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +srun -n 768 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/jobscript-darwin.sh b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/jobscript-darwin.sh new file mode 100644 index 000000000000..5d5789523ce1 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/jobscript-darwin.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-1 on a mac running Darwin +# +# usage: ./jobscript-... + +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/jobscript-edison.sh b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/jobscript-edison.sh new file mode 100644 index 000000000000..d923537547d8 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/jobscript-edison.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-1 on the NERSC Edison machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip1-1 # job name +#SBATCH -o out_dcmip1-1.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 768 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:15:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 + +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +srun -n 768 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/movies/CMakeLists.txt similarity index 100% rename from components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/movies/CMakeLists.txt rename to components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/movies/CMakeLists.txt diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/namelist-default.nl b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/namelist-default.nl new file mode 100644 index 000000000000..8d6281d15ea1 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/namelist-default.nl @@ -0,0 +1,50 @@ +! +! namelist for dcmip2012 test1-1: 3d deformational flow +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test1_1" ! test identifier + ne = 30 ! number of elements per cube face + qsize = 4 ! num tracer fields + ndays = 12 ! num simulation days: 0 = use nmax steps + statefreq = 200 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 = new run + tstep = 33 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 1 ! 1 => default method + smooth = 0 ! timestep smooting + nu = 1.585e13 ! hyperviscosity minimum L2 error at 10^13.2 + nu_s = 1.585e13 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + prescribed_wind = 1 +/ +&filter_nl/ +&solver_nl + precon_method = "identity" + maxits = 50 + tol = 1.e-7 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 0.2549944 ! vertical coordinate at top of atm 254.9 hPa (12km) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 1, ! 1=days, 2=hours, 0=timesteps + output_frequency = 1, ! output every 4 hours + output_varnames1 ='u','geo','Q','Q2','Q3','Q4' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlat = 256 + interp_nlon = 512 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/namelist-lowres.nl similarity index 96% rename from components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/namelist-lowres.nl rename to components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/namelist-lowres.nl index 901efab63cde..255d977b5fdc 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/namelist-lowres.nl +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/namelist-lowres.nl @@ -34,7 +34,7 @@ vtop = 0.2549944 ! vertical coordinate at top of atm 254.9 hPa (12km) / &analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file + output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps output_frequency = 4, ! output every 4 hours output_varnames1 ='u','geo','Q','Q2','Q3','Q4' ! variables to write to file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/plot_q_lat_lon.ncl b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/plot_q_lat_lon.ncl similarity index 99% rename from components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/plot_q_lat_lon.ncl rename to components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/plot_q_lat_lon.ncl index db66ffe0a603..661cb0dc6a81 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/plot_q_lat_lon.ncl +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/plot_q_lat_lon.ncl @@ -11,7 +11,7 @@ load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" begin - f = addfile("movies/dcmip2012_test1_11.nc","r") + f = addfile("./movies/dcmip2012_test1_11.nc","r") lat = f->lat lon = f->lon lev = f->lev @@ -46,6 +46,7 @@ begin g = 9.80616d0; Gravity (m s^2) heights = f->geo(0,:,0,0)/g + ; level = (nlev-1) - closest_val(4900,heights(::-1)) level = (nlev-1) - closest_val(5100,heights(::-1)) diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/plot_tracers_height_lon.ncl b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/plot_tracers_height_lon.ncl new file mode 100644 index 000000000000..1f6d50c5cc98 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/plot_tracers_height_lon.ncl @@ -0,0 +1,182 @@ +;================================================; +; test11-height-lon.ncl +;================================================; +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + + ;=================================================; + ; open file and read in data (time,lev,lat,long) from 0 to n-1 + ;=================================================; + + ; replace your models file name below + + f = addfile("./movies/dcmip2012_test1_11.nc","r") + + lat = f->lat + lon = f->lon + lev = f->lev + time = f->time + + nlat = getfilevardimsizes(f, "lat" ) + nlon = getfilevardimsizes(f, "lon" ) + nlev = getfilevardimsizes(f, "lev" ) + nt = dimsizes(time) + + ; get time index from command line + + t_index = nt-1 + if(isvar("t")) then + t_index = t ; use t from command line if possible + end if + print ("t_index="+t_index) + + ; get zoom factor from command line + + zoom_factor = 1.0 + if(isvar("zoom")) + zoom_factor = zoom + end if + + ; We require plots of q1, q2, q3 and q4 at the equator. + + equator = nlat/2 + + ; Input tracers + + q1 = f->Q (t_index,:,equator,:) + q2 = f->Q2(t_index,:,equator,:) + q3 = f->Q3(t_index,:,equator,:) + q4 = f->Q4(t_index,:,equator,:) + + ; Delete tracer units and long names + + q1@units = " " + q2@units = " " + q3@units = " " + q4@units = " " + + q1@long_name = " " + q2@long_name = " " + q3@long_name = " " + q4@long_name = " " + + ; Set up height coordinate + ; Note that for CAM-FV pressure coordinates are used, + ; therefore the height levels are upside-down + ; i.e. for 60 vertical levels index 0 is at 11900 m + ; and index 59 is at 100 m. + + z = new((/nlev/),double) + dz = 12000.0/nlev + + do kk=0,nlev-1 + z(kk) = (12000.0-0.5*dz)-(kk)*dz + end do + + ; We now produce the plot + + plot = new (4, graphic) ; define plot - need 4 panels + + res1 = True + res1@gsnDraw = False ; panel plot + res1@gsnFrame = False ; don't draw yet + res1@lbLabelAutoStride = True + res1@gsnCenterString = "" + res1@tiMainString = "" + res1@vpWidthF = 2 + res1@vpHeightF = 1 + res1@lbLabelBarOn = False ; label bar + + res1@cnFillOn = True + res1@cnLinesOn = False + res1@cnInfoLabelOn = False ; don't give each + res1@cnLineLabelsOn = False ; panel its own + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = 0.91 ; max contour color label + res1@cnMinLevelValF = 0.01 ; min contour color label + res1@cnLevelSpacingF = 0.05 ; contour color spacing + res1@cnFillPalette = "WhBlGrYeRe" ;"WhiteBlueGreenYellowRed"; + ;pres@lbLabelStride = 1 + + ; zoom in on the center of the plot + + res1@trXMinF = 180.0 - 180.0/zoom_factor + res1@trXMaxF = 180.0 + 180.0/zoom_factor + res1@trYMinF = 5.0 - 5.0/zoom_factor + res1@trYMaxF = 5.0 + 7.0/zoom_factor + + pres = True + pres@gsnMaximize = True + pres@gsnPanelLabelBar = True ; Communal label bar + pres@gsnFrame = False + pres@gsnPaperOrientation = "portrait" + pres@gsnPanelXWhiteSpacePercent = 0 + + pres@pmLabelBarOrthogonalPosF = 0.010 + pres@lbLabelFontHeightF = 0.012 + + res1@sfXArray = lon ; uses lon as plot x-axis + res1@sfYArray = z/1000.0 ; uses z for y axis + res1@trYReverse = False ; reverses y-axis, false + res1@tiYAxisString = "" ; y-axis title + res1@tiXAxisString = "" ; x-axis title + + pltTitle="dcmip 1-1 vertical equatorial slice t="+sprintf("%3.1f", time(t_index))+" days" + pres@txString = pltTitle + pres@txFontHeightF = 0.015 + pres@txFont = 22 ; helvetica-bold +; pres@txPerimOn =True + + pres@tiMainOffsetYF=1 + ;res1@txFont = 22 ; helvetica-bold + res1@tmXBLabelFontHeightF = 0.04 + res1@tiMainFontHeightF = 0.04 ; change font heights + ;res1@tiMainFont = 22 + ;res1@gsnStringFontHeightF = 0.04 + + ; Change the output type and name + + wks_type = "pdf" + wks_type@wkPaperHeightF = 8 + wks_type@wkPaperWidthF = 11 + filename ="test11_heightlon_t"+t_index + wks = gsn_open_wks(wks_type,filename) + print("writing "+filename) + + ;gsn_define_colormap(wks,"gui_default") ; Colormap + ;res1@gsnSpreadColors = True + + res1@gsnCenterString = "q1" + plot(0) = gsn_csm_contour(wks,q1(:,:),res1) ; plot Q1 + + res1@gsnCenterString = "q2" + plot(1) = gsn_csm_contour(wks,q2(:,:),res1) ; plot Q2 + + res1@gsnCenterString = "q3" + plot(2) = gsn_csm_contour(wks,q3(:,:),res1) ; plot Q3 + + res1@gsnCenterString = "q4" + plot(3) = gsn_csm_contour(wks,q4(:,:),res1) ; plot Q4 + + pres@gsnPanelXF = (/-0.50, .55, -0.50, .55/) ; Adjust rightmost plots + pres@gsnPanelYF = (/-1,-1,.49,.49/) ; Adjust bottommost plots. + + gsn_panel(wks,plot,(/2,2/),pres) ; 2x2 plot + + ; Add height and longitude labels + + txres3 = True + txres3@txAngleF = 90. + txres3@txFontHeightF = 0.015 + gsn_text_ndc(wks,"Height (km)",0.010,0.52,txres3) + + ;txres3@txAngleF = 0. + ;gsn_text_ndc(wks,"Longitude",0.52,0.23,txres3) + + frame(wks) + +end diff --git a/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/plot_tracers_lat_lon.ncl b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/plot_tracers_lat_lon.ncl new file mode 100644 index 000000000000..550ed0920842 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.1_3d_deformational_flow/preqx/plot_tracers_lat_lon.ncl @@ -0,0 +1,162 @@ +;================================================; +; test11-lat-lon.ncl +; produces the lat-lon plots for test 11 +;================================================; +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" +; ================================================; + +begin + + ; open file and read in data(time,lev,lat,long) from 0 to n-1 + f = addfile("./movies/dcmip2012_test1_11.nc","r") + + lat = f->lat + lon = f->lon + lev = f->lev + time = f->time + + nlat = getfilevardimsizes(f, "lat" ) + nlon = getfilevardimsizes(f, "lon" ) + nlev = getfilevardimsizes(f, "lev" ) + nt = dimsizes(time) + + ; We want the output at day 6 or 12. + interval = 6.0 + ind2days = 1.0*interval/24 + t_index = nt-1 + if(isvar("t")) then + t_index = t ; use t from command line if possible + end if + print ("t_index="+t_index) + + ; We require plots of q1, q2, q3 and q4 at 4900 m. + ; For 60 vertical levels, this corresponds to model level 35 + + modheight = 35 + + ; Input tracers + + q1 = f->Q (t_index,modheight,:,:) + q2 = f->Q2(t_index,modheight,:,:) + q3 = f->Q3(t_index,modheight,:,:) + q4 = f->Q4(t_index,modheight,:,:) + + ; Delete tracer units and long names + + q1@units = " " + q2@units = " " + q3@units = " " + q4@units = " " + + q1@long_name = " " + q2@long_name = " " + q3@long_name = " " + q4@long_name = " " + + ; Setup the plot + + plot = new (4, graphic) ; define plot - need 4 panels + + res1 = True + res1@gsnDraw = False ; panel plot + res1@gsnFrame = False ; don't draw yet + res1@cnFillOn = True + res1@cnLinesOn = False + res1@gsnSpreadColors = True + res1@lbLabelAutoStride = True + res1@gsnCenterString = "" + res1@tiMainString = "" + res1@vpWidthF = 0.46 + res1@vpHeightF = 0.23 + res1@cnInfoLabelOn = False ; don't give each + res1@cnLineLabelsOn = False ; panel its own + res1@lbLabelBarOn = False ; label bar + res1@sfXArray = lon ; uses lon as plot x-axis + res1@sfYArray = lat ; uses lat for y axis + res1@trYReverse = False ; reverses y-axis, false + res1@tiYAxisString = "" ; y-axis title + res1@tiXAxisString = "" ; x-axis title + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = 0.90 ; max contour color label + res1@cnMinLevelValF = 0.10 ; min contour color label + res1@cnLevelSpacingF = 0.10 ; contour color spacing + res1@txFont = 22 ; helvetica-bold + res1@tmXBLabelFontHeightF = 0.008 + ;res1@gsnContourZeroLineThicknessF = 2.0 + ;res1@cnFillPalette = "gsltod" + + ; zoom in on the center + ;res1@trXMinF = 80.0 + ;res1@trXMaxF = 280.0 + ;res1@trYMinF = -50.0 + ;res1@trYMaxF = 50.0 + + pres = True + pres@gsnMaximize = True + pres@gsnPanelLabelBar = True ; Communal label bar + pres@pmLabelBarOrthogonalPosF = -0.00 + pres@amJust = "TopLeft" + pres@gsnFrame = False + pres@lbLabelStride = 1 + pres@gsnPaperOrientation = "portrait" + ;pres@cnLabelBarEndStyle = "IncludeMinMaxLabels" ; turn on end labels + + pltTitle="DCMIP 1-1 4900 m t="+t_index*ind2days+" days" ; plot title + pres@txString = pltTitle + pres@txFontHeightF = 0.015 + pres@txFont = 22 ; helvetica-bold + + ; set output type and name + + wks_type = "pdf" + wks_type@wkPaperHeightF = 8 + wks_type@wkPaperWidthF = 11 + wks = gsn_open_wks(wks_type,"dcmip2012_test1-1_latlon_t"+t_index) + + + ; use rainbow colormap with a few of color substitutions + + ;gsn_define_colormap(wks,"gui_default") ; choose a colormap + gsn_define_colormap(wks,"WhiteBlueGreenYellowRed") ; choose default colormap + colors = gsn_retrieve_colormap(wks) ; retrieve color map for editing + colors(25 ,:)= (/ 1, 1, 1/) ; white for tracer-free region + colors(2 ,:)= (/ 0.95, 0.95, 0.95 /) ; gray for undershoots + colors(255,:)= (/ 0.4, 0.0, 0.0 /) ; brick-red for overshoots + gsn_define_colormap(wks,colors) + + + res1@gsnCenterString = "q1" + plot(0) = gsn_csm_contour(wks,q1(:,:),res1) ; plot Q1 + + res1@gsnCenterString = "q2" + plot(1) = gsn_csm_contour(wks,q2(:,:),res1) ; plot Q2 + + res1@gsnCenterString = "q3" + plot(2) = gsn_csm_contour(wks,q3(:,:),res1) ; plot Q3 + + res1@gsnCenterString = "q4" + plot(3) = gsn_csm_contour(wks,q4(:,:),res1) ; plot Q4 + + gsn_panel(wks,plot,(/2,2/),pres) ; 2x2 plot + + ; Add latitude and longitude labels + + txres3 = True + txres3@txAngleF = 90. + txres3@txFontHeightF = 0.012 + txres3@txFont = 22 ; helvetica-bold + + gsn_text_ndc(wks,"Latitude",0.01,0.49,txres3) + + txres3@txAngleF = 0. + gsn_text_ndc(wks,"Longitude",0.5,0.19,txres3) + + frame(wks) + +end + + + diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/.DS_Store b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/.DS_Store new file mode 100644 index 000000000000..5008ddfcf53c Binary files /dev/null and b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/.DS_Store differ diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/CMakeLists.txt index 23dc25d6dbac..90f557418413 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/CMakeLists.txt +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/CMakeLists.txt @@ -1,14 +1,10 @@ -file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts -file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts -file(GLOB NAMELISTS "*.nl") # get namelists -file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts -file(GLOB M_FILES "*.m") # get matlab scripts - -# Copy test files to build -install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - # Create a symbolic link to the vertical coordinate directory execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/test/vcoord ${CMAKE_CURRENT_BINARY_DIR}/vcoord ) -# Add subdirectory for results -add_subdirectory(movies) +# Add all subdirectories to cmake build +file(GLOB FILES "*") # get list of all files +foreach(file ${FILES}) # for each file + if(IS_DIRECTORY ${file}) # if the file is a dir + add_subdirectory(${file}) # add subdir to cmake + endif() +endforeach(file) \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/jobscript-darwin-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/jobscript-darwin-preqx.sh deleted file mode 100644 index 6590d09d4569..000000000000 --- a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/jobscript-darwin-preqx.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# -# Jobscript for launching dcmip2012 test1-2 on a mac running Darwin -# -# usage: ./jobscript-... - -EXEC=../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable -openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/CMakeLists.txt new file mode 100644 index 000000000000..ecfb00937223 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/build.sh b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/build.sh new file mode 100644 index 000000000000..c102db1846b2 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cwd=`pwd` +cd ../../.. + echo "make pese-nlev60" + make -j4 pese-nlev60 +cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/jobscript-cori.sh new file mode 100644 index 000000000000..037e7938c842 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/jobscript-cori.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-2 on the NERSC Cori machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip1-2 # job name +#SBATCH -o out_dcmip1-2.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 512 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:05:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 +#SBATCH -C haswell # use Haswell nodes + +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable +srun -n 512 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/jobscript-darwin.sh b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/jobscript-darwin.sh new file mode 100644 index 000000000000..f4b1e96f1b5d --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/jobscript-darwin.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-2 on a mac running Darwin +# +# usage: ./jobscript-... + +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable +openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/jobscript-edison-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/jobscript-edison.sh similarity index 90% rename from components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/jobscript-edison-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/jobscript-edison.sh index 8becaec0a389..dfa36d1965e9 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/jobscript-edison-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/jobscript-edison.sh @@ -11,6 +11,6 @@ #SBATCH -t 00:05:00 # run time (hh:mm:ss) #SBATCH -A acme # charge hours to account 1 -EXEC=../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable srun -n 512 $EXEC < ./namelist-default.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/movies/CMakeLists.txt similarity index 100% rename from components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/movies/CMakeLists.txt rename to components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/movies/CMakeLists.txt diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/namelist-default.nl b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/namelist-default.nl similarity index 96% rename from components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/namelist-default.nl rename to components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/namelist-default.nl index 98d1eea2b9ab..2e6e92c4eeb8 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/namelist-default.nl +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/namelist-default.nl @@ -34,7 +34,7 @@ vtop = 0.2549944 ! vertical coordinate at top of atm 254.9 hPa (12km) / &analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file + output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps output_frequency = 3, ! output every 3 hours output_varnames1 ='u','v','omega','Q','Q2','geo' ! variables to write to file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/namelist-lowres.nl new file mode 100644 index 000000000000..266a58f4075d --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/namelist-lowres.nl @@ -0,0 +1,50 @@ +! +! namelist for dcmip2012 test1-2: hadley-like meridional circulation at 2° +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test1_2" ! test identifier + ne = 15 ! number of elements per cube face + qsize = 2 ! num tracer fields + ndays = 1 ! num simulation days: 0 = use nmax steps + statefreq = 10 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 = new run + tstep = 60 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 1 ! 1 => default method + smooth = 0 ! timestep smooting + nu = 0 ! hyperviscosity + nu_s = 0 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + prescribed_wind = 1 +/ +&filter_nl/ +&solver_nl + precon_method = "identity" + maxits = 50 + tol = 1.e-7 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 0.2549944 ! vertical coordinate at top of atm 254.9 hPa (12km) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps + output_frequency = 1 !3, ! output every 3 hours + output_varnames1 ='u','v','omega','Q','Q2','geo' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlat = 256 + interp_nlon = 512 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/plot_lat_vs_height.ncl b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/plot_lat_vs_height.ncl new file mode 100644 index 000000000000..666e3cb1bccf --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/pese/plot_lat_vs_height.ncl @@ -0,0 +1,142 @@ +;================================================; +; plot-slice.ncl +; David Hall (University of Colorado, Boulder) +;================================================; + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + + f1 = addfile("./movies/dcmip2012_test1_21.nc","r") + lat = f1->lat + lon = f1->lon + lev = f1->lev + time = f1->time + + nlat = getfilevardimsizes(f1, "lat" ) + nlon = getfilevardimsizes(f1, "lon" ) + nlev = getfilevardimsizes(f1, "lev" ) + nt = dimsizes(time) + + t_index = nt-1 ; set time to max time + if(isvar("t")) then + t_index = t ; use t from command line if possible + end if + print ("t_index="+t_index) + + midlon = nlon/2;-1 + q2 = f1->Q2 (t_index,:,:,midlon) + print ("longitude = "+lon(midlon)) + + ; Get height from geopotential + + g = 9.80616d0; Gravity (m s^2) + z = f1->geo(0,:,0,midlon)/g + + ; Remove tracer units + + q2@units = "" + q2@long_name = "" + + res1 = True ; Plot resource + res1@gsnDraw = False ; panel plot + res1@gsnFrame = False ; don't draw yet + res1@lbLabelBarOn = False ; label bar + res1@vpWidthF = 2 ; set aspect ratio 2:1 + res1@vpHeightF = 1 + res1@sfXArray = lat ; uses lat as plot x-axis + res1@sfYArray = z/1000.0 ; vertical axis = z in km + + res1@cnInfoLabelOn = False + res1@cnFillOn = True ; fill contours + res1@cnLinesOn = True ; draw contour lines + res1@cnLineLabelsOn = False ; don't draw labels + res1@cnMonoLineColor = True ; single color lines + res1@cnLineColor = "black" ; black lines + res1@cnLineThicknessF = 0.01 ; thin lines + res1@cnLineDrawOrder = "PreDraw" + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = 1.0 ; max contour level + res1@cnMinLevelValF = 0.0 ; min contour level + res1@cnLevelSpacingF = 0.1 ; contour level spacing + + res1@tiYAxisString = "height (km)" ; y-axis title + res1@tiXAxisString = "latitude" ; x-axis title + res1@txFont = 22 ; helvetica-bold + res1@tiMainFont = 22 ; helvetica-bold + res1@tiMainFontHeightF = 0.06 ; set font sizes + res1@tiXAxisFontHeightF = 0.05 + res1@tiYAxisFontHeightF = 0.05 + res1@tmXBLabelFontHeightF = 0.04 + res1@tiXAxisOffsetYF = 0.015 ; shift x label + res1@tiYAxisOffsetXF = 0.015 ; shift y label + res1@gsnYAxisIrregular2Linear = False + + res1@gsnCenterString = "DCMIP2012 test 1-2 1dg L"+ (nlev) +" t = "+time(t_index)*24.0+" hr" + + ; panel properties + + pres = True ; Panel resource + pres@gsnMaximize = True ; maximize panel size + pres@gsnFrame = False ; don't advance yet + pres@gsnPaperOrientation = "portrait" ; paper orientation + pres@gsnPanelLabelBar = True ; add common label bar + pres@lbLabelFontHeightF =0.020 ; set labelbar font height + + ; Create workstation for file output + + name = "dcmip2012_test1-2_L"+(nlev)+"_t"+t_index + wks_type = "pdf" + wks_type@wkPaperHeightF = 8 + wks_type@wkPaperWidthF = 11 + wks = gsn_open_wks(wks_type,name) + wks@wkAntiAlias = True + + ; Use rainbow colormap with a few of color substitutions + + ;gsn_define_colormap(wks,"gui_default") ; choose default colormap + ;colors = gsn_retrieve_colormap(wks) ; retrieve color map for editing + ;colors(2 ,:)= (/ 0.4, 0.4, 0.4 /) ; dark gray for undershoots + ;colors(4 ,:)= (/ 1.0, 1.0, 1.0 /) ; white for tracer-free region + ;colors(23,:)= (/ 0.6, 0.0, 0.0 /) ; brick-red for overshoots + + ; WBGYR + gsn_define_colormap(wks,"WhiteBlueGreenYellowRed") ; choose default colormap + colors = gsn_retrieve_colormap(wks) ; retrieve color map for editing + colors(25 ,:)= (/ 1, 1, 1/) ; white for tracer-free region + colors(2 ,:)= (/ 0.95, 0.95, 0.95 /) ; gray for undershoots + colors(255,:)= (/ 0.4, 0.0, 0.0 /) ; brick-red for overshoots + gsn_define_colormap(wks,colors) + + ; Add colored contour plot + + plot = new (1, graphic) ; define plot - need 4 panels + plot(0) = gsn_csm_contour(wks,q2(:,:),res1) ; plot q2 at 12 hours + + ; Set properties for horizontal lines + + gsres = True ; graphics resource + gsres@tfPolyDrawOrder = "PostDraw" + gsres@gsLineThicknessF = 0.1 + gsres@gsLineColor = "gray50" + z_line = new(nlev,graphic) + plx = (/-180,180/) + + ; Add line every 5th vertical level + + do k = 0,nlev-1,5 + ply = (/z(k)/1000.0 ,z(k)/1000.0/) + z_line(k) = gsn_add_polyline (wks,plot(0),plx, ply, gsres) + end do + + ; draw panel with contour and lines + + print("writing "+name) + + gsn_panel(wks,plot,(/1,1/),pres) + frame(wks) + +end diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/CMakeLists.txt new file mode 100644 index 000000000000..ecfb00937223 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/build-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/build.sh similarity index 88% rename from components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/build-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/build.sh index 74fd300c1513..217a14d27761 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/build-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/build.sh @@ -1,6 +1,6 @@ #!/bin/bash cwd=`pwd` -cd ../.. +cd ../../.. echo "make preqx-nlev60-interp" make -j4 preqx-nlev60-interp cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/jobscript-cori.sh new file mode 100644 index 000000000000..87be0b8103af --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/jobscript-cori.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-2 on the NERSC Cori machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip1-2 # job name +#SBATCH -o out_dcmip1-2.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 512 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:05:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 +#SBATCH -C haswell # use Haswell nodes + +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +srun -n 512 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/jobscript-darwin.sh b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/jobscript-darwin.sh new file mode 100644 index 000000000000..2cbe93125174 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/jobscript-darwin.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-2 on a mac running Darwin +# +# usage: ./jobscript-... + +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/jobscript-edison.sh b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/jobscript-edison.sh new file mode 100644 index 000000000000..8e8e77a64f60 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/jobscript-edison.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-2 on the NERSC Edison machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip1-2 # job name +#SBATCH -o out_dcmip1-2.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 512 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:05:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 + +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +srun -n 512 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/movies/CMakeLists.txt similarity index 100% rename from components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/movies/CMakeLists.txt rename to components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/movies/CMakeLists.txt diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/namelist-default.nl b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/namelist-default.nl new file mode 100644 index 000000000000..2e6e92c4eeb8 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/namelist-default.nl @@ -0,0 +1,50 @@ +! +! namelist for dcmip2012 test1-2: hadley-like meridional circulation at 1° +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test1_2" ! test identifier + ne = 30 ! number of elements per cube face + qsize = 2 ! num tracer fields + ndays = 1 ! num simulation days: 0 = use nmax steps + statefreq = 50 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 = new run + tstep = 60 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 1 ! 1 => default method + smooth = 0 ! timestep smooting + nu = 0 ! hyperviscosity + nu_s = 0 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + prescribed_wind = 1 +/ +&filter_nl/ +&solver_nl + precon_method = "identity" + maxits = 50 + tol = 1.e-7 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 0.2549944 ! vertical coordinate at top of atm 254.9 hPa (12km) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps + output_frequency = 3, ! output every 3 hours + output_varnames1 ='u','v','omega','Q','Q2','geo' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlat = 256 + interp_nlon = 512 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/namelist-lowres.nl similarity index 96% rename from components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/namelist-lowres.nl rename to components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/namelist-lowres.nl index 7395b4e629c3..d6a0c9139347 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/namelist-lowres.nl +++ b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/namelist-lowres.nl @@ -34,7 +34,7 @@ vtop = 0.2549944 ! vertical coordinate at top of atm 254.9 hPa (12km) / &analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file + output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps output_frequency = 3, ! output every 3 hours output_varnames1 ='u','v','omega','Q','Q2','geo' ! variables to write to file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/plot_lat_vs_height.ncl b/components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/plot_lat_vs_height.ncl similarity index 100% rename from components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/plot_lat_vs_height.ncl rename to components/homme/dcmip_tests/dcmip2012_test1.2_hadley_meridional_circulation/preqx/plot_lat_vs_height.ncl diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/.DS_Store b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/.DS_Store new file mode 100644 index 000000000000..5008ddfcf53c Binary files /dev/null and b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/.DS_Store differ diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/CMakeLists.txt index 23dc25d6dbac..90f557418413 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/CMakeLists.txt +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/CMakeLists.txt @@ -1,14 +1,10 @@ -file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts -file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts -file(GLOB NAMELISTS "*.nl") # get namelists -file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts -file(GLOB M_FILES "*.m") # get matlab scripts - -# Copy test files to build -install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - # Create a symbolic link to the vertical coordinate directory execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/test/vcoord ${CMAKE_CURRENT_BINARY_DIR}/vcoord ) -# Add subdirectory for results -add_subdirectory(movies) +# Add all subdirectories to cmake build +file(GLOB FILES "*") # get list of all files +foreach(file ${FILES}) # for each file + if(IS_DIRECTORY ${file}) # if the file is a dir + add_subdirectory(${file}) # add subdir to cmake + endif() +endforeach(file) \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/jobscript-darwin-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/jobscript-darwin-preqx.sh deleted file mode 100644 index 5f912bb64a4a..000000000000 --- a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/jobscript-darwin-preqx.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# -# Jobscript for launching dcmip2012 test1-3 on a mac running Darwin -# -# usage: ./jobscript-... - -EXEC=../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable -openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/CMakeLists.txt new file mode 100644 index 000000000000..ecfb00937223 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/build.sh b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/build.sh new file mode 100644 index 000000000000..c102db1846b2 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cwd=`pwd` +cd ../../.. + echo "make pese-nlev60" + make -j4 pese-nlev60 +cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/jobscript-cori.sh new file mode 100644 index 000000000000..6b391930462b --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/jobscript-cori.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-3 on the NERSC Cori machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip1-3 # job name +#SBATCH -o out_dcmip1-3.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 768 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:05:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 +#SBATCH -C haswell # use Haswell nodes + +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable +srun -n 768 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/jobscript-darwin.sh b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/jobscript-darwin.sh new file mode 100644 index 000000000000..225fb6995439 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/jobscript-darwin.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-3 on a mac running Darwin +# +# usage: ./jobscript-... + +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable +openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/jobscript-edison-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/jobscript-edison.sh similarity index 90% rename from components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/jobscript-edison-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/jobscript-edison.sh index c2e05ace1474..6915fec3b418 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/jobscript-edison-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/jobscript-edison.sh @@ -11,6 +11,6 @@ #SBATCH -t 00:05:00 # run time (hh:mm:ss) #SBATCH -A acme # charge hours to account 1 -EXEC=../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable srun -n 768 $EXEC < ./namelist-default.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/movies/CMakeLists.txt similarity index 100% rename from components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/movies/CMakeLists.txt rename to components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/movies/CMakeLists.txt diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/namelist-default.nl b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/namelist-default.nl similarity index 90% rename from components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/namelist-default.nl rename to components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/namelist-default.nl index 513ed5ecb7e0..d874896dc5ff 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/namelist-default.nl +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/namelist-default.nl @@ -8,8 +8,8 @@ test_case = "dcmip2012_test1_3" ! test identifier ne = 30 ! number of elements per cube face qsize = 4 ! num tracer fields - ndays = 12 ! num simulation days: 0 = use nmax steps - statefreq = 50 ! number of steps between screen dumps + ndays = 12 ! num simulation days: 0 = use nmax steps + statefreq = 100 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 = new run tstep = 100 ! largest timestep in seconds @@ -35,7 +35,7 @@ vtop = 0.2549944 ! vertical coordinate at top of atm 254.9 hPa (12km) / &analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file + output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 1, ! 1=days, 2=hours, 0=timesteps output_frequency = 1, ! output every 3 hours output_varnames1 ='ps','omega','Q','Q2','Q3','Q4' ! variables to write to file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/namelist-lowres.nl similarity index 96% rename from components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/namelist-lowres.nl rename to components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/namelist-lowres.nl index e8fed847b571..a0e16b763040 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/namelist-lowres.nl +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/namelist-lowres.nl @@ -35,7 +35,7 @@ vtop = 0.2549944 ! vertical coordinate at top of atm 254.9 hPa (12km) / &analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file + output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 1, ! 1=days, 2=hours, 0=timesteps output_frequency = 1, ! output every 3 hours output_varnames1 ='ps','omega','Q','Q2','Q3','Q4' ! variables to write to file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/plot_lon_vs_height.ncl b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/plot_lon_vs_height.ncl new file mode 100644 index 000000000000..e748e7e4c419 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/pese/plot_lon_vs_height.ncl @@ -0,0 +1,267 @@ +;================================================; +; DMCIP Test 1-3 +; plot_tracers.ncl +;================================================; +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + + f = addfile("./movies/dcmip2012_test1_31.nc","r") + lat = f->lat + lon = f->lon + lev = f->lev + time = f->time + + nlat = getfilevardimsizes(f, "lat" ) + nlon = getfilevardimsizes(f, "lon" ) + nlev = getfilevardimsizes(f, "lev" ) + nt = dimsizes(time) + + ; get time_index t from command line + + interval = 24.0;hours + ind2days = 1.0*interval/24 + t_index = nt-1 + if(isvar("t")) then + t_index = t + end if + print ("t_index: t = " + t_index) + + ; Load tracer data and surface pressure + + equator = nlat/2 + q1 = f->Q4(t_index,:,equator,:) + ps = f->ps(t_index,1,equator,:) + + ; round to 4th decimal place, to prevent plotting errors like q=-1.0e-14 + q1=round(q1*1e4,2)/1e4 + + print("min q = "+min(q1)) + print("max q = "+max(q1)) + + ; Set up reference height coordinate + + numz = 4*nlev + refz = new((/numz/),double) + dz = 12000.0/numz + + do kk=0,numz-1 + refz(kk) = (12000.0-0.5*dz)-(kk)*dz + end do + + ; We want to interpolate to the constant height levels, + ; so we need to calculate P and Z at the models levels + + ; Calculate height at each point based on + ; z = Rd T0/g * ln(p0/p) + ; and p = a p0 * b ps + + g = 9.80616 + radius = 6371229. + T0 = 300. + Rd = 287.04 + cp = 1004.64 + H = Rd * T0 / g + + hyam = f->hyam + hybm = f->hybm + + P0 = 100000.0 + + z = new((/nlev,nlon/),double) + P = new((/nlev,nlon/),double) + + do i =0, nlon-1 + do k =0, nlev-1 + + P(k,i) = hyam(k)*P0 + hybm(k)*ps(i) + z(k,i) = H * log(P0/P(k,i)) + + end do + end do + zs = H * log(P0/ps) + + ; Interpolate to height levels between 2000 and 10000m. + + zz = new((/numz/),double) + + do kk = 0, numz-1 + zz(kk) = 000.0 + kk*dz;200.0 + end do + + ; Simple Interpolation + + qq = new((/numz,nlon/),double) + qqpart = new((/numz/),double) + q1part = new((/nlev/),double) + zpart = new((/nlev/),double) + q1part2 = new((/nlev/),double) + zpart2 = new((/nlev/),double) + + q1n = new((/nlev,nlon/),double) + q2n = new((/nlev,nlon/),double) + + do i = 0, nlon-1 + + q1part(:) = q1(:,i) ; put in 1D column + zpart(:) = z(:,i) + q1part2 = q1part(::-1) ; Reverse Q + zpart2 = zpart(::-1) ; Reverse z + ;qqpart = ftcurv(zpart2,q1part2,zz) ; Interpolate to zz + qqpart = linint1(zpart2,q1part2,False,zz,0) + qq(:,i) = qqpart(:) ; Make 2D again + + end do + + ; Remove tracer units and long-names from plot + + q1@units = " " + qq@units = " " + q1@long_name = " " + qq@long_name = " " + + ; Setup the plots + + plot = new (2, graphic) ; define plot - need 4 panels + + res1 = True + res1@gsnDraw = False ; panel plot + res1@gsnFrame = False ; don't draw yet + ;res1@gsnSpreadColors = True + res1@lbLabelAutoStride = True + res1@gsnCenterString = "" + res1@tiMainString = "" + res1@vpWidthF = 2 + res1@vpHeightF = 1 + res1@lbLabelBarOn = False + res1@sfXArray = lon ; use lon as plot x-axis + res1@sfYArray = lev ; use z for y axis + res1@trYReverse = True ; flip y-axis + + res1@cnFillOn = True + res1@cnLinesOn = False + res1@cnInfoLabelOn = False + res1@cnLineLabelsOn = False + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = 1.0 + res1@cnMinLevelValF = 0.0 + res1@cnLevelSpacingF = 0.1 + res1@cnLineThicknessF = 0.2 + ;res1@cnLineDrawOrder = "PreDraw" + + res1@tiYAxisString = "~F33~h"+"~F21~ level" ; y-axis title + res1@tiXAxisString = "longitude" ; x-axis title + res1@txFont = 22 ; helvetica-bold + res1@tiMainFont = 22 ; helvetica-bold + res1@tiMainFontHeightF = 0.06 ; set font sizes + res1@tiXAxisFontHeightF = 0.05 + res1@tiYAxisFontHeightF = 0.05 + res1@tmXBLabelFontHeightF = 0.04 + ;res1@tiXAxisOffsetYF = 0.015 ; shift x label + ;res1@tiYAxisOffsetXF = 0.015 ; shift y label + res1@gsnYAxisIrregular2Linear = True + + res2 = True + res2@gsnDraw = False ; panel plot + res2@gsnFrame = False ; don't draw yet + res2@lbLabelAutoStride = True + res2@lbLabelBarOn = False + res2@gsnCenterString = "" + res2@tiMainString = "" + res2@vpWidthF = 2 + res2@vpHeightF = 1 + res2@sfXArray = lon ; uses lon as plot x-axis + res2@sfYArray = zz ; uses zz for y axis + res2@trYReverse = False ; reverses y-axis, false + + res2@cnFillOn = True + res2@cnLinesOn = False + res2@cnInfoLabelOn = False ; don't give each + res2@cnLineLabelsOn = False ; panel its own + res2@cnLevelSelectionMode = "ManualLevels" + res2@cnMaxLevelValF = 1.0 ; max contour color label + res2@cnMinLevelValF = 0.0 ; min contour color label + res2@cnLevelSpacingF = 0.1 ; contour color spacing + ;res2@cnLineDrawOrder = "PreDraw" + res2@cnLineThicknessF = 0.01 + res2@cnMonoLineColor = True + + res2@tiYAxisString = "z level" ; y-axis title + res2@tiXAxisString = "longitude" ; x-axis title + res2@txFont = 22 ; helvetica-bold + res2@tiMainFont = 22 ; helvetica-bold + res2@tiMainFontHeightF = 0.06 ; set font sizes + res2@tiXAxisFontHeightF = 0.05 + res2@tiYAxisFontHeightF = 0.05 + res2@tmXBLabelFontHeightF = 0.04 + ;res2@tiXAxisOffsetYF = 0.015 ; shift x label + ;res2@tiYAxisOffsetXF = 0.015 ; shift y label + + pres = True + pres@gsnMaximize = True + pres@gsnFrame = False + pres@gsnPaperOrientation = "portrait" + pres@gsnPanelTop = 1 + ;pres@gsnPanelBottom = 0.10 + pres@gsnPanelLabelBar = True ; add common label bar + pres@lbLabelFontHeightF = 0.020 ; set labelbar font height + pres@gsnPanelYWhiteSpacePercent = 0.0 + + ; set output type and name + + wks_type="pdf" + wks_type@wkPaperHeightF = 8 + wks_type@wkPaperWidthF = 6 + + filename ="dcmip2012_test1-3_L60_t"+t_index + wks = gsn_open_wks(wks_type,filename) + print ("writing file: " + filename) + + ; WBGYR + gsn_define_colormap(wks,"WhiteBlueGreenYellowRed") ; choose default colormap + colors = gsn_retrieve_colormap(wks) ; retrieve color map for editing + colors(25 ,:)= (/ 1, 1, 1/) ; white for tracer-free region + colors(2 ,:)= (/ 0.95, 0.95, 0.95 /) ; gray for undershoots + colors(255,:)= (/ 0.4, 0.0, 0.0 /) ; brick-red for overshoots + gsn_define_colormap(wks,colors) + + days = t_index*ind2days + + res1@gsnCenterString = "DCMIP 1-3 1 dg L60 "+sprintf("%6.2f", days)+" days" + plot(0) = gsn_csm_contour(wks,q1(:,:),res1) ; plot Q on model levels + + res2@gsnCenterString = "DCMIP 1-3 1 dg L60 "+sprintf("%6.2f", days)+" days" + plot(1) = gsn_csm_contour(wks,qq(:,:),res2) ; plot Q on height levels + + ; Add mountain range to plot + + gsres = True ; poly res + gsres@gsLineColor = "black" + gsres@tfPolyDrawOrder = "postdraw" + gsres@gsLineOpacityF = 0.5 + + zs(0)=0 + zs(nlon-1)=0 + dummy = gsn_add_polygon (wks,plot(1),lon, zs, gsres) + + ; Add vertical-level contours to plot + + gsres@gsLineColor = "black" + eta_line = new(nlev,graphic) + z_line = new(nlev,graphic) + plx = (/lon(0),lon(nlon-1)/) + + do k = 0,nlev-1,10 + ply = (/lev(k),lev(k)/) + eta_line(k) = gsn_add_polyline (wks,plot(0),plx, ply, gsres) + z_line(k) = gsn_add_polyline (wks,plot(1),lon, z(k,:), gsres) + end do + + gsn_panel(wks,plot,(/2,1/),pres) ; 2x2 plot + + frame(wks) + +end diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/CMakeLists.txt new file mode 100644 index 000000000000..ecfb00937223 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/build-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/build.sh similarity index 88% rename from components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/build-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/build.sh index 74fd300c1513..217a14d27761 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/build-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/build.sh @@ -1,6 +1,6 @@ #!/bin/bash cwd=`pwd` -cd ../.. +cd ../../.. echo "make preqx-nlev60-interp" make -j4 preqx-nlev60-interp cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/jobscript-cori.sh new file mode 100644 index 000000000000..288763f8f454 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/jobscript-cori.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-3 on the NERSC Cori machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip1-3 # job name +#SBATCH -o out_dcmip1-3.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 768 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:05:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 +#SBATCH -C haswell # use Haswell nodes + +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +srun -n 768 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/jobscript-darwin.sh b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/jobscript-darwin.sh new file mode 100644 index 000000000000..01e7d11ef6f0 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/jobscript-darwin.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-3 on a mac running Darwin +# +# usage: ./jobscript-... + +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/jobscript-edison.sh b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/jobscript-edison.sh new file mode 100644 index 000000000000..f91d4be304cf --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/jobscript-edison.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test1-3 on the NERSC Edison machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip1-3 # job name +#SBATCH -o out_dcmip1-3.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 768 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:05:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 + +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +srun -n 768 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/movies/CMakeLists.txt similarity index 100% rename from components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/movies/CMakeLists.txt rename to components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/movies/CMakeLists.txt diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/namelist-default.nl b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/namelist-default.nl new file mode 100644 index 000000000000..eb6a44c078f3 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/namelist-default.nl @@ -0,0 +1,51 @@ +! +! namelist for dcmip2012 test1-3: thin clouds over orography at 1° +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test1_3" ! test identifier + ne = 30 ! number of elements per cube face + qsize = 4 ! num tracer fields + ndays = 12 ! num simulation days: 0 = use nmax steps + statefreq = 100 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 = new run + tstep = 100 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 1 ! 1 => default method + smooth = 0 ! timestep smooting + nu = 0 ! hyperviscosity + nu_s = 0 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + prescribed_wind = 1 + rsplit = 0 +/ +&filter_nl/ +&solver_nl + precon_method = "identity" + maxits = 50 + tol = 1.e-7 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 0.2549944 ! vertical coordinate at top of atm 254.9 hPa (12km) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 1, ! 1=days, 2=hours, 0=timesteps + output_frequency = 1, ! output every 3 hours + output_varnames1 ='ps','omega','Q','Q2','Q3','Q4' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlat = 256 + interp_nlon = 512 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/namelist-lowres.nl new file mode 100644 index 000000000000..a0e16b763040 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/namelist-lowres.nl @@ -0,0 +1,51 @@ +! +! namelist for dcmip2012 test1-3: thin clouds over orography at 3° +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test1_3" ! test identifier + ne = 10 ! number of elements per cube face + qsize = 4 ! num tracer fields + ndays = 12 ! num simulation days: 0 = use nmax steps + statefreq = 10 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 = new run + tstep = 100 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 1 ! 1 => default method + smooth = 0 ! timestep smooting + nu = 0 ! hyperviscosity + nu_s = 0 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + prescribed_wind = 1 + rsplit = 0 +/ +&filter_nl/ +&solver_nl + precon_method = "identity" + maxits = 50 + tol = 1.e-7 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 0.2549944 ! vertical coordinate at top of atm 254.9 hPa (12km) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 1, ! 1=days, 2=hours, 0=timesteps + output_frequency = 1, ! output every 3 hours + output_varnames1 ='ps','omega','Q','Q2','Q3','Q4' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlat = 256 + interp_nlon = 512 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/plot_lon_vs_height.ncl b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/plot_lon_vs_height.ncl similarity index 99% rename from components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/plot_lon_vs_height.ncl rename to components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/plot_lon_vs_height.ncl index f5cef4a32ce1..329439f6c0fb 100644 --- a/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/plot_lon_vs_height.ncl +++ b/components/homme/dcmip_tests/dcmip2012_test1.3_thin_clouds_over_orography/preqx/plot_lon_vs_height.ncl @@ -9,7 +9,7 @@ load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" begin - f = addfile("movies/dcmip2012_test1_31.nc","r") + f = addfile("./movies/dcmip2012_test1_31.nc","r") lat = f->lat lon = f->lon lev = f->lev diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/.DS_Store b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/.DS_Store new file mode 100644 index 000000000000..5008ddfcf53c Binary files /dev/null and b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/.DS_Store differ diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/CMakeLists.txt index 23dc25d6dbac..90f557418413 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/CMakeLists.txt +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/CMakeLists.txt @@ -1,14 +1,10 @@ -file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts -file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts -file(GLOB NAMELISTS "*.nl") # get namelists -file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts -file(GLOB M_FILES "*.m") # get matlab scripts - -# Copy test files to build -install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - # Create a symbolic link to the vertical coordinate directory execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/test/vcoord ${CMAKE_CURRENT_BINARY_DIR}/vcoord ) -# Add subdirectory for results -add_subdirectory(movies) +# Add all subdirectories to cmake build +file(GLOB FILES "*") # get list of all files +foreach(file ${FILES}) # for each file + if(IS_DIRECTORY ${file}) # if the file is a dir + add_subdirectory(${file}) # add subdir to cmake + endif() +endforeach(file) \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/build-preqx-native.sh b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/build-preqx-native.sh deleted file mode 100644 index dba617280246..000000000000 --- a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/build-preqx-native.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -cwd=`pwd` -cd ../.. - echo "make preqx-nlev30-native" - make preqx-nlev30-native -cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/jobscript-darwin-preqx-native.sh b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/jobscript-darwin-preqx-native.sh deleted file mode 100644 index 294d8110f1cf..000000000000 --- a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/jobscript-darwin-preqx-native.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# -# Jobscript for launching dcmip2012 test2-0 on a mac running Darwin -# -# usage: ./jobscript-... - -EXEC=../../test_execs/preqx-nlev30/preqx-nlev30-native # set name of executable -openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/jobscript-darwin-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/jobscript-darwin-preqx.sh deleted file mode 100644 index 754cb3bfc2f6..000000000000 --- a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/jobscript-darwin-preqx.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# -# Jobscript for launching dcmip2012 test2-0 on a mac running Darwin -# -# usage: ./jobscript-... - -EXEC=../../test_execs/preqx-nlev30-interp/preqx-nlev30-interp # set name of executable -openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/CMakeLists.txt new file mode 100644 index 000000000000..ecfb00937223 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/build.sh b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/build.sh new file mode 100644 index 000000000000..3eb5922031c1 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cwd=`pwd` +cd ../../.. + echo "make pese-nlev30" + make -j4 pese-nlev30 +cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/jobscript-cori.sh new file mode 100644 index 000000000000..43c4ad7911a2 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/jobscript-cori.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test3-1 on the NERSC Cori machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip2-0 # job name +#SBATCH -o out_dcmip2-0.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 600 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:05:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 +#SBATCH -C haswell # use Haswell nodes + +EXEC=../../../test_execs/pese-nlev30/pese-nlev30 # set name of executable +srun -n 600 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/jobscript-darwin.sh b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/jobscript-darwin.sh new file mode 100644 index 000000000000..9b3c82f0bf1b --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/jobscript-darwin.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test2-0 on a mac running Darwin +# +# usage: ./jobscript-... + +EXEC=../../../test_execs/pese-nlev30/pese-nlev30 # set name of executable +openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/jobscript-edison-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/jobscript-edison.sh similarity index 86% rename from components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/jobscript-edison-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/jobscript-edison.sh index 0eb2b1195fa2..3223b059cf16 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/jobscript-edison-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/jobscript-edison.sh @@ -11,6 +11,6 @@ #SBATCH -t 00:05:00 # run time (hh:mm:ss) #SBATCH -A acme # charge hours to account 1 -EXEC=../../test_execs/preqx-nlev30-interp/preqx-nlev30-interp # set name of executable -srun -n 600 $EXEC < ./namelist-default.nl # launch simulation +EXEC=../../../test_execs/pese-nlev30/pese-nlev30 # set name of executable +srun -n 600 $EXEC < ./namelist-default.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/movies/CMakeLists.txt similarity index 100% rename from components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/movies/CMakeLists.txt rename to components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/movies/CMakeLists.txt diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/namelist-default.nl b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/namelist-default.nl similarity index 94% rename from components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/namelist-default.nl rename to components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/namelist-default.nl index 3aa734a526c7..726d02da7780 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/namelist-default.nl +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/namelist-default.nl @@ -34,15 +34,15 @@ vtop = 2.05e-1 ! vertical coordinate at top of atm (z=12000m) / &analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file + output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps output_frequency = 4, ! output every 4 hours output_varnames1 ='T','ps','u','v','omega' ! variables to write to file interp_type = 0 ! 0=native grid, 1=bilinear output_type ='netcdf' ! netcdf or pnetcdf num_io_procs = 16 - interp_nlat = 256 - interp_nlon = 512 + interp_nlat = 91 + interp_nlon = 360 / &prof_inparm profile_outpe_num = 100 diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/namelist-lowres.nl similarity index 96% rename from components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/namelist-lowres.nl rename to components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/namelist-lowres.nl index db85ee631d92..d567d69d0bc3 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/namelist-lowres.nl +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/namelist-lowres.nl @@ -35,7 +35,7 @@ vtop = 2.05e-1 ! vertical coordinate at top of atm (z=12000m) / &analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file + output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps output_frequency = 1, ! number of hours (when timeunits=2) output_varnames1 ='T','ps','u','v','omega' ! variables to write to file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/plot_z_lon.ncl b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/plot_z_lon.ncl new file mode 100755 index 000000000000..4445d7725696 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/pese/plot_z_lon.ncl @@ -0,0 +1,199 @@ +; plot_z_lon +; +; Plots vertical cross section from interpolated data +; +; usage plot_z_lon t=(time_index) n=(variable_number) +; n = (0,1,2) => (omega,u,v) + + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + + T0 = 300.d0 ; temperature (K) + lapse = 0.0065d0 ; temperature lapse rate (K/m) + ztop = 12000.d0 ; model top (m) + Rd = 287.0d0 ; Ideal gas const dry air (J kg^-1 K^1) + g = 9.80616d0 ; Gravity (m s^2) + Cp = 1004.5d0 ; Specific heat capacity (J kg^-1 K^1) + pi = 4.d0*atan(1.d0) ; pi + p0 = 100000.0 ; reference pressure + + exponent = g/(Rd*lapse) + exponent_r= 1.0/exponent + + ;_________________________________________________ + ; open file and read in data(time,lev,lat,long) + + f = addfile("./movies/dcmip2012_test2_01.nc","r") + lat = f->lat + lon = f->lon + lev = f->lev + time = f->time + hyam = f->hyam + hybm = f->hybm + + nlat = getfilevardimsizes(f, "lat" ) + nlon = getfilevardimsizes(f, "lon" ) + nlev = getfilevardimsizes(f, "lev" ) + nt = dimsizes(time) + + ;_________________________________________________ + ; get time index from command line + + t_index = nt-1 + if(isvar("t")) then + t_index = t + end if + print ("t_index: t = " + t_index) + + equator = nlat/2 + + ;_________________________________________________ + ; load data + + var_num = 0 + if(isvar("n")) then + var_num = n + end if + + omega = f->omega(t_index,:,equator,:) + u = f->u(t_index,:,equator,:) + v = f->v(t_index,:,equator,:) + + print("max u ="+max(u)+" min u="+min(u)) + print("max v ="+max(v)+" min v="+min(v)) + print("max omega="+max(omega)+" min omega="+min(omega)) + + if(var_num .eq. 0) then + var = f->omega (t_index,:,equator,:) + var_name = "omega" + end if + + if(var_num .eq. 1) then + var = f->u(t_index,:,equator,:) + var_name = "u" + end if + + if(var_num .eq. 2) then + var = f->v(t_index,:,equator,:) + var_name = "v" + end if + + ps = f->ps(t_index,1,equator,:) + + norm = max(abs(var)) + + amax = 1.2 + step = amax/6 + + var = var/norm + ;var@long_name = " " + + ;_________________________________________________ + ; Set up height coordinate with constant z spacing + + z = new((/nlev/),double) + dz = 12000.0/nlev + + do kk=0,nlev-1 + z(kk) = (12000.0-0.5*dz)-(kk)*dz + end do + + ; Get surface elevation curve + + zs = (T0/lapse)*(1.0-(ps/p0)^exponent_r) + + ;_________________________________________________ + ; Get pressure and z position of eta-level midpoints + + z_m = new((/nlev,nlon/),double) + p_m = new((/nlev,nlon/),double) + + do k =0, nlev-1 + p_m(k,:) = hyam(k)*p0 + hybm(k)*ps + z_m(k,:) = (T0/lapse)*(1.0-(p_m(k,:)/p0)^exponent_r) + end do + + ;_________________________________________________ + ; Interpolate var field from eta coord to z coord + + var_z = new( (/nlev,nlon/), double) + + do i = 0, nlon-1 + z_col = z_m(::-1,i) ; single column of z field, inverted + var_col = var(::-1,i) ; single column of var field, inverted + var_z(:,i) = ftcurv(z_col, var_col, z) ; interpolate to evenly spaced z using splines + end do + + ;_________________________________________________ + ; Setup the plot + + res1 = True + res1@gsnDraw = False + res1@gsnFrame = False ; don't draw yet + res1@gsnMaximize = True + res1@cnFillOn = True + res1@cnLinesOn = False ; turn off contour lines + res1@vpWidthF = 1 + res1@vpHeightF = 0.5 + res1@cnInfoLabelOn = False + res1@cnLineLabelsOn = False + res1@lbLabelBarOn = True + res1@sfXArray = lon ; use lon as plot x-axis + res1@sfYArray = z ; use z for y axis + res1@trYReverse = False ; reverses y-axis, false + res1@tiXAxisString = "longitude" ; x-axis title + res1@tiYAxisString = "height" ; y-axis title + res1@gsnPaperOrientation = "portrait" + res1@lbBoxLinesOn = False ; Turn off labelbar box lines + res1@lbLabelStride = 1 + res1@lbAutoManage = False + res1@lbTitlePosition = "Bottom" + res1@lbOrientation = "Vertical" + res1@lbTitleString = sprintf("*%7.3f", norm) + res1@cnFillPalette = "temp_diff_18lev"; "WhBlGrYeRe"; "BlWhRe"; "BlueWhiteOrangeRed"; "BlueRed"; "cmp_b2r"; + res1@cnSpanFillPalette = True + res1@tiXAxisFontHeightF = 0.020 + res1@tmXBLabelFontHeightF = 0.015 + res1@lbTitleFontHeightF = 0.020 + res1@lbLabelFontHeightF = 0.020 + res1@tiMainFont = 22 + res1@tiMainFontHeightF = 0.020 + res1@gsnCenterStringFontHeightF = 0.025 ; to set the gsnLeft/Center/RightString font heights, + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = amax-0.5*step; 0.9; ; max contour color label + res1@cnMinLevelValF =-amax+0.5*step; -0.9; ; min contour color label + res1@cnLevelSpacingF = step; 0.1; ; contour color spacing + res1@gsnCenterString ="" + res1@tiMainString = "DCMIP 2012 Test 2-0 "+var_name+" t = "+sprintf("%5.2f", time(t_index)) + " days" + + wks_type="pdf" + wks_type@wkPaperWidthF = 11 + wks_type@wkPaperHeightF = 7.5 + filename ="dcmip2012_test2_0_"+var_name+"_t"+t_index + wks = gsn_open_wks(wks_type,filename) + print("outputfilename = "+filename+".pdf") + + gsres = True + gsres@gsLineColor = "black" + gsres@gsFillColor = "black" + gsres@tfPolyDrawOrder = "PostDraw" + gsres@gsLineThicknessF = 5.0 + + zs(0)=0 + zs(nlon-1)=0 + + plot1 = gsn_csm_contour (wks,var(:,:),res1) + mountains = gsn_add_polygon (wks, plot1,lon, zs, gsres) + mt = gsn_add_polyline(wks, plot1, lon, zs, gsres) + + draw(plot1) + + frame(wks) + +end + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/CMakeLists.txt new file mode 100644 index 000000000000..ecfb00937223 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/build-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/build.sh similarity index 88% rename from components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/build-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/build.sh index b345186a1cca..17afdae1c7e9 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/build-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/build.sh @@ -1,6 +1,6 @@ #!/bin/bash cwd=`pwd` -cd ../.. +cd ../../.. echo "make preqx-nlev30-interp" make -j4 preqx-nlev30-interp cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/jobscript-cori.sh new file mode 100644 index 000000000000..672e517756e2 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/jobscript-cori.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test3-1 on the NERSC Cori machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip2-0 # job name +#SBATCH -o out_dcmip2-0.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 600 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:05:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 +#SBATCH -C haswell # use Haswell nodes + +EXEC=../../../test_execs/preqx-nlev30-interp/preqx-nlev30-interp # set name of executable +srun -n 600 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/jobscript-darwin.sh b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/jobscript-darwin.sh new file mode 100644 index 000000000000..91f7d4df9084 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/jobscript-darwin.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test2-0 on a mac running Darwin +# +# usage: ./jobscript-... + +EXEC=../../../test_execs/preqx-nlev30-interp/preqx-nlev30-interp # set name of executable +openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/jobscript-edison.sh b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/jobscript-edison.sh new file mode 100644 index 000000000000..eb308baa836f --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/jobscript-edison.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test3-1 on the NERSC Edison machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip2-0 # job name +#SBATCH -o out_dcmip2-0.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 600 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:05:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 + +EXEC=../../../test_execs/preqx-nlev30-interp/preqx-nlev30-interp # set name of executable +srun -n 600 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/movies/CMakeLists.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/namelist-default.nl b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/namelist-default.nl new file mode 100644 index 000000000000..726d02da7780 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/namelist-default.nl @@ -0,0 +1,50 @@ +! +! namelist for dcmip2012 test2-0: steady-state atmosphere with orography +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test2_0" ! test identifier + ne = 30 ! number of elements per cube face + qsize = 1 ! num tracer fields + ndays = 6 ! num simulation days: 0 = use nmax steps + statefreq = 200 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 = new run + tstep = 20 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 1 ! 1 => default method + smooth = 0 ! timestep smooting + nu = 3e14 ! hyperviscosity + nu_s = 3e14 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + omega = 0.0 ! earth angular speed = 0.0 +/ +&filter_nl/ +&solver_nl + precon_method = "identity" + maxits = 50 + tol = 1.e-7 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 2.05e-1 ! vertical coordinate at top of atm (z=12000m) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps + output_frequency = 4, ! output every 4 hours + output_varnames1 ='T','ps','u','v','omega' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlat = 91 + interp_nlon = 360 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/namelist-lowres.nl new file mode 100644 index 000000000000..d567d69d0bc3 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/namelist-lowres.nl @@ -0,0 +1,49 @@ +! +! namelist for dcmip2012 test2-0: steady-state atmosphere with orography +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test2_0" ! test identifier + ne = 7 ! number of elements per cube face 7 -> 4dg + qsize = 1 ! num tracer fields + ndays = 6 ! num simulation days: 0 = use nmax steps + statefreq = 100 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 = new run + tstep = 20 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 1 ! 1 => default method + smooth = 0 ! timestep smooting (nonzero smoothing breaks this test) + nu = 1e15 ! hyperviscosity + nu_s = 1e15 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + omega = 0.0 ! earth angular speed = 0.0 + dcmip2_0_zetam = 0.785 ! mountain half-width = pi/4 +/ +&filter_nl/ +&solver_nl + precon_method = "identity" + maxits = 50 + tol = 1.e-7 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 2.05e-1 ! vertical coordinate at top of atm (z=12000m) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps + output_frequency = 1, ! number of hours (when timeunits=2) + output_varnames1 ='T','ps','u','v','omega' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/plot_z_lon.ncl b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/plot_z_lon.ncl similarity index 100% rename from components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/plot_z_lon.ncl rename to components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/preqx/plot_z_lon.ncl diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/.DS_Store b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/.DS_Store new file mode 100644 index 000000000000..5008ddfcf53c Binary files /dev/null and b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/.DS_Store differ diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/CMakeLists.txt index 23dc25d6dbac..90f557418413 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/CMakeLists.txt +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/CMakeLists.txt @@ -1,14 +1,10 @@ -file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts -file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts -file(GLOB NAMELISTS "*.nl") # get namelists -file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts -file(GLOB M_FILES "*.m") # get matlab scripts - -# Copy test files to build -install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - # Create a symbolic link to the vertical coordinate directory execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/test/vcoord ${CMAKE_CURRENT_BINARY_DIR}/vcoord ) -# Add subdirectory for results -add_subdirectory(movies) +# Add all subdirectories to cmake build +file(GLOB FILES "*") # get list of all files +foreach(file ${FILES}) # for each file + if(IS_DIRECTORY ${file}) # if the file is a dir + add_subdirectory(${file}) # add subdir to cmake + endif() +endforeach(file) \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/CMakeLists.txt new file mode 100644 index 000000000000..ecfb00937223 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/build.sh b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/build.sh new file mode 100644 index 000000000000..67dfb4202649 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cwd=`pwd` +cd ../../.. +echo "make pese-nlev60" +make -j4 pese-nlev60 +cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/jobscript-cori.sh new file mode 100644 index 000000000000..babfae8e6d80 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/jobscript-cori.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test2-1 on the NERSC Cori machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip2-1 # job name +#SBATCH -o out_dcmip2-1.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 600 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:10:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 +#SBATCH -C haswell # use Haswell nodes + +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable +srun -n 600 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/jobscript-darwin.sh b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/jobscript-darwin.sh new file mode 100644 index 000000000000..7384cf0bd21d --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/jobscript-darwin.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test2-1 on a mac running Darwin +# +# usage: ./jobscript-... + +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable +openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/jobscript-edison-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/jobscript-edison.sh similarity index 90% rename from components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/jobscript-edison-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/jobscript-edison.sh index 325755be28db..50015aa60a54 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/jobscript-edison-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/jobscript-edison.sh @@ -11,6 +11,6 @@ #SBATCH -t 00:10:00 # run time (hh:mm:ss) #SBATCH -A acme # charge hours to account 1 -EXEC=../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable srun -n 600 $EXEC < ./namelist-default.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/movies/CMakeLists.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/namelist-default.nl b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/namelist-default.nl similarity index 97% rename from components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/namelist-default.nl rename to components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/namelist-default.nl index 5104e175c677..2c8e5ab4a062 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/namelist-default.nl +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/namelist-default.nl @@ -40,7 +40,7 @@ vtop = 3.2818e-2 ! vertical coordinate at top of atm (z=30km) / &analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file + output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 0, ! 1=days, 2=hours, 0=timesteps output_frequency = 1000, ! 100s /0.1s = 1000 steps between outputs output_varnames1 ='T','ps','u','v','omega' ! variables to write to file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/namelist-lowres.nl similarity index 97% rename from components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/namelist-lowres.nl rename to components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/namelist-lowres.nl index f0208df70d3c..89ced178a5df 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/namelist-lowres.nl +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/namelist-lowres.nl @@ -40,7 +40,7 @@ vtop = 3.2818e-2 ! vertical coordinate at top of atm (z=30km) / &analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file + output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 0, ! 1=days, 2=hours, 0=timesteps output_frequency = 500, ! 100s /0.1s = 1000 steps between outputs output_varnames1 ='T','ps','u','v','omega' ! variables to write to file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/plot_lon_vs_z.ncl b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/plot_lon_vs_z.ncl similarity index 100% rename from components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/plot_lon_vs_z.ncl rename to components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/pese/plot_lon_vs_z.ncl diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/CMakeLists.txt new file mode 100644 index 000000000000..ecfb00937223 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/build-preqx-native.sh b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/build-native.sh similarity index 88% rename from components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/build-preqx-native.sh rename to components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/build-native.sh index d3b96a2b4a3f..ccbef348df49 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/build-preqx-native.sh +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/build-native.sh @@ -1,6 +1,6 @@ #!/bin/bash cwd=`pwd` -cd ../.. +cd ../../.. echo "make preqx-nlev60-native" make -j preqx-nlev60-native cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/build-preqx-interp.sh b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/build.sh similarity index 88% rename from components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/build-preqx-interp.sh rename to components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/build.sh index 5e6c472d3873..962c9a4e82c8 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/build-preqx-interp.sh +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/build.sh @@ -1,6 +1,6 @@ #!/bin/bash cwd=`pwd` -cd ../.. +cd ../../.. echo "make preqx-nlev60-interp" make -j preqx-nlev60-interp cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/jobscript-cori.sh new file mode 100644 index 000000000000..113fc21c89dc --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/jobscript-cori.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test2-1 on the NERSC Cori machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip2-1 # job name +#SBATCH -o out_dcmip2-1.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 600 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:10:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 +#SBATCH -C haswell # use Haswell nodes + +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +srun -n 600 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/jobscript-darwin-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/jobscript-darwin.sh similarity index 71% rename from components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/jobscript-darwin-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/jobscript-darwin.sh index ddad687df7a4..cf3f5d5c639f 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/jobscript-darwin-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/jobscript-darwin.sh @@ -5,5 +5,5 @@ # usage: ./jobscript-... # launch the simulation -EXEC=../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable -openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/jobscript-edison.sh b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/jobscript-edison.sh new file mode 100644 index 000000000000..e061759484eb --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/jobscript-edison.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test2-1 on the NERSC Edison machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip2-1 # job name +#SBATCH -o out_dcmip2-1.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 600 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:10:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 + +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +srun -n 600 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/movies/CMakeLists.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/namelist-default.nl b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/namelist-default.nl new file mode 100644 index 000000000000..2c8e5ab4a062 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/namelist-default.nl @@ -0,0 +1,54 @@ +! +! namelist for dcmip2012 test2-1: nonhydro mountain waves without shear +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test2_1" ! test identifier + ne = 20 ! number of elements per cube face + qsize = 1 ! num tracer fields + ndays = 0 ! num simulation days: 0 => use nmax steps + nmax = 72000 ! 7200s / 0.1s per step = 72000 steps + statefreq = 500 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 0.1 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 1 ! 1 => default method + smooth = 0.05 ! timestep smooting + nu = 8e6 ! reduced planet hyperviscosity hv/500^3 + nu_s = 8e6 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + rearth = 12752.0 ! reduced planet radius rearth = a/500.0 + omega = 0.0 ! earth angular speed = 0.0 + dcmip2_x_ueq = 20.0 ! windspeed at equator (m/s) + dcmip2_x_h0 = 250.0 ! mountain height (m) + dcmip2_x_d = 5000.0 ! mountain half-width (m) + dcmip2_x_xi = 4000.0 ! mountain wavelength (m) +/ +&filter_nl/ +&solver_nl + precon_method = "identity" + maxits = 50 + tol = 1.e-7 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 3.2818e-2 ! vertical coordinate at top of atm (z=30km) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 0, ! 1=days, 2=hours, 0=timesteps + output_frequency = 1000, ! 100s /0.1s = 1000 steps between outputs + output_varnames1 ='T','ps','u','v','omega' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/namelist-lowres.nl new file mode 100644 index 000000000000..89ced178a5df --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/namelist-lowres.nl @@ -0,0 +1,54 @@ +! +! namelist for dcmip2012 test2-1: nonhydro mountain waves without shear +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test2_1" ! test identifier + ne = 7 ! number of elements per cube face + qsize = 1 ! num tracer fields + ndays = 0 ! num simulation days: 0 => use nmax steps + nmax = 72000 ! 7200s / 0.1s per step = 72000 steps + statefreq = 20 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 0.2 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 1 ! 1 => default method + smooth = 0.05 ! timestep smooting + nu = 7e5 ! reduced planet hyperviscosity hv/500^3 + nu_s = 7e5 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + rearth = 12752.0 ! reduced planet radius rearth = a/500.0 + omega = 0.0 ! earth angular speed = 0.0 + dcmip2_x_ueq = 20.0 ! windspeed at equator (m/s) + dcmip2_x_h0 = 250.0 ! mountain height (m) + dcmip2_x_d = 5000.0 ! mountain half-width (m) + dcmip2_x_xi = 8000.0 ! mountain wavelength (m) +/ +&filter_nl/ +&solver_nl + precon_method = "identity" + maxits = 50 + tol = 1.e-7 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 3.2818e-2 ! vertical coordinate at top of atm (z=30km) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 0, ! 1=days, 2=hours, 0=timesteps + output_frequency = 500, ! 100s /0.1s = 1000 steps between outputs + output_varnames1 ='T','ps','u','v','omega' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/plot_lon_vs_z.ncl b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/plot_lon_vs_z.ncl new file mode 100644 index 000000000000..9f49f9ecc8c3 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/preqx/plot_lon_vs_z.ncl @@ -0,0 +1,217 @@ +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + + ;_____________________________________________________________________ + ; open file and read in data(time,lev,lat,long) from 0 to n-1 + + data_dir ="./movies/dcmip2012_test2_11.nc" + print ("data_dir="+data_dir) + f = addfile(data_dir,"r") + + lat = f->lat + lon = f->lon + lev = f->lev + time = f->time + + nlat = dimsizes(lat) + nlon = dimsizes(lon) + nlev = dimsizes(lev) + nt = dimsizes(time) + + ti = nt-1 ; set t to max time + if(isvar("t")) then + ti = t ; use t from command line if possible + end if + + X = 500 ; small-planet scale factor + dt = 0.1 ; seconds per step for hydrostatic + output_interval = 100 ;1000 ; steps per output + sec_per_day = 3600.0*24 + + + t_sec = time(ti)*sec_per_day + earth_days = time(ti)*X + print ("ti="+ti+" t= "+t_sec+"s earth_days="+earth_days) + + equator = nlat/2 ; The equator is at nlat/2 + + ; Select var_choice + ; 1 - Temperature Pert + ; 2 - Zonal Velocity Pert + ; 3 - Vertical Pressure Velocity + + var_choice = 1 + if(isvar("n")) then + var_choice = n + end if + + if (var_choice .eq. 1) then ; Select T + var1 = f->T(ti,:,equator,:) - 300 + end if + + if (var_choice .eq. 2) then ; Select u + var1 = f->u(ti,:,equator,:) - 20.0 + end if + + if (var_choice .eq. 3) then ; Select omega + var1 = f->omega(ti,:,equator,:)-0.0 + end if + + if (var_choice .eq. 4) then ; Select v + var1 = f->v(ti,:,equator,:)-0.0 + end if + + ;var1@units = "" + ;var1@long_name = " " + + ;_____________________________________________________________________ + ; Get z positions of eta levels + + Teq = 300.d0 ; Temperature at Equator + Rd = 287.0d0 ; Ideal gas const dry air (J kg^-1 K^1) + g = 9.80616d0 ; Gravity (m s^2) + H = Rd*Teq/g + z_eta = -H * log(lev) + + + epsilon = 0.001 ;meter + do i=1,nlev-1 + if( z_eta(i) .eq. z_eta(i-1) ) then + z_eta(i) = z_eta(i)-epsilon + end if + end do + +;print("z_eta="+z_eta) + + ;_____________________________________________________________________ + ; Get evenly spaced z levels + + z = new((/nlev/),double) + dz = 30000.0/nlev + do kk=0,nlev-1 + z(kk) = (30000.0-0.5*dz)-(kk)*dz + end do + + ;_________________________________________________ + ; Interpolate from z_eta to evenly spaced z + + + var_z = new( (/nlev,nlon/), double) + + do i = 0, nlon-1 + + z_col = z_eta(::-1) ; single column of z field, inverted + var_col = var1 (::-1,i) ; single column of var field, inverted + var_z(:,i) = ftcurv(z_col, var_col, z) ; interpolate to evenly spaced z using splines + + end do + + + ;_____________________________________________________________________ + ;plot = new (1, graphic) ; define plot - need 3 panels + res1 = True + res1@gsnDraw = False ; panel plot + res1@gsnFrame = False ; don't draw yet + res1@gsnMaximize = True + res1@cnFillOn = True + res1@cnLinesOn = True + res1@gsnContourLineThicknessesScale = 1.5 + ;res1@gsnSpreadColors = True + res1@lbLabelAutoStride = True + res1@gsnCenterString = "" + res1@tiMainString = "" + res1@vpWidthF = 2.0 + res1@vpHeightF = 1.0 + res1@cnInfoLabelOn = False + res1@cnLineLabelsOn = False + res1@lbLabelBarOn = True + res1@lbOrientation = "horizontal" + res1@lbLabelStride = 1 + res1@gsnPaperOrientation = "portrait" + res1@sfXArray = lon ; uses lon as plot x-axis + res1@sfYArray = z/1000.0 ; uses z for y axis + res1@trYReverse = False ; reverses y-axis, false + res1@tiYAxisString = "height (km)" ; y-axis title + res1@tiXAxisString = "" ; x-axis title + res1@gsnStringFontHeightF = 0.04 + res1@tmXBLabelFontHeightF = 0.03 + res1@tmYLLabelFontHeightF = 0.03 + res1@tiYAxisFontHeightF = 0.03 + res1@cnLevelSelectionMode = "AutomaticLevels" + ;res1@cnFillPalette ="temp_diff_18lev"; "BlWhRe"; "BlueWhiteOrangeRed"; "BlueRed"; "cmp_b2r"; + res1@lbOrientation = "Vertical" + +; res1@gsnContourZeroLineThicknessF = 0.0 + + + print("max = "+max(var1)+" min="+min(var1)) + prefix = "dcmip2012_test2_1_" + + if (var_choice .eq. 1) then ; Setup for T' + + pltTitle = sprintf("CAM-SE Test 2-1 T' t=%.1f s",t_sec) + fileName = sprinti(prefix + "T_t%i",ti) + + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = 1.2 ; max contour color label + res1@cnMinLevelValF =-1.2 ; min contour color label + res1@cnLevelSpacingF = 0.2 ; contour color spacing + + end if + + if (var_choice .eq. 2) then ; Setup for U' + + pltTitle = sprintf("CAM-SE Test 2-1 U' t=%.1f s",t_sec) + fileName = sprinti(prefix + "u_t%i",ti) + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = 2 ; max contour color label + res1@cnMinLevelValF =-2 ; min contour color label + res1@cnLevelSpacingF = 0.4 ; contour color spacing + + end if + + if (var_choice .eq. 3) then ; Setup for OMEGA + + pltTitle = sprintf("CAM-SE Test 2-1 Omega t=%.1f s",t_sec) + fileName = sprinti(prefix + "omega_t%i",ti) + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = 30.0 ; max contour color label + res1@cnMinLevelValF =-30.0 ; min contour color label + res1@cnLevelSpacingF = 5.0 ; contour color spacing + + end if + + if (var_choice .eq. 4) then ; Setup for V + + pltTitle = sprintf("CAM-SE Test 2-1 V t=%.1f s",t_sec) + fileName = sprinti(prefix + "v_t%i",ti) + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = 1.8 ; max contour color label + res1@cnMinLevelValF =-1.8 ; min contour color label + res1@cnLevelSpacingF = 0.4 ; contour color spacing + + end if + print("filename = "+fileName) + + wks_type = "pdf" + wks_type@wkPaperHeightF = 8 + wks_type@wkPaperWidthF = 11 + wks_type@wkOrientation = "portrait" + wks = gsn_open_wks(wks_type,fileName) + gsn_define_colormap(wks,"gui_default") ; Colormap + + res1@gsnCenterString = pltTitle + + ;plot = gsn_csm_contour(wks,var1(:,:),res1) ; plot var1 + plot = gsn_csm_contour(wks,var_z(:,:),res1) ; plot var1 + plot1 = ColorNegDashZeroPosContour(plot,"transparent","black","transparent") + + draw(plot1) + frame(wks) + +end + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/.DS_Store b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/.DS_Store new file mode 100644 index 000000000000..5008ddfcf53c Binary files /dev/null and b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/.DS_Store differ diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/CMakeLists.txt index 23dc25d6dbac..90f557418413 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/CMakeLists.txt +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/CMakeLists.txt @@ -1,14 +1,10 @@ -file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts -file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts -file(GLOB NAMELISTS "*.nl") # get namelists -file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts -file(GLOB M_FILES "*.m") # get matlab scripts - -# Copy test files to build -install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - # Create a symbolic link to the vertical coordinate directory execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/test/vcoord ${CMAKE_CURRENT_BINARY_DIR}/vcoord ) -# Add subdirectory for results -add_subdirectory(movies) +# Add all subdirectories to cmake build +file(GLOB FILES "*") # get list of all files +foreach(file ${FILES}) # for each file + if(IS_DIRECTORY ${file}) # if the file is a dir + add_subdirectory(${file}) # add subdir to cmake + endif() +endforeach(file) \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/build-preqx-native.sh b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/build-preqx-native.sh deleted file mode 100644 index d3b96a2b4a3f..000000000000 --- a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/build-preqx-native.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -cwd=`pwd` -cd ../.. - echo "make preqx-nlev60-native" - make -j preqx-nlev60-native -cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/CMakeLists.txt new file mode 100644 index 000000000000..ecfb00937223 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/build.sh b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/build.sh new file mode 100644 index 000000000000..67dfb4202649 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cwd=`pwd` +cd ../../.. +echo "make pese-nlev60" +make -j4 pese-nlev60 +cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/jobscript-cori.sh new file mode 100644 index 000000000000..babfae8e6d80 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/jobscript-cori.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test2-1 on the NERSC Cori machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip2-1 # job name +#SBATCH -o out_dcmip2-1.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 600 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:10:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 +#SBATCH -C haswell # use Haswell nodes + +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable +srun -n 600 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/jobscript-darwin.sh b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/jobscript-darwin.sh new file mode 100644 index 000000000000..7384cf0bd21d --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/jobscript-darwin.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test2-1 on a mac running Darwin +# +# usage: ./jobscript-... + +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable +openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/jobscript-edison.sh b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/jobscript-edison.sh new file mode 100644 index 000000000000..50015aa60a54 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/jobscript-edison.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test2-1 on the NERSC Edison machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip2-1 # job name +#SBATCH -o out_dcmip2-1.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 600 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:10:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 + +EXEC=../../../test_execs/pese-nlev60/pese-nlev60 # set name of executable +srun -n 600 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/movies/CMakeLists.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/namelist-default.nl b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/namelist-default.nl similarity index 97% rename from components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/namelist-default.nl rename to components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/namelist-default.nl index d21d434b4ed5..5bd85c0cce31 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/namelist-default.nl +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/namelist-default.nl @@ -40,7 +40,7 @@ vtop = 3.2818e-2 ! vertical coordinate at top of atm (z=30km) / &analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file + output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 0, ! 1=days, 2=hours, 0=timesteps output_frequency = 1000, ! 100s /0.1s = 1000 steps between outputs output_varnames1 ='T','ps','u','v','omega' ! variables to write to file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/namelist-lowres.nl similarity index 97% rename from components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/namelist-lowres.nl rename to components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/namelist-lowres.nl index fd14e7c72547..a1b8e23bab74 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/namelist-lowres.nl +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/namelist-lowres.nl @@ -40,7 +40,7 @@ vtop = 3.2818e-2 ! vertical coordinate at top of atm (z=30km) / &analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file + output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 0, ! 1=days, 2=hours, 0=timesteps output_frequency = 500, ! 100s /0.1s = 1000 steps between outputs output_varnames1 ='T','ps','u','v','omega' ! variables to write to file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/plot_lon_vs_z.ncl b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/plot_lon_vs_z.ncl similarity index 100% rename from components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/plot_lon_vs_z.ncl rename to components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/pese/plot_lon_vs_z.ncl diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/CMakeLists.txt new file mode 100644 index 000000000000..ecfb00937223 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/build-preqx-interp.sh b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/build.sh similarity index 88% rename from components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/build-preqx-interp.sh rename to components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/build.sh index 5e6c472d3873..962c9a4e82c8 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.1_nh_mountain_waves_no_shear/build-preqx-interp.sh +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/build.sh @@ -1,6 +1,6 @@ #!/bin/bash cwd=`pwd` -cd ../.. +cd ../../.. echo "make preqx-nlev60-interp" make -j preqx-nlev60-interp cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/jobscript-cori.sh new file mode 100644 index 000000000000..0d03734e984e --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/jobscript-cori.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test3-1 on the NERSC Cori machine +# +# usage: sbatch jobscript-... + +#SBATCH -J dcmip2-2 # job name +#SBATCH -o out_dcmip2-2.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 600 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:10:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 +#SBATCH -C haswell # use Haswell nodes + +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +srun -n 600 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/jobscript-darwin-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/jobscript-darwin.sh similarity index 69% rename from components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/jobscript-darwin-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/jobscript-darwin.sh index a8480ff4bf87..3a1cfdc2ad5a 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/jobscript-darwin-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/jobscript-darwin.sh @@ -4,5 +4,5 @@ # # launch the simulation -EXEC=../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable -openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/jobscript-edison-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/jobscript-edison.sh similarity index 90% rename from components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/jobscript-edison-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/jobscript-edison.sh index eac3854e3f5a..4c71cf81f205 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/jobscript-edison-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/jobscript-edison.sh @@ -11,6 +11,6 @@ #SBATCH -t 00:10:00 # run time (hh:mm:ss) #SBATCH -A acme # charge hours to account 1 -EXEC=../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable +EXEC=../../../test_execs/preqx-nlev60-interp/preqx-nlev60-interp # set name of executable srun -n 600 $EXEC < ./namelist-default.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/movies/CMakeLists.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/namelist-default.nl b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/namelist-default.nl new file mode 100644 index 000000000000..5bd85c0cce31 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/namelist-default.nl @@ -0,0 +1,54 @@ +! +! namelist for dcmip2012 test2-2: nonhydro mountain waves without shear +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test2_2" ! test identifier + ne = 20 ! number of elements per cube face + qsize = 1 ! num tracer fields + ndays = 0 ! num simulation days: 0 => use nmax steps + nmax = 72000 ! 7200s / 0.1s per step = 72000 steps + statefreq = 500 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 0.1 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 1 ! 1 => default method + smooth = 0.05 ! timestep smooting + nu = 8e6 ! reduced planet hyperviscosity hv/500^3 + nu_s = 8e6 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + rearth = 12752.0 ! reduced planet radius rearth = a/500.0 + omega = 0.0 ! earth angular speed = 0.0 + dcmip2_x_ueq = 20.0 ! windspeed at equator (m/s) + dcmip2_x_h0 = 250.0 ! mountain height (m) + dcmip2_x_d = 5000.0 ! mountain half-width (m) + dcmip2_x_xi = 4000.0 ! mountain wavelength (m) +/ +&filter_nl/ +&solver_nl + precon_method = "identity" + maxits = 50 + tol = 1.e-7 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 3.2818e-2 ! vertical coordinate at top of atm (z=30km) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 0, ! 1=days, 2=hours, 0=timesteps + output_frequency = 1000, ! 100s /0.1s = 1000 steps between outputs + output_varnames1 ='T','ps','u','v','omega' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/namelist-lowres.nl new file mode 100644 index 000000000000..a1b8e23bab74 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/namelist-lowres.nl @@ -0,0 +1,54 @@ +! +! namelist for dcmip2012 test2-2: nonhydro mountain waves without shear +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test2_2" ! test identifier + ne = 7 ! number of elements per cube face + qsize = 1 ! num tracer fields + ndays = 0 ! num simulation days: 0 => use nmax steps + nmax = 72000 ! 7200s / 0.1s per step = 72000 steps + statefreq = 20 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 0.2 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 1 ! 1 => default method + smooth = 0.05 ! timestep smooting + nu = 7e5 ! reduced planet hyperviscosity hv/500^3 + nu_s = 7e5 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + rearth = 12752.0 ! reduced planet radius rearth = a/500.0 + omega = 0.0 ! earth angular speed = 0.0 + dcmip2_x_ueq = 20.0 ! windspeed at equator (m/s) + dcmip2_x_h0 = 250.0 ! mountain height (m) + dcmip2_x_d = 5000.0 ! mountain half-width (m) + dcmip2_x_xi = 8000.0 ! mountain wavelength (m) +/ +&filter_nl/ +&solver_nl + precon_method = "identity" + maxits = 50 + tol = 1.e-7 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 3.2818e-2 ! vertical coordinate at top of atm (z=30km) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 0, ! 1=days, 2=hours, 0=timesteps + output_frequency = 500, ! 100s /0.1s = 1000 steps between outputs + output_varnames1 ='T','ps','u','v','omega' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/plot_lon_vs_z.ncl b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/plot_lon_vs_z.ncl new file mode 100644 index 000000000000..33ebfda73f8a --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.2_nh_mountain_waves_with_shear/preqx/plot_lon_vs_z.ncl @@ -0,0 +1,216 @@ +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + + ;_____________________________________________________________________ + ; open file and read in data(time,lev,lat,long) from 0 to n-1 + + data_dir ="./movies/dcmip2012_test2_21.nc" + print ("data_dir="+data_dir) + f = addfile(data_dir,"r") + + lat = f->lat + lon = f->lon + lev = f->lev + time = f->time + + nlat = dimsizes(lat) + nlon = dimsizes(lon) + nlev = dimsizes(lev) + nt = dimsizes(time) + + ti = nt-1 ; set t to max time + if(isvar("t")) then + ti = t ; use t from command line if possible + end if + + X = 500 ; small-planet scale factor + dt = 0.1 ; seconds per step for hydrostatic + output_interval = 100 ;1000 ; steps per output + sec_per_day = 3600.0*24 + + + t = time(ti)*sec_per_day + earth_days = time(ti)*X + print ("ti="+ti+" t= "+t+"s earth_days="+earth_days) + + equator = nlat/2 ; The equator is at nlat/2 + + ; Select var_choice + ; 1 - Temperature Pert + ; 2 - Zonal Velocity Pert + ; 3 - Vertical Pressure Velocity + + var_choice = 1 + if(isvar("n")) then + var_choice = n + end if + + if (var_choice .eq. 1) then ; Select T + var1 = f->T(ti,:,equator,:) - 300 + end if + + if (var_choice .eq. 2) then ; Select u + var1 = f->u(ti,:,equator,:) - 20.0 + end if + + if (var_choice .eq. 3) then ; Select omega + var1 = f->omega(ti,:,equator,:)-0.0 + end if + + if (var_choice .eq. 4) then ; Select v + var1 = f->v(ti,:,equator,:)-0.0 + end if + + ;var1@units = "" + ;var1@long_name = " " + + ;_____________________________________________________________________ + ; Get z positions of eta levels + + Teq = 300.d0 ; Temperature at Equator + Rd = 287.0d0 ; Ideal gas const dry air (J kg^-1 K^1) + g = 9.80616d0 ; Gravity (m s^2) + H = Rd*Teq/g + z_eta = -H * log(lev) + + + epsilon = 0.001 ;meter + do i=1,nlev-1 + if( z_eta(i) .eq. z_eta(i-1) ) then + z_eta(i) = z_eta(i)-epsilon + end if + end do + +;print("z_eta="+z_eta) + + ;_____________________________________________________________________ + ; Get evenly spaced z levels + + z = new((/nlev/),double) + dz = 30000.0/nlev + do kk=0,nlev-1 + z(kk) = (30000.0-0.5*dz)-(kk)*dz + end do + + ;_________________________________________________ + ; Interpolate from z_eta to evenly spaced z + + + var_z = new( (/nlev,nlon/), double) + + do i = 0, nlon-1 + + z_col = z_eta(::-1) ; single column of z field, inverted + var_col = var1 (::-1,i) ; single column of var field, inverted + var_z(:,i) = ftcurv(z_col, var_col, z) ; interpolate to evenly spaced z using splines + + end do + + + ;_____________________________________________________________________ + res1 = True + res1@gsnDraw = False ; panel plot + res1@gsnFrame = False ; don't draw yet + res1@gsnMaximize = True + res1@cnFillOn = True + res1@cnLinesOn = True + res1@gsnContourLineThicknessesScale = 1.5 + ;res1@gsnSpreadColors = True + res1@lbLabelAutoStride = True + res1@gsnCenterString = "" + res1@tiMainString = "" + res1@vpWidthF = 2.0 + res1@vpHeightF = 1.0 + res1@cnInfoLabelOn = False + res1@cnLineLabelsOn = False + res1@lbLabelBarOn = True + res1@lbOrientation = "horizontal" + res1@lbLabelStride = 1 + res1@gsnPaperOrientation = "portrait" + res1@sfXArray = lon ; uses lon as plot x-axis + res1@sfYArray = z/1000.0 ; uses z for y axis + res1@trYReverse = False ; reverses y-axis, false + res1@tiYAxisString = "height (km)" ; y-axis title + res1@tiXAxisString = "" ; x-axis title + res1@gsnStringFontHeightF = 0.04 + res1@tmXBLabelFontHeightF = 0.03 + res1@tmYLLabelFontHeightF = 0.03 + res1@tiYAxisFontHeightF = 0.03 + res1@cnLevelSelectionMode = "AutomaticLevels" + res1@lbOrientation = "Vertical" + + ;res1@gsnContourZeroLineThicknessF = 0.0 + + ;res1@cnFillPalette ="temp_diff_18lev"; "BlWhRe"; "BlueWhiteOrangeRed"; "BlueRed"; "cmp_b2r"; + + print("max = "+max(var1)+" min="+min(var1)) + prefix = "dcmip2012_test2_2_" + + if (var_choice .eq. 1) then ; Setup for T' + + pltTitle = sprintf("CAM-SE Test 2-2 T' t=%.1f s",t) + fileName = sprinti(prefix + "T_t%i",ti) + + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = 1.2 ; max contour color label + res1@cnMinLevelValF =-1.2 ; min contour color label + res1@cnLevelSpacingF = 0.2 ; contour color spacing + + end if + + if (var_choice .eq. 2) then ; Setup for U' + + pltTitle = sprintf("CAM-SE Test 2-2 U' t=%.1f s",t) + fileName = sprinti(prefix + "u_t%i",ti) + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = 2 ; max contour color label + res1@cnMinLevelValF =-2 ; min contour color label + res1@cnLevelSpacingF = 0.4 ; contour color spacing + + end if + + if (var_choice .eq. 3) then ; Setup for OMEGA + + pltTitle = sprintf("CAM-SE Test 2-2 Omega t=%.1f s",t) + fileName = sprinti(prefix + "omega_t%i",ti) + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = 30.0 ; max contour color label + res1@cnMinLevelValF =-30.0 ; min contour color label + res1@cnLevelSpacingF = 5.0 ; contour color spacing + + end if + + if (var_choice .eq. 4) then ; Setup for V + + pltTitle = sprintf("CAM-SE Test 2-2 V t=%.1f s",t) + fileName = sprinti(prefix + "v_t%i",ti) + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = 1.8 ; max contour color label + res1@cnMinLevelValF =-1.8 ; min contour color label + res1@cnLevelSpacingF = 0.4 ; contour color spacing + + end if + print("filename = "+fileName) + + wks_type = "pdf" + wks_type@wkPaperHeightF = 8 + wks_type@wkPaperWidthF = 11 + wks_type@wkOrientation = "portrait" + wks = gsn_open_wks(wks_type,fileName) + gsn_define_colormap(wks,"gui_default") ; Colormap + + res1@gsnCenterString = pltTitle + +; plot = gsn_csm_contour(wks,var1(:,:),res1) ; plot var1 + plot = gsn_csm_contour(wks,var_z(:,:),res1) ; plot var1 + plot1 = ColorNegDashZeroPosContour(plot,"transparent","black","transparent") + + draw(plot1) + frame(wks) + +end + diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/.DS_Store b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/.DS_Store new file mode 100644 index 000000000000..5008ddfcf53c Binary files /dev/null and b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/.DS_Store differ diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/CMakeLists.txt index 23dc25d6dbac..90f557418413 100644 --- a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/CMakeLists.txt +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/CMakeLists.txt @@ -1,14 +1,10 @@ -file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts -file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts -file(GLOB NAMELISTS "*.nl") # get namelists -file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts -file(GLOB M_FILES "*.m") # get matlab scripts - -# Copy test files to build -install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - # Create a symbolic link to the vertical coordinate directory execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/test/vcoord ${CMAKE_CURRENT_BINARY_DIR}/vcoord ) -# Add subdirectory for results -add_subdirectory(movies) +# Add all subdirectories to cmake build +file(GLOB FILES "*") # get list of all files +foreach(file ${FILES}) # for each file + if(IS_DIRECTORY ${file}) # if the file is a dir + add_subdirectory(${file}) # add subdir to cmake + endif() +endforeach(file) \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/CMakeLists.txt new file mode 100644 index 000000000000..aa274b623c40 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/build-pese.sh b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/build-pese.sh new file mode 100644 index 000000000000..28aef63dace3 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/build-pese.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cwd=`pwd` +cd ../../.. + echo "make -j 4 pese-nlev20" + make -j 4 pese-nlev20 +cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/jobscript-cori.sh new file mode 100644 index 000000000000..e277b0f89710 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/jobscript-cori.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test3-1 on the NERSC Cori machine +# +#SBATCH -J dcmip3-1 # job name +#SBATCH -o out_dcmip3-1.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 600 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:05:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 +#SBATCH -C haswell # use Haswell nodes + +EXEC=../../../test_execs/pese-nlev20/pese-nlev20 # set name of executable +srun -n 600 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/jobscript-darwin-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/jobscript-darwin.sh similarity index 76% rename from components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/jobscript-darwin-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/jobscript-darwin.sh index 8e3502c10f80..fcbfb049d0df 100644 --- a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/jobscript-darwin-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/jobscript-darwin.sh @@ -3,5 +3,5 @@ # Jobscript for launching dcmip2012 test3-1 on a mac running Darwin # -EXEC=../../test_execs/preqx-nlev20-interp/preqx-nlev20-interp # set name of executable +EXEC=../../../test_execs/pese-nlev20/pese-nlev20 # set name of executable openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/jobscript-edison.sh b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/jobscript-edison.sh new file mode 100644 index 000000000000..89052aefff01 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/jobscript-edison.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test3-1 on the NERSC Edison machine +# +#SBATCH -J dcmip3-1 # job name +#SBATCH -o out_dcmip3-1.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 600 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:05:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 + +EXEC=../../../test_execs/pese-nlev20/pese-nlev20 # set name of executable +srun -n 600 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/movies/CMakeLists.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/namelist-default.nl b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/namelist-default.nl similarity index 97% rename from components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/namelist-default.nl rename to components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/namelist-default.nl index 63f363249859..0038c0f1bc90 100644 --- a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/namelist-default.nl +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/namelist-default.nl @@ -36,7 +36,7 @@ vtop = 2.73919e-1 ! vertical coordinate at top of atm (z=10000m) / &analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file + output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 0, ! 1=days, 2=hours, 0=timesteps output_frequency = 1000, ! 500 sec / 0.5 sec per step output_varnames1 ='T','ps','u','v','omega' ! variables to write to file diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/namelist-lowres.nl new file mode 100644 index 000000000000..ab3cc91995e4 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/namelist-lowres.nl @@ -0,0 +1,53 @@ +! +! PESE namelist for dcmip2012 test 3-1 nonhydrostatic gravity waves +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test3" ! test identifier + ne = 7 ! number of elements per cube face + qsize = 1 ! num tracer fields + ndays = 0 ! num simulation days: 0 = use nmax steps + nmax = 7200 ! total number of steps: 7200 = 3600s / tstep=0.5s + statefreq = 50 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 = new run + tstep = 0.5 ! largest timestep + integration = 'explicit' ! explicit time integration + tstep_type = 3 !1 ! 1 => default method + smooth = 0 ! timestep smooting (nonzero smoothing breaks this test) + nu = 8.0e6 ! reduced earth hyperviz + nu_s = 8.0e6 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + rearth = 50969.76 ! scaled earth radius = a/125.0 + omega = 0.0 ! earth angular speed = 0.0 + rsplit = 0 +/ +&filter_nl/ +&solver_nl + precon_method = "identity" + maxits = 50 + tol = 1.e-7 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 2.73919e-1 ! vertical coordinate at top of atm (z=10000m) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 0, ! 1=days, 2=hours, 0=timesteps + output_frequency = 200, ! 100 sec / 0.5 sec per step + output_varnames1 ='T','ps','u','v','omega' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + io_stride = 8 + interp_nlat = 65 + interp_nlon = 128 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/plot_omega.ncl b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/plot_omega.ncl similarity index 100% rename from components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/plot_omega.ncl rename to components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/plot_omega.ncl diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/plot_theta.ncl b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/plot_theta.ncl similarity index 100% rename from components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/plot_theta.ncl rename to components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/pese/plot_theta.ncl diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/CMakeLists.txt new file mode 100644 index 000000000000..aa274b623c40 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/build-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/build.sh similarity index 89% rename from components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/build-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/build.sh index 9f3412c5f654..b1821d61f993 100644 --- a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/build-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/build.sh @@ -1,6 +1,6 @@ #!/bin/bash cwd=`pwd` -cd ../.. +cd ../../.. echo "make -j 4 preqx-nlev20-interp" make -j 4 preqx-nlev20-interp cd $cwd \ No newline at end of file diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/jobscript-cori.sh new file mode 100644 index 000000000000..37b1c52cab6a --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/jobscript-cori.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test3-1 on the NERSC Cori machine +# +#SBATCH -J dcmip3-1 # job name +#SBATCH -o out_dcmip3-1.o%j # output and error file name (%j expands to jobID) +#SBATCH -n 600 # total number of mpi tasks requested +#SBATCH -p debug # queue (partition) -- normal, development, etc. +#SBATCH -t 00:05:00 # run time (hh:mm:ss) +#SBATCH -A acme # charge hours to account 1 +#SBATCH -C haswell # use Haswell nodes + +EXEC=../../../test_execs/preqx-nlev20-interp/preqx-nlev20-interp # set name of executable +srun -n 600 $EXEC < ./namelist-default.nl # launch simulation + diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/jobscript-darwin.sh b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/jobscript-darwin.sh new file mode 100644 index 000000000000..6882f5c280fc --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/jobscript-darwin.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# +# Jobscript for launching dcmip2012 test3-1 on a mac running Darwin +# + +EXEC=../../../test_execs/preqx-nlev20-interp/preqx-nlev20-interp # set name of executable +openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/jobscript-edison-preqx.sh b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/jobscript-edison.sh similarity index 90% rename from components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/jobscript-edison-preqx.sh rename to components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/jobscript-edison.sh index 02230cee724f..cab1298614e0 100644 --- a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/jobscript-edison-preqx.sh +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/jobscript-edison.sh @@ -9,6 +9,6 @@ #SBATCH -t 00:05:00 # run time (hh:mm:ss) #SBATCH -A acme # charge hours to account 1 -EXEC=../../test_execs/preqx-nlev20-interp/preqx-nlev20-interp # set name of executable +EXEC=../../../test_execs/preqx-nlev20-interp/preqx-nlev20-interp # set name of executable srun -n 600 $EXEC < ./namelist-default.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/movies/CMakeLists.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/namelist-default.nl b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/namelist-default.nl new file mode 100644 index 000000000000..0038c0f1bc90 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/namelist-default.nl @@ -0,0 +1,52 @@ +! +! namelist for dcmip2012 test 3-1 nonhydrostatic gravity waves +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test3" ! test identifier + ne = 27 ! number of elements per cube face + qsize = 1 ! num tracer fields + ndays = 0 ! num simulation days: 0 = use nmax steps + nmax = 18000 ! total number of steps: 3600s / tstep=0.2s + statefreq = 100 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 = new run + tstep = 0.2 ! largest timestep + integration = 'explicit' ! explicit time integration + tstep_type = 1 ! 1 => default method + smooth = 0 ! timestep smooting (nonzero smoothing breaks this test) + nu = 5.0e8 ! reduced earth hyperviz + nu_s = 5.0e8 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + rearth = 50969.76 ! scaled earth radius = a/125.0 + omega = 0.0 ! earth angular speed = 0.0 +/ +&filter_nl/ +&solver_nl + precon_method = "identity" + maxits = 50 + tol = 1.e-7 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 2.73919e-1 ! vertical coordinate at top of atm (z=10000m) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 0, ! 1=days, 2=hours, 0=timesteps + output_frequency = 1000, ! 500 sec / 0.5 sec per step + output_varnames1 ='T','ps','u','v','omega' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlat = 128 + interp_nlon = 256 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/namelist-lowres.nl similarity index 97% rename from components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/namelist-lowres.nl rename to components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/namelist-lowres.nl index f4c6987c33dc..5a5e9de6b50c 100644 --- a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/namelist-lowres.nl +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/namelist-lowres.nl @@ -36,7 +36,7 @@ vtop = 2.73919e-1 ! vertical coordinate at top of atm (z=10000m) / &analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file + output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 0, ! 1=days, 2=hours, 0=timesteps output_frequency = 200, ! 100 sec / 0.5 sec per step output_varnames1 ='T','ps','u','v','omega' ! variables to write to file diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/plot_omega.ncl b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/plot_omega.ncl new file mode 100644 index 000000000000..135dcac4b4f0 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/plot_omega.ncl @@ -0,0 +1,217 @@ +;_____________________________________________________________________ +; +; Plot potential temperature differences from background state +;_____________________________________________________________________ + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + +;_____________________________________________________________________ +; dcmip3-1 model parameters (must match those in dcmip123_mod.F90) + +ts = 0.1 ; timestep in seconds +interval= 100 ; steps per write + +g = 9.80616 ; grav const +a = 6371229.0 ; earth radius in meters +Rd = 287.0 ; dry gas const +cp = 1004.5 ; heat capacity const pressure +kappa = Rd/cp +pi = 3.141592654 +p_ref = 100000.0 ; reference pressure + +ztop = 10000.0 ; top of the atmosphere in z coords +X = 125.0 ; reduced Earth reduction factor +Om = 0.0 ; rotation rate of Earth +as = a/X ; new radius of small Earth +u0 = 20.0 ; max zonal wind speed +Teq = 300.0 ; temperature at equator +Peq = 100000.0 ; reference ps at equator +lambdac = 2.0*pi/3.0 ; lon of pert center +d = 5000.0 ; width for pert +phic = 0.0 ; lat of pert Center +delta_theta = 1.0 ; max amplitude of temp perturbation +Lz = 20000.0 ; vertical wavelength of pert +N = 0.010 ; Brunt-Vaisala frequency +N2 = N*N ; Brunt-Vaisala frequency squared +bigG = (g*g)/(N2*cp) ; constant + +;_____________________________________________________________________ +; open file, read in data (time,lev,lat,long) in range [0 to n-1] + +f = addfile("./movies/dcmip2012_test31.nc","r") ; open ncl data file + +lat1d = f->lat*pi/180 ; load 1d latitutde array +lon1d = f->lon*pi/180 ; load 1d longitude array +lev = f->lev ; load 1d vertical-level array +time = f->time + +nlon = getfilevardimsizes(f, "lon" ) ; get number of lon points +nlat = getfilevardimsizes(f, "lat" ) ; get number of lat points +nlev = getfilevardimsizes(f, "lev" ) ; get number of vertical levels +nt = dimsizes(time) + +lat = new( (/nlat,nlon/), typeof(lat1d)) +lon = new( (/nlat,nlon/), typeof(lon1d)) + +do j=0, nlon-1 + do i=0, nlat-1 + lat(i,j)=lat1d(i) + lon(i,j)=lon1d(j) + end do +end do + +print("nlon = "+nlon+" nlat = "+nlat+" nlev = "+nlev+" nt = "+nt) ; display data dimensions + +t_index = nt-1 ; set time to max time +if(isvar("t")) then + t_index = t ; use t from command line if possible +end if +print ("t_index="+t_index) + + +;_____________________________________________________________________ +; compute evenly spaced z-levels + +dz = ztop / nlev +z = new((/nlev/),double) + +do k=0,nlev-1 + z(k) = ztop - dz*(k+0.5) +end do + +;_____________________________________________________________________ +; define data fields + +T = f->T(0,:,:,:) ; read Temperature field +Ts0 = new( (/nlat,nlon/) , typeof(T), -9999.0) ; define initial surf temp field +ps0 = new( (/nlat,nlon/) , typeof(T), -9999.0) ; define initial surf pressure field +p0 = new( (/nlev,nlat,nlon/) , typeof(T), -9999.0) ; define mean temperature field +T_mean= new( (/nlev,nlat,nlon/) , typeof(T), -9999.0) ; define mean temperature field +theta_mean= new( (/nlev,nlat,nlon/) , typeof(T), -9999.0) ; define mean potential temperature field + +;_____________________________________________________________________ +; recompute initial conditions + +Ts0 = bigG + (Teq-bigG)*exp( -(u0*N2/(4.d0*g*g))*(u0+2.d0*Om*as)*(cos(2.d0*lat)-1.d0)) +ps0 = Peq * exp((u0/(4.0*bigG*Rd))*(u0+2.0*Om*as)*(cos(2.0*lat)-1.0))*(Ts0/Teq)^(cp/Rd) + +do k=0, nlev-1 + p0(k,:,:) = ps0*( (bigG/Ts0) * exp(-N2*z(k)/g) +1.d0 -(bigG/Ts0) )^(cp/Rd) + T_mean(k,:,:) = bigG*(1.d0 - exp(N2*z(k)/g))+ Ts0*exp(N2*z(k)/g) +end do + +theta_mean = T_mean*(p_ref/p0)^kappa + +;_____________________________________________________________________ +; get potential-temperature differences at several times + +ts = 0.50 ; timestep in seconds +interval = 100 ; steps per write + +t_sec =ts*interval*t_index ; get time from time-index + +theta = f->T(t_index,:,:,:)*(p_ref/p0)^kappa +theta_diff = theta - theta_mean + +equator = nlat/2 ; lat index of equator for odd # latitudes +theta_diff_slice = theta_diff(:,equator,:) + +;w = f->w(t_index,:,equator,:) ; read w field +omega = f->omega(t_index,:,equator,:) ; read w field + +;_____________________________________________________________________ +; plot contour field of lon vs height at the equator + +maxval= 2.0 +minval=-2.0 +gap = (maxval-minval)/10 + +print("maxval = "+maxval) +print("minval = "+minval) +print("spacing = "+gap) + + res1 = True + res1@gsnMaximize = True + + res1@gsnDraw = True + res1@gsnFrame = False + res1@cnFillOn = True + res1@cnLinesOn = False + + res1@gsnSpreadColors = True + res1@lbLabelAutoStride = True + + res1@gsnCenterString = "" + res1@tiMainString = "" + res1@vpWidthF = 0.50 + res1@vpHeightF = 0.25 + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnInfoLabelOn = False + res1@cnLineLabelsOn = False + res1@lbLabelBarOn = True + res1@lbLabelStride = 2 + + res1@lbOrientation = "horizontal" + res1@sfYArray = z ; use z for y axis + res1@sfXArray = lon1d*180.0/pi ; use lon1d for x axis + res1@trYReverse = False ; reverses y-axis, false + res1@tiYAxisString = "height (m)" ; y-axis title + ;res1@tiXAxisString = "Longitude" ; x-axis title + res1@cnMaxLevelValF = maxval + res1@cnMinLevelValF = minval + res1@cnLevelSpacingF = gap + res1@gsnPaperOrientation = "portrait" + + pres = True + pres@gsnMaximize = True ; make ps, eps, pdf as large as possible + pres@gsnPanelLabelBar = True + pres@pmLabelBarOrthogonalPosF = -0.03 + pres@amJust = "TopLeft" + pres@gsnFrame = False + pres@lbLabelStride = 1 + pres@gsnPaperOrientation = "portrait" + + wks_type = "png" + wks_type@wkWidth = 2500 + wks_type@wkHeight = 2500 + + ;wks_type = "pdf" + ;wks_type@wkPaperHeightF = 8 + ;wks_type@wkPaperWidthF = 11 + ;wks_type@wkOrientation = "portrait" + +if(isvar("t")) then + name =sprinti("test31_omega_%0.4i",t_index) +else + name ="./test31_omega" +end if + +print("name = "+name) +wks = gsn_open_wks(wks_type,name) + +gsn_define_colormap(wks,"gui_default") ; choose a colormap +;gsn_define_colormap(wks,"BlWhRe") ; choose a colormap + +plot = new(1,graphic) + + res1@gsnStringFontHeightF = 0.015 + res1@tmXBLabelFontHeightF = 0.01 + res1@tmYLLabelFontHeightF = 0.01 + res1@tiYAxisFontHeightF = 0.01 + res1@tiXAxisFontHeightF = 0.01 + ;res1@gsnCenterString = "omega " +t_sec+ " sec " + res1@tiMainString="omega " +t_sec+ " sec " + res1@tiMainFontHeightF = 0.01 + +plot(0) = gsn_csm_contour(wks,omega,res1) +; gsn_panel(wks,plot,(/1,1/),pres) + +frame(wks) + +end + diff --git a/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/plot_theta.ncl b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/plot_theta.ncl new file mode 100644 index 000000000000..1e7534eb34ad --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test3.1_nh_gravity_waves/preqx/plot_theta.ncl @@ -0,0 +1,193 @@ +;_____________________________________________________________________ +; +; Plot potential temperature differences from background state +; works for unevenly spaced z-levels +;_____________________________________________________________________ + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + +;_____________________________________________________________________ +; dcmip3-1 model parameters (must match those in dcmip123_mod.F90) + +g = 9.80616 ; grav const +a = 6371229.0 ; earth radius in meters +Rd = 287.0 ; dry gas const +cp = 1004.5 ; heat capacity const pressure +kappa = Rd/cp +pi = 3.141592654 +p_ref = 100000.0 ; reference pressure + +ztop = 10000.0 +X = 125.0 ; reduced Earth reduction factor +Om = 0.0 ; rotation rate of Earth +as = a/X ; new radius of small Earth +u0 = 20.0 ; max zonal wind speed +Teq = 300.0 ; temperature at equator +Peq = 100000.0 ; reference ps at equator +lambdac = 2.0*pi/3.0 ; lon of pert center +d = 5000.0 ; width for pert +phic = 0.0 ; lat of pert Center +delta_theta = 1.0 ; max amplitude of temp perturbation +Lz = 20000.0 ; vertical wavelength of pert +N = 0.010 ; Brunt-Vaisala frequency +N2 = N*N ; Brunt-Vaisala frequency squared +bigG = (g*g)/(N2*cp) ; constant + +;_____________________________________________________________________ +; open file, read in data (time,lev,lat,long) in range [0 to n-1] + +f = addfile("./movies/dcmip2012_test31.nc","r") ; open ncl data file +lat1d = f->lat*pi/180 ; load 1d latitutde array +lon1d = f->lon*pi/180 ; load 1d longitude array +lev = f->lev ; load 1d vertical-level array +time = f->time +hyam = f->hyam +hybm = f->hybm +nlon = getfilevardimsizes(f, "lon" ) ; get number of lon points +nlat = getfilevardimsizes(f, "lat" ) ; get number of lat points +nlev = getfilevardimsizes(f, "lev" ) ; get number of vertical levels +nt = dimsizes(time) +equator = nlat/2 ; lat index of equator for odd # latitudes + +lat = new( (/nlat,nlon/), typeof(lat1d)) +lon = new( (/nlat,nlon/), typeof(lon1d)) + +do j=0, nlon-1 + do i=0, nlat-1 + lat(i,j)=lat1d(i) + lon(i,j)=lon1d(j) + end do +end do + +print("nlon = "+nlon+" nlat = "+nlat+" nlev = "+nlev+" nt = "+nt) ; display data dimensions +print("equator = "+lat(equator,0)) + +t_index = nt-1 ; set time to max time +if(isvar("t")) then + t_index = t ; use t from command line if possible +end if +print ("t_index="+t_index) + +;_____________________________________________________________________ +; define data fields + +T = f->T(0,:,:,:) ; read Temperature field +Ts0 = new( (/nlat,nlon/) , typeof(T), -9999.0) ; define initial surf temp field +ps0 = new( (/nlat,nlon/) , typeof(T), -9999.0) ; define initial surf pressure field +p0 = new( (/nlev,nlat,nlon/) , typeof(T), -9999.0) ; define mean pressure at midpoints +T_mean= new( (/nlev,nlat,nlon/) , typeof(T), -9999.0) ; define mean temperature field +theta_mean= new( (/nlev,nlat,nlon/) , typeof(T), -9999.0) ; define mean potential temperature field + +;_____________________________________________________________________ +; recompute initial conditions + +Ts0 = bigG + (Teq-bigG)*exp( -(u0*N2/(4.d0*g*g))*(u0+2.d0*Om*as)*(cos(2.d0*lat)-1.d0)) +ps0 = Peq * exp((u0/(4.0*bigG*Rd))*(u0+2.0*Om*as)*(cos(2.0*lat)-1.0))*(Ts0/Teq)^(cp/Rd) + +; get height of eta levels from the pressure +h = new( (/nlev/), typeof(T)) + +do k=0, nlev-1 + p = hyam(k)*p_ref + hybm(k)*Peq + h(k) = (-g/N2)*log( (Teq/bigG)*( (p/Peq)^(Rd/cp) - 1.d0 ) + 1.d0 ) +end do + +do k=0, nlev-1 + print( "height (" +k+ ")= " +h(k)) +end do + +do k=0, nlev-1 + p0(k,:,:) = hyam(k)*p_ref + hybm(k)*ps0 + T_mean(k,:,:) = bigG*(1.d0 - exp(N2*h(k)/g))+ Ts0*exp(N2*h(k)/g) +end do + +;_____________________________________________________________________ +; get potential-temperature differences at several times + +days2sec = 3600.0*24.0 +t_sec = time(t_index)*days2sec + +theta = f->T(t_index,:,:,:)*(p_ref/p0)^kappa +theta_mean = T_mean*(p_ref/p0)^kappa + +theta_diff = theta - theta_mean + +theta_diff_slice = theta_diff(0:nlev-1,equator,:) + + ;_____________________________________________________________________ + ; plot contour field of lon vs height at the equator + + maxval= delta_theta/10 + minval=-delta_theta/10 + gap = (maxval-minval)/10 + + print("maxval = "+maxval) + print("minval = "+minval) + print("gap = "+gap) + + res1 = True + res1@gsnMaximize = True + res1@gsnDraw = False + res1@gsnFrame = False + res1@gsnSpreadColors = True + res1@gsnCenterString = "" + res1@tiMainString = "" + res1@vpWidthF = 2 + res1@vpHeightF = 1 + res1@lbLabelBarOn = True + res1@lbOrientation = "vertical" + res1@lbLabelStride = 1 + res1@cnFillOn = True + res1@cnLinesOn = False + res1@cnInfoLabelOn = False + res1@cnLineLabelsOn = False + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = maxval + res1@cnMinLevelValF = minval + res1@cnLevelSpacingF = gap + res1@sfYArray = h + res1@sfXArray = lon1d*180.0/pi ; use lon1d for x axis + res1@trYReverse = False ;True ; reverses y-axis, false + res1@tiYAxisString = "height (m)" ; y-axis title + res1@tiXAxisString = "Longitude" ; x-axis title + res1@gsnPaperOrientation = "portrait" + res1@gsnYAxisIrregular2Linear = True + ;res1@trYTensionF = 1.0 + + wks_type = "pdf" + wks_type@wkPaperHeightF = 6 + wks_type@wkPaperWidthF = 11 + wks_type@wkOrientation = "portrait" + + if(isvar("t")) then + filename =sprinti("./dcmip2012_test3_theta_diff_%0.4i",t_index) + else + filename ="./dcmip2012_test3_theta_diff" + end if + + print("filename = "+filename) + wks = gsn_open_wks(wks_type,filename) + gsn_define_colormap(wks,"gui_default") ; choose a colormap + + plot = new(1,graphic) + res1@gsnStringFontHeightF = 0.04 + res1@tmXBLabelFontHeightF = 0.03 + res1@tmYLLabelFontHeightF = 0.03 + res1@tiYAxisFontHeightF = 0.03 + res1@tiXAxisFontHeightF = 0.03 + + pltTitle = sprintf("CAM-SE Test 3-1 Theta' t=%5.2f s",t_sec) + res1@gsnCenterString = pltTitle + + plot(0) = gsn_csm_contour(wks,theta_diff_slice,res1) + +draw(wks) +frame(wks) + +end + diff --git a/components/homme/physics/column_model_mod.F90 b/components/homme/physics/column_model_mod.F90 deleted file mode 100644 index a63a88ca7e18..000000000000 --- a/components/homme/physics/column_model_mod.F90 +++ /dev/null @@ -1,148 +0,0 @@ -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -module column_model_mod -#ifdef _PRIM - use element_mod, only : element_t - use hybvcoord_mod, only : hvcoord_t - use hybrid_mod, only : hybrid_t - use kinds, only : real_kind, int_kind - use time_mod, only : TimeLevel_t - use physics_mod, only : Specific_Humidity, Saturation_Specific_Humidity, getsurfpress, Temp2PotTemp - use dimensions_mod, only : nlev, nlevp, np, qsize, nelemd - use control_mod, only : integration, columnpackage, test_case - use held_suarez_mod, only : hs_forcing - use forcing_mod, only : Apply_Forcing,EXP_EULER,INTERP_BDF2 - use reduction_mod, only : parallelmax,parallelmin - - use parallel_mod, only : abortmp, global_shared_buf, global_shared_sum - use global_norms_mod, only: wrap_repro_sum - use physical_constants, only : Cp, DD_PI, p0, Cp - use global_norms_mod, only : global_integral - use column_types_mod, only : HeldSuarezForcing_t, ColumnModel_t - - implicit none - - private - - public :: InitColumnModel - public :: ApplyColumnModel - -contains - - subroutine InitColumnModel(elem, cm,hvcoord,hybrid,tl,nets,nete,runtype) - - type(element_t), intent(inout) :: elem(:) - type (ColumnModel_t) :: cm - type (hvcoord_t), intent(in), target :: hvcoord - type (TimeLevel_t), intent(in), target :: tl - type (hybrid_t), intent(in),target :: hybrid - integer, intent(in), target :: nets,nete,runtype - integer :: i,j,k,ie - - if(runtype.ne.1) then - ! for all column type - do ie=nets,nete - do k=1,nlev - do j=1,np - do i=1,np - !elem(ie)%state%Q(i,j,k,1,1:3) = 0_real_kind - !elem(ie)%state%Qdp(i,j,k,1,1:3) = 0_real_kind - elem(ie)%state%Q(i,j,k,1) = 0_real_kind - elem(ie)%state%Qdp(i,j,k,1,1:2) = 0_real_kind - - elem(ie)%derived%FQ(i,j,k,1,1:3) = 0_real_kind - elem(ie)%derived%FM(i,j,1:2,k,1:3) = 0_real_kind - elem(ie)%derived%FT(i,j,k,1:3) = 0_real_kind - enddo - enddo - enddo - enddo - endif - cm%hvcoord = hvcoord - cm%hybrid => hybrid - cm%tl=> tl - cm%nets = nets - cm%nete = nete - - ! The MC needs the sounding - if(columnpackage == "none" .AND. test_case(1:12) == "held_suarez0")then - call InitHeldSuarezForcing(cm%cm_hs) - endif - - - end subroutine InitColumnModel - - subroutine ApplyColumnModel(elem, hybrid, hvcoord, cm,dt) - use hybvcoord_mod, only : hvcoord_t - - type (element_t), intent(inout) :: elem(:) - type (ColumnModel_t),intent(inout) :: cm - real (kind=real_kind),intent(in) :: dt - type (hvcoord_t) :: hvcoord - type (hybrid_t), intent(in) :: hybrid - - integer :: i,j,k,ie, nm1, forcing=0 - real (kind=real_kind) :: tau_damp - - - nm1 = cm%tl%nm1 - - ! forcings are accumulated then applied - - do ie=cm%nets,cm%nete - do k=1,nlev - do j=1,np - do i=1,np - elem(ie)%derived%FQ(i,j,k,:,nm1) = 0_real_kind - elem(ie)%derived%FM(i,j,1:2,k,nm1) = 0_real_kind - elem(ie)%derived%FT(i,j,k,nm1) = 0_real_kind - enddo - enddo - enddo - enddo - if(test_case(1:12) == "held_suarez0")then - forcing=1 - call ApplyHeldSuarezForcing(elem, cm%cm_hs,dt, cm%hvcoord,cm%tl,cm%nets,cm%nete) - endif - if (forcing==1) then - do ie=cm%nets,cm%nete - call Apply_Forcing(EXP_EULER,elem(ie), hvcoord, cm%tl,dt) - end do - endif - - end subroutine ApplyColumnModel - - - ! The rest of the file is private - - subroutine InitHeldSuarezForcing(cm) - - type (HeldSuarezForcing_t) :: cm - - if(.NOT.cm%INIT)then - cm%INIT = .TRUE. - endif - - end subroutine InitHeldSuarezForcing - - subroutine ApplyHeldSuarezForcing(elem, cm,dt, hvcoord, tl, nets, nete) - type (element_t),intent(inout) :: elem(:) - type (HeldSuarezForcing_t),intent(inout) :: cm - real (kind=real_kind),intent(in) :: dt - type (hvcoord_t) :: hvcoord - type (TimeLevel_t) :: tl - integer,target :: nets,nete - integer :: ie - - if(cm%INIT)then - do ie=nets,nete - call hs_forcing(elem(ie),hvcoord,tl,dt) - enddo - endif - - end subroutine ApplyHeldSuarezForcing -#endif - -end module column_model_mod diff --git a/components/homme/physics/column_types_mod.F90 b/components/homme/physics/column_types_mod.F90 deleted file mode 100644 index 0ba4911439ec..000000000000 --- a/components/homme/physics/column_types_mod.F90 +++ /dev/null @@ -1,27 +0,0 @@ -module column_types_mod - - use hybvcoord_mod, only : hvcoord_t - use hybrid_mod, only : hybrid_t - use time_mod, only : TimeLevel_t - use dimensions_mod, only : nlev, nlevp, np - use kinds, only : real_kind, int_kind - - implicit none - - public - - - type, public :: HeldSuarezForcing_t - logical :: INIT = .FALSE. - end type HeldSuarezForcing_t - - type, public :: ColumnModel_t - type (HeldSuarezForcing_t) :: cm_hs - type (hvcoord_t) :: hvcoord - type (hybrid_t), pointer :: hybrid - type (TimeLevel_t), pointer :: tl - integer :: nets,nete - end type ColumnModel_t - - -end module column_types_mod diff --git a/components/homme/src/forcing_mod.F90 b/components/homme/src/forcing_mod.F90 deleted file mode 100644 index 015a82dc71f7..000000000000 --- a/components/homme/src/forcing_mod.F90 +++ /dev/null @@ -1,158 +0,0 @@ -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -module forcing_mod - use kinds, only : real_kind - - implicit none - - private - - integer,public,parameter :: INTERP_BDF2=1,EXP_EULER=2,EXP_EULER_NP1=3 - - ! Used for the BDF scheme - real (kind=real_kind),parameter :: an = 4.0D0/3.0D0 - real (kind=real_kind),parameter :: anm1 =-1.0D0/3.0D0 - real (kind=real_kind),parameter :: bn = 4.0D0/3.0D0 - real (kind=real_kind),parameter :: bnm1 =-2.0D0/3.0D0 - - public :: Apply_Forcing, Zero_Forcing - -contains - - ! - ! Called after timestep, before Robert filter - ! - ! EXP_EULER: uses F^n-1 - ! EXTRAPOLATED BDF-2: uses F^n-1 and F^n - subroutine Apply_Forcing(METHOD,elemin,hvcoord,tld,dt,tlp) - use hybvcoord_mod, only : hvcoord_t - use element_mod, only : element_t - use dimensions_mod, only : np, nlev, qsize - use time_mod, only : TimeLevel_t - - !input - type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinate struct - type (element_t), intent(inout) :: elemin - integer,intent(in) :: METHOD - type (TimeLevel_t),intent(in) :: tld - type (TimeLevel_t),intent(in),optional :: tlp - real (kind=real_kind) :: dt - - !local - real (kind=real_kind) :: dt2,dp, dt2_q - integer :: i,j,k,ie, m,q - integer :: np1_d,n0_d,nm1_d - integer :: np1_p,n0_p,nm1_p - - np1_d = tld%np1 - n0_d = tld%n0 - nm1_d = tld%nm1 - if(present(tlp)) then - np1_p = tlp%np1 - n0_p = tlp%n0 - nm1_p = tlp%nm1 - else - np1_p = tld%np1 - n0_p = tld%n0 - nm1_p = tld%nm1 - end if - dt2 = 2_real_kind*dt - dt2_q = dt2 - - select case(METHOD) - case(INTERP_BDF2) - - do k=1,nlev - do j=1,np - do i=1,np - elemin%state%v(i,j,1,k,np1_d) = & - an*elemin%state%v(i,j,1,k,n0_d) + anm1*elemin%state%v(i,j,1,k,nm1_d) + & - bn*dt*elemin%derived%FM(i,j,1,k,n0_p) + bnm1*dt*elemin%derived%FM(i,j,1,k,nm1_p) - elemin%state%v(i,j,2,k,np1_d) = & - an*elemin%state%v(i,j,2,k,n0_d) + anm1*elemin%state%v(i,j,2,k,nm1_d) + & - bn*dt*elemin%derived%FM(i,j,2,k,n0_p) + bnm1*dt*elemin%derived%FM(i,j,2,k,nm1_p) - elemin%state%T(i,j,k,np1_d) = & - an*elemin%state%T(i,j,k,n0_d) + anm1*elemin%state%T(i,j,k,nm1_d) + & - bn*dt*elemin%derived%FT(i,j,k,n0_p) + bnm1*dt*elemin%derived%FT(i,j,k,nm1_p) - enddo - enddo - enddo - do m=1,qsize - do k=1,nlev - do j=1,np - do i=1,np - elemin%state%Q(i,j,k,m) = elemin%state%Q(i,j,k,m) + dt2_q*elemin%derived%FQ(i,j,k,m,nm1_p) - end do - end do - end do - end do - - case(EXP_EULER) - do k=1,nlev - do j=1,np - do i=1,np - elemin%state%v(i,j,1,k,np1_d) = elemin%state%v(i,j,1,k,np1_d) + dt2*elemin%derived%FM(i,j,1,k,nm1_p) - elemin%state%v(i,j,2,k,np1_d) = elemin%state%v(i,j,2,k,np1_d) + dt2*elemin%derived%FM(i,j,2,k,nm1_p) - elemin%state%T(i,j,k,np1_d) = elemin%state%T(i,j,k,np1_d) + dt2*elemin%derived%FT(i,j,k,nm1_p) - enddo - enddo - enddo - do m=1,qsize - elemin%state%Q(:,:,:,m) = elemin%state%Q(:,:,:,m)& - + dt2_q*elemin%derived%FQ(:,:,:,m,nm1_p) - end do - - case default - ! Selects no forcing - end select - - ! forcing always applied to Q, update Qdp: - do q=1,qsize - do k=1,nlev - do j=1,np - do i=1,np - dp = ( hvcoord%hyai(k+1) - hvcoord%hyai(k) )*hvcoord%ps0 + & - ( hvcoord%hybi(k+1) - hvcoord%hybi(k) )*elemin%state%ps_v(i,j,np1_d) - elemin%state%Qdp(i,j,k,q,np1_d) = elemin%state%Q(i,j,k,q)*dp - enddo - enddo - enddo - enddo - end subroutine Apply_Forcing - - - subroutine Zero_Forcing(elemin,tl) - use element_mod, only : element_t - use time_mod, only : timelevel_t - use dimensions_mod, only : np, nlev, qsize - type(element_t), intent(inout) :: elemin - type(timelevel_t), intent(in) :: tl - - integer :: i, j, k, nm1, m - - nm1=tl%nm1 - do k=1,nlev - do j=1,np - do i=1,np - elemin%derived%FM(i,j,1,k,nm1) = 0.0_real_kind - elemin%derived%FM(i,j,2,k,nm1) = 0.0_real_kind - elemin%derived%FT(i,j,k,nm1) = 0.0_real_kind - enddo - enddo - enddo - do m=1,qsize - do k=1,nlev - do j=1,np - do i=1,np - elemin%derived%FQ(i,j,k,m,nm1) = 0.0_real_kind - end do - end do - end do - end do - - end subroutine Zero_Forcing - - -end module forcing_mod diff --git a/components/homme/src/interp_movie_mod.F90 b/components/homme/src/interp_movie_mod.F90 index 1b6732144173..2168e032a210 100644 --- a/components/homme/src/interp_movie_mod.F90 +++ b/components/homme/src/interp_movie_mod.F90 @@ -886,7 +886,7 @@ subroutine interp_movie_output(elem, tl, par, phimean, hvcoord) do ie=1,nelemd en=st+interpdata(ie)%n_interp-1 call interpolate_vector(interpdata(ie), elem(ie), & - elem(ie)%derived%FM(:,:,:,:,tl%np1), nlev, var3d(st:en,:,:,1), 0) + elem(ie)%derived%FM(:,:,:,:), nlev, var3d(st:en,:,:,1), 0) st=st+interpdata(ie)%n_interp enddo diff --git a/components/homme/src/netcdf_interp_mod.F90 b/components/homme/src/netcdf_interp_mod.F90 new file mode 100644 index 000000000000..bbdc3419bb80 --- /dev/null +++ b/components/homme/src/netcdf_interp_mod.F90 @@ -0,0 +1,670 @@ +! +! Interpolates data in both horizontal and vertical directions +! and writes it to a netcdf file using PIO (which uses pNetCDF) +! +! For questions, please contact: david.hall@colorado.edu +!_______________________________________________________________________ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +module netcdf_interp_mod + + !$ use omp_lib ! load openmp library code + use control_mod, only: test_case!, vinterp_nlev + use dimensions_mod, only: nelemd, np, ne, nlev + use element_mod, only: element_t + use hybrid_mod, only: hybrid_t,hybrid_create + use hybvcoord_mod, only: hvcoord_t + use kinds, only: real_kind + use element_ops, only: get_field, get_vector_field + use parallel_mod, only: parallel_t, haltmp, syncmp + use shr_const_mod, only: shr_const_cday + use time_mod, only: timelevel_t,time_at + use vertical_se, only: v_interpolate, L_interp + + use common_io_mod, only: & + max_output_streams, output_frequency, & + output_start_time, output_end_time, varname_len, piofs, num_io_procs + + use interpolate_mod, only: & + interpolate_t,interpdata_t, setup_latlon_interp, interpolate_scalar, & + get_interp_parameter, get_interp_lat, get_interp_lon, interpolate_vector + + use pio_io_mod, only: & + nf_output_init_begin, nf_handle, & + nf_output_register_variables, nf_output_register_dims, & + nf_variable_attributes, nf_output_init_complete, pio_double, & + get_current_varnames, nf_selectedvar, nf_get_frame, nf_put_var => nf_put_var_pio, & + nfsizekind, get_varindex, pio_put_var, nf_close_all, pio_setDebugLevel, & + pio_put_att, pio_put_var, pio_global, nf_init_decomp, nf_advance_frame,& + pio_syncfile + + implicit none + + integer, parameter :: n_dims = 4 ! number of dimensions + character*(*),parameter :: dim_names(n_dims)=(/'lon ','lat ','lev ','time'/) + + !_____________________________________________________________________ + ! scalar field descriptor + + type nc_sfield_t + integer :: var_type ! pio type of variable + integer :: dim_ids(n_dims) ! dimension ids + integer :: tensor_dim ! tensor dim:0=scalar,1=vector,2=tensor + logical :: is_required ! variable required flag + character*(5) :: short_name ! short name of variable + character*(256) :: units ! physical units + character*(256) :: long_name ! full name of variable + endtype + + !_____________________________________________________________________ + ! vector field descriptor + + type nc_vfield_t + character*(4) :: vec_name ! vector name + integer :: n_comp ! number of components + character*(4) :: short_name(2) ! name of each component + endtype + + !_____________________________________________________________________ + ! possible output variables + + integer, parameter :: n_vars= 38 ! number of possible output fields + + type(nc_sfield_t), parameter :: nc_vars(n_vars) =& + (/& + nc_sfield_t(pio_double, (/1,0,0,0/),0, .true. , 'lon ', 'degrees_east', 'column longitude' ),& + nc_sfield_t(pio_double, (/2,0,0,0/),0, .true. , 'lat ', 'degrees_north', 'column latitude' ),& + nc_sfield_t(pio_double, (/3,0,0,0/),0, .true. , 'lev ', '', 'vertical coordinate' ),& + nc_sfield_t(pio_double, (/4,0,0,0/),0, .true. , 'time ', 'days', 'model elapsed time' ),& + nc_sfield_t(pio_double, (/3,0,0,0/),0, .true. , 'hyam ', '', 'hybrid A at midpoints' ),& + nc_sfield_t(pio_double, (/3,0,0,0/),0, .true. , 'hybm ', '', 'hybrid B at midpoints' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'ps ', 'Pa', 'hydrostatic surf pressure' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'z ', 'meters', 'altitiude' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'geo ', 'meters^2/sec^2', 'geopotential height' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'phi ', 'meters^2/sec^2', 'geopotential height' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'phis ', 'meters^2/sec^2', 'surface geopotential' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'p ', 'Pa', 'hydrostatic atmo pressure' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'pt ', 'Pa', 'total atmospheric pressure' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'T ', 'degrees Kelvin', 'temperature' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'theta', 'degrees Kelvin', 'potential temperature' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),1, .false., 'u ', 'meters/second', 'longitudinal wind component'),& + nc_sfield_t(pio_double, (/1,2,3,4/),1, .false., 'v ', 'meters/second', 'latitudinal wind component' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'w ', 'meters/second', 'vertical wind component' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'pp ', 'Pa', 'pressure deviation' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'Q ', 'kg/kg', 'tracer 1 mixing ratio' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'Q2 ', 'kg/kg', 'tracer 2 mixing ratio' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'Q3 ', 'kg/kg', 'tracer 3 mixing ratio' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'Q4 ', 'kg/kg', 'tracer 4 mixing ratio' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'omega', '', '' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'D3 ', '', '3d divergence of velocity' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'speed', 'meters/second', 'wind speed' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'KE ', 'Joules', 'kinetic energy' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'alpha', '', 'specific volume' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'rho ', '', 'density' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'dpdn ', '', 'pseudo-density' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'm ', '', 'pseudo-density' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'cflux', '', 'column int horiz mass flux' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'ndot ', '', 'vertical eta velocity' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'nflux', '', 'vertical eta flux' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'O1 ', '', 'debugging output 1' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'O2 ', '', 'debugging output 2' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'O3 ', '', 'debugging output 3' ),& + nc_sfield_t(pio_double, (/1,2,3,4/),0, .false., 'O4 ', '', 'debugging output 4' ) & + !nc_sfield_t(type, dim_ids, n_spatial_dims, required?, short_name, units, long_name) + /) + + type(nc_vfield_t),parameter::nc_vfields(1)=& ! list vector fields + (/& + !nc_vfield_t(vec_name, n_comps, (/component_names/)) + nc_vfield_t ('v' , 2 , (/'u ','v '/)) & + /) + + integer :: ni ! number of vertical levels + integer :: dim_sizes(n_dims) ! size of each dimension + + type(interpdata_t), allocatable :: interpdata(:) ! interpolated data-locations + type(nf_handle), save :: files(max_output_streams) ! netcdf file-handles + + integer :: & + n_interp, & ! num horiztonal interp points on this proc + n_lat, & ! num points in lat direction + n_lon ! num points in lon direction + + integer(kind=nfsizekind):: & + start2d(3), & ! netcdf offset for 2d data + start3d(4), & ! netcdf offset for 3d data + count2d(3), & ! netcdf data size for 2d data + count3d(4) ! netcdf data size for 3d data + + character(len=varname_len), pointer :: output_varnames(:) ! list of variables to write + + + ! convert nc_vars to 1d-arrays + + integer, parameter :: n_vfields = size(nc_vfields) ! count num vector fields + character*(*), parameter :: var_names (n_vars)= nc_vars(:)%short_name ! get array of variable names + integer, parameter :: var_type (n_vars)= nc_vars(:)%var_type ! get array of variable types + logical, parameter :: var_req (n_vars)= nc_vars(:)%is_required ! get array of requirement flags + integer :: var_dim_ids(n_dims,n_vars) ! get array of dimension ids + + contains + + !_____________________________________________________________________________ + function index_of(var_name) result(index) + character*(*), intent(in) :: var_name + + integer :: i, index + + index=-1 + do i=1,n_vars + if(var_name == var_names(i)) index=i + enddo + end function + + !_____________________________________________________________________________ + integer function size_of_dim(index) result(r) + integer, intent(in) :: index + r = 1 + if(index>0) r = dim_sizes(index) + if (r==0) r=1 + end function + + !_____________________________________________________________________________ + subroutine netcdf_interp_init(elem, hybrid, hvcoord) + + ! Define netcdf variables, dimensions, and attributes + + type(element_t), intent(in):: elem(:) + type(hybrid_t), target, intent(in):: hybrid + type(hvcoord_t), intent(in):: hvcoord + + type(parallel_t), pointer:: par + + integer, parameter :: run_type = 0 ! set run_type = 'new run' + + integer i ! loop variable + integer sz(3) ! buffers for dimension sizes + integer iorank + integer :: ios ! io stream number + integer result + + call pio_setDebugLevel(0) ! disable pio debugging + par=> hybrid%par + + if (par%masterproc) then + print *,'initializing netcdf output =',var_names + ! print *,"vinterp_type = ",vinterp_type + ! print *,"vinterp_nlev = ",vinterp_nlev + endif + + ni =nlev + !if(vinterp_type==1) ni = vinterp_nlev + + ! convert nc_vars dimensions into 2d-array + + do i=1,n_vars + var_dim_ids(:,i)= nc_vars(i)%dim_ids(:) + enddo + + ! allocate memory for interpolated points + + allocate(interpdata(nelemd)) ! allocate one interpdata struct per element + + ! initialize lat-lon interpolation + + call setup_latlon_interp(elem, interpdata, par) + + n_lat = get_interp_parameter('nlat') ! get num lat points in interpolation + n_lon = get_interp_parameter('nlon') ! get num lon points in interpolation + n_interp = sum(interpdata(1:nelemd)%n_interp) ! count horizontal interp points on this proc + + print *,par%rank,': n_lat=',n_lat,' n_lon=',n_lon,' n_interp=',n_interp + + ! set dimension sizes + + dim_sizes=(/n_lon, n_lat, ni, 0/) ! set size of each dimension + + ! initialize netcdf file: add dimensions and variables + + if (par%masterproc) print *,'known output variables =',var_names + + call nf_output_init_begin(files, par%masterproc, par%nprocs, par%rank, par%comm, test_case, run_type) + call nf_output_register_dims(files, n_dims, dim_names, dim_sizes) + call nf_output_register_variables(files, n_vars, var_names, var_dim_ids, var_type, var_req) + + ! set variable attributes: long name and dimensional units + + do i=1,n_vars + call nf_variable_attributes(files, nc_vars(i)%short_name, nc_vars(i)%long_name, nc_vars(i)%units) + enddo + + ! save horizontal NP and NE as global attributes + + do ios = 1, max_output_streams + if((output_frequency(ios) .gt. 0) ) then + result = pio_put_att(files(ios)%fileid, pio_global, 'np', np) + result = pio_put_att(files(ios)%fileid, pio_global, 'ne', ne) + endif + enddo + + ! exit netcdf definition mode + + call nf_output_init_complete(files) + + ! write lat and lon coordinates to file + + call write_coordinates_to_file(hybrid, hvcoord) + + ! assign sets of lat-lon points to the different output streams + + call decompose_output() + + end subroutine + + !_____________________________________________________________________ + subroutine netcdf_interp_finish() + + ! Flush data and close netcdf output streams + + call nf_close_all(files) + end subroutine + + !_____________________________________________________________________ + subroutine write_coordinates_to_file(hybrid,hvcoord) + + ! Write latitude and longitude interpolation coordinates to file + + type(hybrid_t), intent(in):: hybrid + type(hvcoord_t),intent(in):: hvcoord + + integer :: ios ! io-stream index + integer :: iorank ! rank of iostream + integer :: varid ! variable id + integer :: vindex ! variable index + integer :: ierr ! error number + + real(real_kind), allocatable :: lat(:),lon(:),lev(:),hyam(:),hybm(:) ! array of lat and lon values + + iorank=piofs%io_rank ! get iostream rank + + allocate(lon(n_lon)); lon = get_interp_lon() ! allocate and fill longitude array + allocate(lat(n_lat)); lat = get_interp_lat() ! allocate and fill latitiude array + allocate(lev(ni)); lev = L_interp ! allocate and fill level array + allocate(hyam(ni)); hyam = v_interpolate(hvcoord%hyam,ni) ! allocate and fill hybrid A array + allocate(hybm(ni)); hybm = v_interpolate(hvcoord%hybm,ni) ! allocate and fill hybrid B array + + do ios=1,max_output_streams + if((output_frequency(ios) .gt. 0) ) then + if(iorank==0) print *,"writing coordinates to ios=",ios + + ! write lon values to file + + vindex = get_varindex('lon', files(ios)%varlist) ! look up index of longitude var + varid = files(ios)%varlist(vindex)%vardesc%varid ! get longitude variable id + ierr = pio_put_var(files(ios)%fileID, varid, lon) ! write longitude array to file + + ! write lat values to file + + vindex = get_varindex('lat',files(ios)%varlist) ! look up index of latitide var + varid = files(ios)%varlist(vindex)%vardesc%varid ! get latitide variable id + ierr = pio_put_var(files(ios)%fileID, varid, lat) ! write latitude array to file + + ! write lev values to file + + vindex = get_varindex('lev',files(ios)%varlist) ! look up index of lev var + varid = files(ios)%varlist(vindex)%vardesc%varid ! get lev variable id + ierr = pio_put_var(files(ios)%fileID, varid, lev) ! write level array to file + + ! write hyam values to file + + vindex = get_varindex('hyam',files(ios)%varlist) ! look up index of lev var + varid = files(ios)%varlist(vindex)%vardesc%varid ! get lev variable id + ierr = pio_put_var(files(ios)%fileID, varid, hyam) ! write level array to file + + ! write hyam values to file + + vindex = get_varindex('hybm',files(ios)%varlist) ! look up index of lev var + varid = files(ios)%varlist(vindex)%vardesc%varid ! get lev variable id + ierr = pio_put_var(files(ios)%fileID, varid, hybm) ! write level array to file + + call pio_syncfile(files(ios)%fileid) + + endif + end do + + call syncmp(hybrid%par) ! synchronize openmp threads + + end subroutine + + !_____________________________________________________________________________ + subroutine decompose_output() + + ! convert lon,lat,lev positions to 1d indices and distrubute amongsts procs + + integer, pointer ::& + ldof_2d(:) , & + ldof_3d(:) , & + io_dof_2d(:), & + io_dof_3d(:) + + integer ie,i,i1d,k, io_dof , iorank ! loop indices + + !map 2d(lon,lat) indices to 1d offsets + + allocate(ldof_2d(n_interp)) ! allocate space for 1d index array + i1d=0 ! initialize 1d index + do ie=1,nelemd ! loop over each element + do i=1,interpdata(ie)%n_interp ! loop over each interp-point in element + i1d = i1d + 1 ! increment 1d index + ldof_2d(i1d)= interpdata(ie)%ilon(i)+ (interpdata(ie)%ilat(i)-1)*n_lon! covert 2d lat-lon index to 1d index + end do + end do + + ! map 3d(lon,lat,level) indices to 1d-offsets + + allocate(ldof_3d(n_interp * ni)) ! allocate space for 1d index array + i1d=0 ! initialize 1d index + do k=1,ni ! loop over each vertical level + do ie=1,nelemd ! loop over each element + do i=1,interpdata(ie)%n_interp ! loop over each interp point in element + i1d=i1d+1 + ldof_3d(i1d)=interpdata(ie)%ilon(i)+(interpdata(ie)%ilat(i)-1)*n_lon+(k-1)*n_lat*n_lon + end do + end do + end do + + ! divide output data into slices + + iorank = piofs%io_rank + call getiodof(2,(/n_lon,n_lat/), iorank, io_dof_2d, start2d(1:2), count2d(1:2)) + call getiodof(3,(/n_lon,n_lat,ni/),iorank, io_dof_3d, start3d(1:3), count3d(1:3)) + + ! initialize output decomposition + + call nf_init_decomp(files, (/1,2/), ldof_2d, io_dof_2d, start2d(1:2), count2d(1:2)) + call nf_init_decomp(files, (/1,2,3/), ldof_3d, io_dof_3d, start3d(1:3), count3d(1:3)) + + end subroutine + + !_____________________________________________________________________________ + subroutine write_scalar_field_1d(elem, var, ios, n0, hybrid, hvcoord) + + ! Get 3d scalar field data by name, and write to file + + type(element_t), intent(in) :: elem(:) + type(nc_sfield_t),intent(in) :: var ! variable descriptor + integer, intent(in) :: ios ! io stream number + integer, intent(in) :: n0 ! time-level index + type(hybrid_t), intent(in) :: hybrid + type(hvcoord_t), intent(in) :: hvcoord + + real(real_kind) :: element_data(np,np,ni) ! native grid data + real(real_kind), allocatable :: result(:,:) ! interpolated data + integer :: st, en ! start and end index for interpolation + integer :: ie, var_index, vertical_dim ! loop indicies + integer :: n_vert ! number of vertical levels + + output_varnames=>get_current_varnames(ios) ! get list of output variables + + ! check to see if field is selected and if it is a scalar field + + if(.not. nf_selectedvar(var%short_name, output_varnames)) return ! skip if not selected + if(var%tensor_dim .ne. 0) return ! skip if part of a vector or tensor + if (hybrid%par%masterproc) print *,'writing interpolated scalar field "',var%short_name,'"' + + var_index = index_of(var%short_name) + vertical_dim = var_dim_ids(3,var_index) + n_vert = size_of_dim(vertical_dim) + !if (hybrid%par%masterproc) print *,"index_of(",var%short_name,")=",var_index," vertical_dim=",vertical_dim," n_vert=",n_vert + + ! interpolate scalar field data to result field + + allocate(result(n_interp,n_vert)) ! allocate buffer for interpolated data + result = 0.0d0 + + st = 1 ! set start index + do ie=1,nelemd ! loop over elem + en = st + interpdata(ie)%n_interp-1 ! set end index + call get_field(elem(ie),var%short_name,element_data,hvcoord,n0,n0) ! get scalar field by name + call interpolate_scalar(interpdata(ie),element_data, np, n_vert, result(st:en,:)) + st=st+interpdata(ie)%n_interp ! shift start index + enddo + + if(n_vert>1) then + ! write 3d interpolated scalar field to file + + start3d(4) = nf_get_frame(files(ios)) + count3d(4) = 1 + count3d(1:3) = dim_sizes( (/1,2,3/) ) ! set size of output + call nf_put_var(files(ios), result, start3d, count3d, name=var%short_name) + endif + + if(n_vert==1) then + ! write 2d interpolated scalar field to file + + start2d(3) = nf_get_frame(files(ios)) + count2d(3) = 1 + count2d(1:2) = dim_sizes( (/1,2/) ) ! set size of output + call nf_put_var(files(ios), result, start2d, count2d, name=var%short_name) + endif + + deallocate(result) + + end subroutine + + !_____________________________________________________________________________ + ! write_vector_field: write vector field components to file, if selected + + subroutine write_vector_field(elem, var, ios, n0, hybrid, hvcoord) + + type (element_t), intent(in) :: elem(:) + type(nc_vfield_t),intent(in) :: var ! variable descriptor + integer, intent(in) :: ios ! io stream number + integer, intent(in) :: n0 ! time-level index + type(hybrid_t), intent(in) :: hybrid + type(hvcoord_t), intent(in) :: hvcoord + + real(real_kind) :: vdata(np,np,2,ni) ! nodal vector data + + real(real_kind), allocatable :: result(:,:,:) ! interpolated lat lon data + + character*(10) :: short_name ! buffer for current var short name + + integer :: st, en ! start and end index for interpolation + integer :: ie,i ! loop indicies + logical :: selected ! indicates if vector is selected + logical :: comp_selected + + output_varnames=>get_current_varnames(ios) ! get list of output variables + + ! check to see if any vector component is selected + + selected = .false. + do i=1,var%n_comp + selected = selected .or. nf_selectedvar(var%short_name(i), output_varnames) + end do + if(.not. selected) return ! skip if no components selected + + ! Interpolate vector data + + allocate(result(n_interp,ni,2)) ! allocate buffer for 3d interpolated data + st = 1 ! set start index + do ie=1,nelemd ! loop over elem + en = st + interpdata(ie)%n_interp-1 ! set end index + vdata = get_vector_field(elem(ie), var%vec_name, n0, ni) ! get vector field data by name + + call interpolate_vector(interpdata(ie), elem(ie), vdata, ni, result(st:en,:,:), 0) + st=st+interpdata(ie)%n_interp ! shift start index + enddo + + ! Write selected components to file + start3d(4)=nf_get_frame(files(ios)) + count3d(4)=1 + count3d(1:3) = dim_sizes( (/1,2,3/) ) ! set size of output + + do i=1,var%n_comp ! loop over each vec component + if( .not. nf_selectedvar(var%short_name(i), output_varnames) ) cycle ! skip output if not selected + + if (hybrid%par%masterproc) print *,'writing interpolated vector component ',var%short_name(i) + call nf_put_var(files(ios), result(:,:,i), start3d, count3d, name=var%short_name(i)) + enddo + + deallocate(result) + + end subroutine + + !_____________________________________________________________________________ + subroutine netcdf_interp_write(elem, tl, hybrid, hvcoord) + + ! Check each variable against output list and write it if present + + type (element_t), intent(in) :: elem(:) + type (TimeLevel_t), intent(in) :: tl + type(hybrid_t), intent(in) :: hybrid + type(hvcoord_t), intent(in) :: hvcoord + + integer :: n0 ! time-level to write + integer :: ios, i, nf_frame ! loop indicies + + n0 = tl%n0 + + do ios=1,max_output_streams ! loop over all output streams + if(output_frequency(ios)==0) cycle + + if(( output_start_time(ios) <= tl%nstep) .and. & + ( output_end_time(ios) >= tl%nstep) .and. & + ( modulo(tl%nstep, output_frequency(ios)) .eq. 0)) then + + nf_frame = nf_get_frame(files(ios)) ! set netcdf record index + start2d(3) = nf_frame; count2d(3)=1 + start3d(4) = nf_frame; count3d(4)=1 + + if (hybrid%par%masterproc) print *,'current frame = ',start3d(4) + + ! write interpolated scalar fields + + do i=1,n_vars ! loop over all possible fields + call write_scalar_field_1d(elem, nc_vars(i), ios, n0, hybrid, hvcoord) ! write if scalar field and is-selected + enddo + + ! write interpolated vector fields + + do i=1,n_vfields ! loop over all vector fields + call write_vector_field(elem,nc_vfields(i), ios, n0, hybrid, hvcoord) ! write if at least one component selected + enddo + + ! write simulation time + + call nf_put_var(files(ios),real(time_at(tl%nstep)/shr_const_cday,kind=real_kind),start2d(3:3),count2d(3:3),name='time') + + ! advance to next data-frame + + call nf_advance_frame(files(ios)) ! advance output-frame number + + ! flush data to file + call pio_syncfile(files(ios)%fileid) + + endif + enddo + + end subroutine + + !_____________________________________________________________________________ + ! getiodof: copied unmodified from interp_movie_mod + + subroutine GetIODOF(ndims, gdims, iorank, iodof, start, count) + integer, intent(in) :: gdims(ndims) + integer, intent(in) :: iorank + integer(kind=nfsizekind), intent(out) :: start(ndims), count(ndims) + integer, pointer :: iodof(:) ! gcc4.2 didn't like intent(out) + + integer :: nzrank, nxrank, nx, k, i, j, ndims, icnt + + if(iorank>=0) then + nx=num_io_procs + count(ndims)=max(1,gdims(ndims)/num_io_procs) + nx=max(1,num_io_procs/gdims(ndims)) + nzrank=iorank/nx + + k=num_io_procs-gdims(ndims)*nx + + if(iorank>num_io_procs-k-1.and.k>0) then + nzrank=gdims(ndims)-1 + end if + + start(ndims)=nzrank*count(ndims)+1 + if(gdims(ndims)>num_io_procs) then + k=gdims(ndims)-num_io_procs*count(ndims) + if(k>iorank) then + count(ndims)=count(ndims)+1 + end if + if(k>=iorank) then + start(ndims)=start(ndims)+iorank + else + start(ndims)=start(ndims)+k + end if + + end if + + if(k>0 .and.nzrank==gdims(ndims)-1 ) then + nx=nx+k + end if + nxrank=iorank + + do i=ndims-1,1,-1 + ! print *, nxrank, nx + + nxrank=mod(nxrank,nx) + count(i)=gdims(i)/nx + k=gdims(i)-count(i)*nx + if(nxranktimelevels +use hybrid_mod, only: hybrid_t +use hybvcoord_mod, only: hvcoord_t, set_layer_locations +use kinds, only: rl=>real_kind, iulog +use parallel_mod, only: abortmp +use vertical_se, only: init_vertical_interp, ddn_hyai,ddn_hybi +implicit none + +! physical constants used by dcmip2012 test 3.1 +real(rl), parameter :: & + g = 9.80616, & ! grav const + a = 6371229.0, & ! earth radius in meters + Rd = 287.0, & ! dry gas const + cp = 1004.5, & ! heat capacity const pressure + kappa = Rd/cp, & + pi = 3.141592654, & + p0 = 100000.0 ! reference pressure + +real(rl), dimension(:,:,:,:), allocatable :: u0, v0 ! storage for dcmip2-x sponge layer +real(rl):: zi(nlevp), zm(nlev) ! z coordinates + +contains + +!_____________________________________________________________________ +subroutine dcmip2012_test1_1(elem,hybrid,hvcoord,nets,nete,time,n0,n1) + + ! 3d deformational flow + + type(element_t), intent(inout), target :: elem(:) ! element array + type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure + type(hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinates + integer, intent(in) :: nets,nete ! start, end element index + real(rl), intent(in) :: time ! current time + integer, intent(in) :: n0,n1 ! time level indices + + logical :: initialized = .false. + + integer, parameter :: zcoords = 0 ! we are not using z coords + logical, parameter :: use_eta = .true. ! we are using hybrid eta coords + real(rl), parameter :: & + T0 = 300.d0, & ! temperature (K) + ztop = 12000.d0, & ! model top (m) + H = Rd * T0 / g ! scale height + + integer :: i,j,k,ie ! loop indices + real(rl):: lon,lat ! pointwise coordiantes + real(rl):: p,z,phis,u,v,w,T,phis_ps,ps,rho,q(4),dp,eta_dot,dp_dn ! pointwise field values + real(rl):: etai(nlevp),etam(nlev) ! vertical interpolation levels + + ! set vertical interpolation coordinates + if(.not. initialized) then + initialized = .true. + if (hybrid%masterthread) write(iulog,*) 'initializing dcmip2012 test 1-1: 3d deformational flow' + call get_evenly_spaced_z(zi,zm, 0.0_rl,ztop) ! get evenly spaced z levels + etai = exp(-zi/H) ! set interpolated eta levels from z + etam = 0.5_rl*(etai(1:nlev)+etai(2:nlevp)) + call init_vertical_interp(etam,nlev) + endif + + ! set prescribed state at level midpoints + do ie = nets,nete; do k=1,nlev; do j=1,np; do i=1,np + lon = elem(ie)%spherep(i,j)%lon; lat = elem(ie)%spherep(i,j)%lat + z = H * log(1.0d0/hvcoord%etam(k)) + p = p0 * hvcoord%etam(k) + call test1_advection_deformation(time,lon,lat,p,z,zcoords,u,v,w,T,phis,ps,rho,q(1),q(2),q(3),q(4)) + + dp = pressure_thickness(ps,k,hvcoord) + call set_state(u,v,T,ps,phis,dp,zm(k), i,j,k,elem(ie),n0,n1) + if(time==0) then + call set_tracers(q,qsize,dp,i,j,k,lat,lon,elem(ie)) + endif + + enddo; enddo; enddo; enddo + + ! set prescribed state at level interfaces + do ie = nets,nete; do k=1,nlevp; do j=1,np; do i=1,np + lon = elem(ie)%spherep(i,j)%lon; lat = elem(ie)%spherep(i,j)%lat + z = H * log(1.0d0/hvcoord%etai(k)) + p = p0 * hvcoord%etai(k) + call test1_advection_deformation(time,lon,lat,p,z,zcoords,u,v,w,T,phis,ps,rho,q(1),q(2),q(3),q(4)) + + ! get vertical derivative of p at point i,j,k + dp_dn = ddn_hyai(k)*p0 + ddn_hybi(k)*ps + + ! get vertical eta velocity at point i,j,k + eta_dot = -g*rho*w/p0 + + ! store vertical mass flux + elem(ie)%derived%eta_dot_dpdn(i,j,k) = eta_dot * dp_dn + + enddo; enddo; enddo; enddo + +end subroutine + +!_____________________________________________________________________ +subroutine dcmip2012_test1_2(elem,hybrid,hvcoord,nets,nete,time,n0,n1) + + ! Hadley-like Meridional Circulation + + type(element_t), intent(inout), target :: elem(:) ! element array + type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure + type(hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinates + integer, intent(in) :: nets,nete ! start, end element index + real(rl), intent(in) :: time ! current time + integer, intent(in) :: n0,n1 ! time level indices + + logical :: initialized = .false. + + integer, parameter :: zcoords = 0 ! we are not using z coords + logical, parameter :: use_eta = .true. ! we are using hybrid eta coords + real(rl), parameter :: & + T0 = 300.d0, & ! temperature (K) + ztop = 12000.d0, & ! model top (m) + H = Rd * T0 / g ! scale height + + integer :: i,j,k,ie ! loop indices + real(rl):: lon,lat ! pointwise coordiantes + real(rl):: p,z,phis,u,v,w,T,phis_ps,ps,rho,q(2),dp,eta_dot,dp_dn ! pointwise field values + real(rl):: etai(nlevp),etam(nlev) ! vertical interpolation levels + + ! set vertical interpolation coordinates + if(.not. initialized) then + initialized = .true. + if (hybrid%masterthread) write(iulog,*) 'initializing dcmip2012 test 1-2: Hadley-like Meridional Circulation' + call get_evenly_spaced_z(zi,zm, 0.0_rl,ztop) ! get evenly spaced z levels + etai = exp(-zi/H) ! set interpolated eta levels from z + etam = 0.5_rl*(etai(1:nlev)+etai(2:nlevp)) + call init_vertical_interp(etam,nlev) + endif + + ! set prescribed state at level midpoints + do ie = nets,nete; do k=1,nlev; do j=1,np; do i=1,np + lon = elem(ie)%spherep(i,j)%lon; lat = elem(ie)%spherep(i,j)%lat + z = H * log(1.0d0/hvcoord%etam(k)) + p = p0 * hvcoord%etam(k) + call test1_advection_hadley(time,lon,lat,p,z,zcoords,u,v,w,t,phis,ps,rho,q(1),q(2)) + dp = pressure_thickness(ps,k,hvcoord) + call set_state(u,v,T,ps,phis,dp,zm(k), i,j,k,elem(ie),n0,n1) + if(time==0) call set_tracers(q,qsize,dp,i,j,k,lat,lon,elem(ie)) + + enddo; enddo; enddo; enddo + + ! set prescribed state at level interfaces + do ie = nets,nete; do k=1,nlevp; do j=1,np; do i=1,np + lon = elem(ie)%spherep(i,j)%lon; lat = elem(ie)%spherep(i,j)%lat + z = H * log(1.0d0/hvcoord%etai(k)) + p = p0 * hvcoord%etai(k) + call test1_advection_hadley(time,lon,lat,p,z,zcoords,u,v,w,T,phis,ps,rho,q(1),q(2)) + + ! get vertical derivative of p at point i,j,k + dp_dn = ddn_hyai(k)*p0 + ddn_hybi(k)*ps + + ! get vertical eta velocity at point i,j,k + eta_dot = -g*rho*w/p0 + + ! store vertical mass flux + elem(ie)%derived%eta_dot_dpdn(i,j,k) = eta_dot * dp_dn + + enddo; enddo; enddo; enddo + +end subroutine + +!_____________________________________________________________________ +subroutine dcmip2012_test1_3(elem,hybrid,hvcoord,nets,nete,time,n0,n1,deriv) + + ! Horizontal advection of thin cloud-like tracers over orography + + type(element_t), intent(inout), target :: elem(:) ! element array + type (derivative_t),intent(in) :: deriv + type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure + type(hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinates + integer, intent(in) :: nets,nete ! start, end element index + real(rl), intent(in) :: time ! current time + integer, intent(in) :: n0,n1 ! time level indices + + logical :: initialized = .false. + + integer, parameter :: cfv = 0 ! h-vel is not coordinate following + integer, parameter :: zcoords = 0 ! we are not using z coords + logical, parameter :: use_eta = .true. ! we are using hybrid eta coords + real(rl), parameter :: & + T0 = 300.d0, & ! temperature (K) + ztop = 12000.d0, & ! model top (m) + H = Rd * T0 / g ! scale height + + integer :: i,j,k,ie ! loop indices + real(rl):: lon,lat,hyam,hybm,hyai,hybi ! pointwise coordiantes + real(rl):: p,z,phis,u,v,w,T,phis_ps,ps,rho,q(4),dp,gc ! pointwise field values + real(rl):: grad_p(np,np,2),p_i(np,np),u_i(np,np),v_i(np,np) + real(rl):: etai(nlevp),etam(nlev) ! vertical interpolation levels + + ! set vertical interpolation coordinates + if(.not. initialized) then + initialized = .true. + if (hybrid%masterthread) write(iulog,*) 'initializing dcmip2012 test 1-1: 3d deformational flow' + call get_evenly_spaced_z(zi,zm, 0.0_rl,ztop) ! get evenly spaced z levels + etai = exp(-zi/H) ! set interpolated eta levels from z + etam = 0.5_rl*(etai(1:nlev)+etai(2:nlevp)) + call init_vertical_interp(etam,nlev) + endif + + ! set prescribed state at level midpoints + do ie = nets,nete; do k=1,nlev; do j=1,np; do i=1,np + hyam=hvcoord%hyam(k); hybm=hvcoord%hybm(k) + lon = elem(ie)%spherep(i,j)%lon; lat = elem(ie)%spherep(i,j)%lat + call test1_advection_orography(lon,lat,p,z,zcoords,cfv,use_eta,hyam,hybm,gc,u,v,w,t,phis,ps,rho,q(1),q(2),q(3),q(4)) + dp = pressure_thickness(ps,k,hvcoord) + call set_state(u,v,T,ps,phis,dp,zm(k), i,j,k,elem(ie),n0,n1) + if(time==0) call set_tracers(q,qsize,dp,i,j,k,lat,lon,elem(ie)) + + enddo; enddo; enddo; enddo + + ! set prescribed state at level interfaces + do ie = nets,nete; + do k=1,nlevp; + do j=1,np; do i=1,np + hyai=hvcoord%hyai(k); hybi=hvcoord%hybi(k) + lon = elem(ie)%spherep(i,j)%lon; lat = elem(ie)%spherep(i,j)%lat + call test1_advection_orography (lon,lat,p,z,zcoords,cfv,use_eta,hyai,hybi,gc,u,v,w,t,phis,ps,rho,q(1),q(2),q(3),q(4)) + p_i(i,j) = p + u_i(i,j) = u + v_i(i,j) = v + enddo; enddo + + ! get vertical mass flux + grad_p = gradient_sphere(p_i,deriv,elem(ie)%Dinv) + elem(ie)%derived%eta_dot_dpdn(:,:,k) = -u_i*grad_p(:,:,1) - v_i*grad_p(:,:,2) + enddo; + elem(ie)%derived%eta_dot_dpdn(:,:,1) = 0 + elem(ie)%derived%eta_dot_dpdn(:,:,nlevp) = 0 + enddo; + +end subroutine + +!_____________________________________________________________________ +subroutine dcmip2012_test2_0(elem,hybrid,hvcoord,nets,nete) + + ! steady state atmosphere with orography + + type(element_t), intent(inout), target :: elem(:) ! element array + type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure + type(hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinates + integer, intent(in) :: nets,nete ! start, end element index + + integer, parameter :: zcoords = 0 ! we are not using z coords + logical, parameter :: use_eta = .true. ! we are using hybrid eta coords + real(rl), parameter :: & + T0 = 300.d0, & ! temperature (K) + gamma = 0.0065d0, & ! temperature lapse rate (K/m) + ztop = 12000.d0, & ! model top (m) + exponent= g/(Rd*gamma) + + integer :: i,j,k,ie ! loop indices + real(rl):: lon,lat,hyam,hybm ! pointwise coordiantes + real(rl):: p,z,phis,u,v,w,T,phis_ps,ps,rho,q(1),dp ! pointwise field values + real(rl):: etai(nlevp),etam(nlev) ! vertical interpolation levels + + if (hybrid%masterthread) write(iulog,*) 'initializing dcmip2012 test 2-0: steady state atmosphere with orography' + + ! set vertical interpolation coordinates + call get_evenly_spaced_z(zi,zm, 0.0_rl,ztop) ! get evenly spaced z levels + etai = (1.d0 - gamma/T0*zi)**exponent ! set eta levels from z in orography-free region + etam = 0.5_rl*(etai(1:nlev)+etai(2:nlevp)) + call init_vertical_interp(etam,nlev) + + ! set initial conditions + do ie = nets,nete; do k=1,nlev; do j=1,np; do i=1,np + call get_coordinates(lat,lon,hyam,hybm, i,j,k,elem(ie),hvcoord) + call test2_steady_state_mountain(lon,lat,p,z,zcoords,use_eta,hyam,hybm,u,v,w,T,phis,ps,rho,q(1)) + dp = pressure_thickness(ps,k,hvcoord) + call set_state(u,v,T,ps,phis,dp,zm(k), i,j,k,elem(ie),1,nt) + call set_tracers(q,1,dp,i,j,k,lat,lon,elem(ie)) + enddo; enddo; enddo; enddo + +end subroutine + +!_____________________________________________________________________ +subroutine dcmip2012_test2_x(elem,hybrid,hvcoord,nets,nete,shear) + + ! nonhydrostatic orographic waves (with or without shear) + + type(element_t), intent(inout), target :: elem(:) ! element array + type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure + type(hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinates + integer, intent(in) :: nets,nete ! start, end element index + integer, intent(in) :: shear ! flag: 1=shear 0=no shear + + integer, parameter :: zcoords = 0 ! we are not using z coords + logical, parameter :: use_eta = .true. ! we are using hybrid eta coords + real(rl), parameter :: & + Teq = 300.d0, & ! temperature at equator + ztop = 30000.d0, & ! model top (m) + H = Rd*Teq/g ! characteristic height scale + + integer :: i,j,k,ie ! loop indices + real(rl):: lon,lat,hyam,hybm ! pointwise coordiantes + real(rl):: p,z,phis,u,v,w,T,phis_ps,ps,rho,q(1),dp ! pointwise field values + real(rl):: etai(nlevp),etam(nlev) ! vertical interpolation levels + + if (hybrid%masterthread) write(iulog,*) 'initializing dcmip2012 test 2-x: steady state atmosphere with orography' + + ! set vertical interpolation coordinates + call get_evenly_spaced_z(zi,zm, 0.0_rl,ztop) ! get evenly spaced z levels + etai = exp(-zi/H) ! set interpolated eta levels from z + etam = 0.5_rl*(etai(1:nlev)+etai(2:nlevp)) + call init_vertical_interp(etam,nlev) + + ! set initial conditions + do ie = nets,nete; do k=1,nlev; do j=1,np; do i=1,np + call get_coordinates(lat,lon,hyam,hybm, i,j,k,elem(ie),hvcoord) + call test2_schaer_mountain(lon,lat,p,z,zcoords,use_eta,hyam,hybm,shear,u,v,w,T,phis,ps,rho,q(1)) + dp = pressure_thickness(ps,k,hvcoord) + call set_state(u,v,T,ps,phis,dp,zm(k), i,j,k,elem(ie),1,nt) + call set_tracers(q,1,dp,i,j,k,lat,lon,elem(ie)) + enddo; enddo; enddo; enddo + + ! store initial velocity fields for use in sponge layer + allocate( u0(np,np,nlev,nelemd) ) + allocate( v0(np,np,nlev,nelemd) ) + + do ie = nets,nete + u0(:,:,:,ie) = elem(ie)%state%v(:,:,1,:,1) + v0(:,:,:,ie) = elem(ie)%state%v(:,:,2,:,1) + enddo + +end subroutine + +!_____________________________________________________________________ +subroutine dcmip2012_test2_x_forcing(elem,hybrid,hvcoord,nets,nete,n,dt) + + type(element_t), intent(inout), target :: elem(:) ! element array + type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure + type(hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinates + integer, intent(in) :: nets,nete ! start, end element index + integer, intent(in) :: n ! time level index + real(rl), intent(in) :: dt ! time-step size + + integer :: ie, k + + real(rl), parameter :: & + tau = 25.0, & ! rayleigh function relaxation time + ztop = 30000.d0, & ! model top + zh = 20000.d0 ! sponge-layer cutoff height + + real(rl) :: f_d(nlev) ! damping function + real(rl) :: z(np,np,nlev) + + ! Compute damping as a function of layer-midpoint height + where(zm .ge. zh) + f_d = sin(pi/2 *(zm - zh)/(ztop - zh))**2 + elsewhere + f_d = 0.0d0 + end where + + ! apply sponge layer forcing to momentum terms + do ie=nets,nete + do k=1,nlev + elem(ie)%derived%FM(:,:,1,k) = -f_d(k)/tau * ( elem(ie)%state%v(:,:,1,k,n) - u0(:,:,k,ie) ) + elem(ie)%derived%FM(:,:,2,k) = -f_d(k)/tau * ( elem(ie)%state%v(:,:,2,k,n) - v0(:,:,k,ie) ) + enddo + enddo + +end subroutine + +!_____________________________________________________________________ +subroutine dcmip2012_test3(elem,hybrid,hvcoord,nets,nete) + + ! nonhydrostatic gravity waves + + type(element_t), intent(inout), target :: elem(:) ! element array + type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure + type(hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinates + integer, intent(in) :: nets,nete ! start, end element index + + integer, parameter :: zcoords = 0 ! we are not using z coords + logical, parameter :: use_eta = .true. ! we are using hybrid eta coords + + real(rl), parameter :: & ! parameters needed to get eta from z + T0 = 300.d0, & ! temperature (k) + ztop = 10000.d0, & ! model top (m) + N = 0.01d0, & ! Brunt-Vaisala frequency + bigG = (g*g)/(N*N*Cp) ! temperature, isothermal + + integer :: i,j,k,ie ! loop indices + real(rl):: lon,lat,hyam,hybm ! pointwise coordiantes + real(rl):: p,z,phis,u,v,w,T,T_mean,phis_ps,ps,rho,rho_mean,q(1),dp ! pointwise field values + real(rl):: etai(nlevp),etam(nlev) ! vertical interpolation levels + + if (hybrid%masterthread) write(iulog,*) 'initializing dcmip2012 test 3-0: nonhydrostatic gravity waves' + + ! set vertical interpolation coordinates + call get_evenly_spaced_z(zi,zm, 0.0_rl,ztop) ! get evenly spaced z levels + etai = ( (bigG/T0)*(exp(-zi*N*N/g) -1 )+1 ) **(1.0/kappa) ! set eta levels from z at equator + etam = 0.5_rl*(etai(1:nlev)+etai(2:nlevp)) + call init_vertical_interp(etam,nlev) + + ! set initial conditions + do ie = nets,nete; do k=1,nlev; do j=1,np; do i=1,np + call get_coordinates(lat,lon,hyam,hybm, i,j,k,elem(ie),hvcoord) + call test3_gravity_wave(lon,lat,p,z,zcoords,use_eta,hyam,hybm,u,v,w,T,T_mean,phis,ps,rho,rho_mean,q(1)) + dp = pressure_thickness(ps,k,hvcoord) + call set_state(u,v,T,ps,phis,dp,zm(k), i,j,k,elem(ie),1,nt) + call set_tracers(q,1, dp,i,j,k,lat,lon,elem(ie)) + enddo; enddo; enddo; enddo + +end subroutine + +#if 0 + +!_____________________________________________________________________ +subroutine dcmip2016_test1(elem,hybrid,hvcoord,nets,nete) + + ! moist baroclininc wave + + type(element_t), intent(inout), target :: elem(:) ! element array + type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure + type(hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinates + integer, intent(in) :: nets,nete ! start, end element index + + integer, parameter :: zcoords = 0 ! we are not using z coords + logical, parameter :: use_eta = .true. ! we are using hybrid eta coords + + real(rl), parameter :: & ! parameters needed to get eta from z + T0 = 300.d0, & ! temperature (k) + ztop = 10000.d0, & ! model top (m) + N = 0.01d0, & ! Brunt-Vaisala frequency + bigG = (g*g)/(N*N*Cp) ! temperature, isothermal + + integer :: i,j,k,ie ! loop indices + real(rl):: lon,lat,hyam,hybm ! pointwise coordiantes + real(rl):: p,z,phis,u,v,w,T,T_mean,phis_ps,ps,rho,rho_mean,q(1),dp ! pointwise field values + + if (hybrid%masterthread) write(iulog,*) 'initializing dcmip2012 test 3-0: nonhydrostatic gravity waves' + + ! set analytic vertical coordinates + call get_evenly_spaced_z(zi,zm, 0.0_rl,ztop) ! get evenly spaced z levels + hvcoord%etai = ( (bigG/T0)*(exp(-zi*N*N/g) -1 )+1 ) **(1.0/kappa) ! set eta levels from z at equator + call set_hybrid_coefficients(hvcoord,hybrid, hvcoord%etai(1), 1.0_rl)! set hybrid A and B from eta levels + call set_layer_locations(hvcoord, .true., hybrid%masterthread) + + ! set initial conditions + do ie = nets,nete; do k=1,nlev; do j=1,np; do i=1,np + call get_coordinates(lat,lon,hyam,hybm, i,j,k,elem(ie),hvcoord) + !call test3_gravity_wave(lon,lat,p,z,zcoords,use_eta,hyam,hybm,u,v,w,T,T_mean,phis,ps,rho,rho_mean,q(1)) + dp = pressure_thickness(ps,k,hvcoord) + call set_state(u,v,T,ps,phis,dp,zm(k), i,j,k,elem(ie),1,nt) + call set_tracers(q,1, dp,i,j,k,lat,lon,elem(ie)) + enddo; enddo; enddo; enddo + +end subroutine + +!_____________________________________________________________________ +subroutine dcmip2016_test2(elem,hybrid,hvcoord,nets,nete) + + ! tropical cyclone + + type(element_t), intent(inout), target :: elem(:) ! element array + type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure + type(hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinates + integer, intent(in) :: nets,nete ! start, end element index + + logical, parameter :: use_eta = .true. ! we are using hybrid eta coords + + real(rl), parameter :: ztop = 15000.d0 ! model top (m) + + integer :: i,j,k,ie,zcoords ! loop indices + real(rl):: lon,lat ! pointwise coordiantes + real(rl):: p,z,phis,u,v,w,T,thetav,ps,rho,q(3),dp,eta ! pointwise field values + real(rl):: A,B,etai(nlevp),etam(nlev) + + if (hybrid%masterthread) write(iulog,*) 'initializing dcmip2016 test 2: tropical cyclone' + + ! compute interpolation points at evenly spaced z + call get_evenly_spaced_z(zi,zm, 0.0_rl,ztop) ! get evenly spaced z levels + + zcoords=1; lon =0; lat = 0 + do k=1,nlevp + B = 1.0 - zi(k)/zi(1) + call tropical_cyclone_test(lon,lat,p,zi(k),1,u,v,t,thetav,phis,ps,rho,q(1)) + A = (p - B*ps)/p0 + etai(k) = A+B + enddo + + etam = 0.5_rl*(etai(1:nlev)+etai(2:nlevp)) + call init_vertical_interp(etam,nlev) + + ! set initial conditions + do ie = nets,nete; do k=1,nlev; do j=1,np; do i=1,np + lon = elem(ie)%spherep(i,j)%lon + lat = elem(ie)%spherep(i,j)%lat + + ! call routine just to get ps value + z=0; call tropical_cyclone_test(lon,lat,p,z,1,u,v,t,thetav,phis,ps,rho,q(1)) + + ! get pressure from ps and level + p = hvcoord%hyam(k)*p0 + hvcoord%hybm(k)*ps + + ! call routine again using vertical pressure on eta level + call tropical_cyclone_test(lon,lat,p,z,0,u,v,t,thetav,phis,ps,rho,q(1)) + q(2)=0 + q(3)=0 + dp = pressure_thickness(ps,k,hvcoord) + call set_state(u,v,T,ps,phis,dp,zm(k), i,j,k,elem(ie),1,nt) + call set_tracers(q,3, dp,i,j,k,lat,lon,elem(ie)) + enddo; enddo; enddo; enddo + +end subroutine + +!_____________________________________________________________________ +subroutine dcmip2016_test2_forcing(elem,hybrid,hvcoord,nets,nete,n,n_q,dt) + + use dcmip2016_physics_z, only: dcmip2016_physics + use vertical_se, only: compress, decompress, n_unique + + ! get forcing terms for tropical cyclone tests + + type(element_t), intent(inout), target :: elem(:) ! element array + type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure + type(hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinates + integer, intent(in) :: nets,nete ! start, end element index + integer, intent(in) :: n, n_q ! dynamics and tracer time indices + real(rl), intent(in) :: dt ! time-step size + + real(rl), dimension(n_unique) :: u0,v0,T0,qv0,qc0,qr0,Am,Bm + real(rl), dimension(n_unique) :: u,v,p,qv,qc,qr,rho,T,dp,z_m + real(rl) :: lat, precl, z_i(n_unique+1), ps + + integer :: ie, i,j,k + integer, parameter :: test = 2 + integer, parameter :: pbl_type = 0 ! 0=Reed_jablonowski, 1=Bryan boundary layer + integer, parameter :: prec_type= 0 ! 0=Kessler physics, 1=Reed-Jabolonowski microphysics + + Am = compress(hvcoord%hyam) + Bm = compress(hvcoord%hybm) + + ! call physics routine one column at a time + do ie = nets,nete; do j=1,np; do i=1,np + + ps = elem(ie)%state%ps_v(i,j,n) + forall(k=1:n_unique) p(k)= Am(k)*p0 + Bm(k)*ps + + T = compress( elem(ie)%state%T (i,j,:,n) ) + z_m = compress( elem(ie)%derived%phi(i,j,:)/g ) + z_i(1:n_unique)=z_m; z_i(n_unique+1)=0 ! (TODO: get accurate levels for precl from Reed-Jablonowski) + z_i(2:n_unique) = 0.5*( z_m(1:n_unique-1) + z_m(2:n_unique)) + z_i(1) = z_m(1) + z_i(n_unique+1) = z_m(n_unique) + + ! get dry air density from ideal gas law + rho = p/(Rd*T) + + ! get each state vector at column i,j + lat = elem(ie)%spherep(i,j)%lat + u = compress( elem(ie)%state%v(i,j,1,:,n) ) + v = compress( elem(ie)%state%v(i,j,2,:,n) ) + qv = compress( elem(ie)%state%Q(i,j,:,1) ) ! vapor mixing ratio (gm/gm) + qc = compress( elem(ie)%state%Q(i,j,:,2) ) ! cloud water mixing ratio (gm/gm) + qr = compress( elem(ie)%state%Q(i,j,:,3) ) ! rain water mixing ratio (gm/gm) + + precl = 0 ! precipitation rate (output from physics routine) + + ! store current column values to compute differences after update + u0=u; v0=v; T0=T; qv0=qv; qc0=qc; qr0=qr + + ! apply KESSLER/Reed-Jabolonowski microphysics and Reed-Jablonoswki/Bryan boundary layer + call dcmip2016_physics(test,u,v,p,qv,qc,qr,rho,dt,z_m,z_i,lat,n_unique,precl,pbl_type,prec_type) + + ! compute new temperatures from ideal gas law applied to new pressures + T = p/(Rd*rho) + + ! get forcing terms from difference between old and new column values + elem(ie)%derived%FM(i,j,1,:,n ) = decompress( u - u0 ) + elem(ie)%derived%FM(i,j,2,:,n ) = decompress( v - v0 ) + elem(ie)%derived%FT(i,j, :,n ) = decompress( T - T0 ) + elem(ie)%derived%FQ(i,j,:,1,n_q) = decompress( qv - qv0) + elem(ie)%derived%FQ(i,j,:,2,n_q) = decompress( qc - qc0) + elem(ie)%derived%FQ(i,j,:,3,n_q) = decompress( qr - qr0) + +if(any(isnan(u - u0))) call abortmp("nan in u") +if(any(isnan(v - v0))) call abortmp("nan in v") +if(any(isnan(T - T0))) call abortmp("nan in T") +if(any(isnan(qv - qv0))) call abortmp("nan in qv") +if(any(isnan(qc - qc0))) call abortmp("nan in qc") +if(any(isnan(qr - qr0))) call abortmp("nan in qr") + + enddo; enddo; + + enddo + +end subroutine + +!_____________________________________________________________________ +subroutine dcmip2016_test3(elem,hybrid,hvcoord,nets,nete) + + ! supercell storm + + type(element_t), intent(inout), target :: elem(:) ! element array + type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure + type(hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinates + integer, intent(in) :: nets,nete ! start, end element index + + integer, parameter :: zcoords = 0 ! we are not using z coords + logical, parameter :: use_eta = .true. ! we are using hybrid eta coords + + real(rl), parameter :: & ! parameters needed to get eta from z + T0 = 300.d0, & ! temperature (k) + ztop = 10000.d0, & ! model top (m) + N = 0.01d0, & ! Brunt-Vaisala frequency + bigG = (g*g)/(N*N*Cp) ! temperature, isothermal + + integer :: i,j,k,ie ! loop indices + real(rl):: lon,lat,hyam,hybm ! pointwise coordiantes + real(rl):: p,z,phis,u,v,w,T,T_mean,phis_ps,ps,rho,rho_mean,q(1),dp ! pointwise field values + + if (hybrid%masterthread) write(iulog,*) 'initializing dcmip2012 test 3-0: nonhydrostatic gravity waves' + + ! set analytic vertical coordinates + call get_evenly_spaced_z(zi,zm, 0.0_rl,ztop) ! get evenly spaced z levels + hvcoord%etai = ( (bigG/T0)*(exp(-zi*N*N/g) -1 )+1 ) **(1.0/kappa) ! set eta levels from z at equator + call set_hybrid_coefficients(hvcoord,hybrid, hvcoord%etai(1), 1.0_rl)! set hybrid A and B from eta levels + call set_layer_locations(hvcoord, .true., hybrid%masterthread) + + ! set initial conditions + do ie = nets,nete; do k=1,nlev; do j=1,np; do i=1,np + call get_coordinates(lat,lon,hyam,hybm, i,j,k,elem(ie),hvcoord) + !call test3_gravity_wave(lon,lat,p,z,zcoords,use_eta,hyam,hybm,u,v,w,T,T_mean,phis,ps,rho,rho_mean,q(1)) + dp = pressure_thickness(ps,k,hvcoord) + call set_state(u,v,T,ps,phis,dp,zm(k), i,j,k,elem(ie),1,nt) + call set_tracers(q,1, dp,i,j,k,lat,lon,elem(ie)) + enddo; enddo; enddo; enddo + +end subroutine + +#endif + +!_____________________________________________________________________ +subroutine get_evenly_spaced_z(zi,zm, zb,zt) + + real(rl), intent(in) :: zb,zt ! top and bottom coordinates + real(rl), intent(inout) :: zi(nlevp) ! z at interfaces + real(rl), intent(inout) :: zm(nlev) ! z at midpoints + integer :: k + + forall(k=1:nlevp) zi(k) = zt-(k-1)*(zt-zb)/(nlevp-1) + zm = 0.5_rl*( zi(2:nlevp) + zi(1:nlev) ) + +end subroutine + +!_____________________________________________________________________ +subroutine set_hybrid_coefficients(hv, hybrid, eta_t, c) + + ! create an analytical set of A,B coefficients, given known eta levels + + type(hvcoord_t), intent(inout) :: hv ! hybrid vertical coordinate stucture + type(hybrid_t), intent(in) :: hybrid ! hybrid parallal structure + real(rl), intent(in) :: eta_t ! top eta level + real(rl), intent(in) :: c ! exponent + + real(rl) :: eta_c, tmp + integer :: k + + ! place cutoff halfway between bottom and top eta coordiantes + eta_c = hv%etai(nlev/2) + + do k=1,nlevp + ! get values of hybrid coefficients + tmp = max( (hv%etai(k)-eta_c)/(1.0-eta_c), 0.0_rl) + hv%hybi(k) = tmp**c + hv%hyai(k) = hv%etai(k) - hv%hybi(k) + if(hybrid%par%masterproc) print *,k,': etai = ',hv%etai(k),' Ai = ',hv%hyai(k),' Bi = ',hv%hybi(k); + + ! get derivatives of hybrid coefficients + ddn_hybi(k) = c*tmp**(c-1) + if(hv%etai(k)>eta_c) ddn_hybi(k)=0.0d0 + ddn_hyai(k) = 1.0d0 - ddn_hybi(k) + enddo + + hv%hyam = 0.5_rl *(hv%hyai(2:nlev+1) + hv%hyai(1:nlev)) + hv%hybm = 0.5_rl *(hv%hybi(2:nlev+1) + hv%hybi(1:nlev)) + hv%etam = hv%hyam + hv%hybm + +end subroutine + +!_____________________________________________________________________ +subroutine get_coordinates(lat,lon,hyam,hybm, i,j,k,elem,hvcoord) + + ! get lat,lon, vertical coords at node(i,j,k) + + real(rl), intent(out):: lon,lat,hyam,hybm + integer, intent(in) :: i,j,k + type(element_t), intent(in) :: elem + type(hvcoord_t), intent(in) :: hvcoord + + ! get horizontal coordinates at column i,j + lon = elem%spherep(i,j)%lon + lat = elem%spherep(i,j)%lat + + ! get hybrid coeffiecients at midpoint of vertical level k + hyam = hvcoord%hyam(k) + hybm = hvcoord%hybm(k) + +end subroutine + +!_____________________________________________________________________ +real(rl) function pressure_thickness(ps,k,hv) + + real(rl), intent(in) :: ps + integer, intent(in) :: k + type(hvcoord_t), intent(in) :: hv + pressure_thickness = (hv%hyai(k+1)-hv%hyai(k))*p0 + (hv%hybi(k+1)-hv%hybi(k))*ps + +end function + +!_____________________________________________________________________ +subroutine set_state(u,v,T,ps,phis,dp,zm, i,j,k,elem,n0,n1) + + ! set state variables at node(i,j,k) at layer midpoints + + real(rl), intent(in) :: u,v,T,ps,phis,dp,zm + integer, intent(in) :: i,j,k,n0,n1 + type(element_t), intent(inout) :: elem + + ! set prognostic state variables at level midpoints + elem%state%v (i,j,1,k,n0:n1) = u + elem%state%v (i,j,2,k,n0:n1) = v + elem%state%T (i,j,k,n0:n1) = T + elem%state%dp3d(i,j,k,n0:n1) = dp + elem%state%ps_v(i,j,n0:n1) = ps + elem%state%phis(i,j) = phis + + ! set some diagnostic variables + elem%derived%dp(i,j,k) = dp + elem%derived%phi(i,j,k) = g*zm + +end subroutine + +!_____________________________________________________________________ +subroutine set_tracers(q,nq, dp,i,j,k,lat,lon,elem) + + ! set tracer values at node(i,j,k) + + real(rl), intent(in) :: q(nq), dp, lat, lon + integer, intent(in) :: i,j,k,nq + type(element_t), intent(inout) :: elem + + real(rl), parameter :: wl = 1.0 ! checkerboard wavelength in dg + integer :: qi + + ! set known tracers to q and the rest to a checkerboard pattern + elem%state%Q(i,j,k,:) = 0.5d0*(1.0d0+sign(1.0d0,sin(lat*360.0/wl)*sin(lon*360.0/wl))) + elem%state%Q(i,j,k,1:nq) = q + + ! compute tracer mass qdp from mixing ratio q + do qi = 1,nq + elem%state%Qdp (i,j,k,qi,:) = q(qi)*dp + enddo + +end subroutine +end module dcmip_tests + +#endif diff --git a/components/homme/src/pese/derivative_mod.F90 b/components/homme/src/pese/derivative_mod.F90 new file mode 100644 index 000000000000..d30d3f8e719b --- /dev/null +++ b/components/homme/src/pese/derivative_mod.F90 @@ -0,0 +1,34 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +module derivative_mod + use derivative_mod_base, only: & + allocate_subcell_integration_matrix,& + curl_sphere, & + curl_sphere_wk_testcov,& + derivative_t,& + derivinit,& + divergence,& + divergence_sphere,& + divergence_sphere_wk,& + edge_flux_u_cg,& + element_boundary_integral,& + gradient,& + gradient_sphere,& + gradient_sphere_wk_testcontra,& + gradient_sphere_wk_testcov,& + gradient_wk,& + laplace_sphere_wk,& + limiter_optim_iter_full,& + subcell_Laplace_fluxes,& + subcell_div_fluxes,& + subcell_dss_fluxes,& + subcell_integration,& + ugradv_sphere, vorticity_sphere,& + vlaplace_sphere_wk,& + vorticity,& + vorticity_sphere_diag + + implicit none +end module derivative_mod diff --git a/components/homme/src/pese/edge_mod.F90 b/components/homme/src/pese/edge_mod.F90 new file mode 100644 index 000000000000..1fdcfabe08eb --- /dev/null +++ b/components/homme/src/pese/edge_mod.F90 @@ -0,0 +1,35 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +module edge_mod + use edge_mod_base, only:& + FreeEdgeBuffer,& + FreeGhostBuffer3D,& + FreeLongEdgeBuffer,& + LongEdgeVpack,& + LongEdgeVunpackMIN,& + edgeDGVpack,& + edgeDGVunpack,& + edgeDefaultVal,& + edgeSpack,& + edgeSunpackMax,& + edgeSunpackMin,& + edgeVpack,& + edgeVunpack,& + edgeVunpackMAX,& + edgeVunpackMIN,& + edgeVunpackVert,& + ghostVpack3d,& + ghostVpack_unoriented,& + ghostVpackfull,& + ghostVunpack3d,& + ghostVunpack_unoriented,& + ghostVunpackfull,& + initEdgeBuffer,& + initEdgeSBuffer,& + initGhostBuffer3D,& + initLongEdgeBuffer + + implicit none +end module edge_mod diff --git a/components/homme/src/pese/element_ops.F90 b/components/homme/src/pese/element_ops.F90 new file mode 100644 index 000000000000..6a8ea350b9a9 --- /dev/null +++ b/components/homme/src/pese/element_ops.F90 @@ -0,0 +1,269 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +! get, set functions provided by each model + +module element_ops + + use control_mod, only: statefreq + use derivative_mod, only: derivative_t + use dimensions_mod, only: np, nlev, nlevp, nelemd, qsize, qsize_d + use edge_mod, only: edgevpack, edgevunpack + use edgetype_mod, only: edgebuffer_t + use element_mod, only: element_t + use element_state, only: timelevels, elem_state_t, derived_state_t + use hybrid_mod, only: hybrid_t + use hybvcoord_mod, only: hvcoord_t + use kinds, only: rl => real_kind, real_kind, iulog + use perf_mod, only: t_startf, t_stopf, t_barrierf, t_adj_detailf ! _EXTERNAL + use parallel_mod, only: abortmp + use physical_constants, only: kappa, p0, g + use shr_const_mod, only: unset => shr_const_spval + use vertical_se, only: eta_derivative, elem_height, vertical_dss, v_interpolate + + implicit none + +contains + + !_____________________________________________________________________ + subroutine get_field(elem,name,field,hvcoord,nt,ntQ) + + ! get scalar field by name + + implicit none + type (element_t), intent(in) :: elem + character(len=*), intent(in) :: name + real (kind=real_kind), intent(out):: field(np,np,nlev) + type (hvcoord_t), intent(in) :: hvcoord + integer, intent(in) :: nt + integer, intent(in) :: ntQ + + integer :: i,j,k, qi + real(rl) :: var(np,np,nlev) + + ! get prognostic variables + + select case(name) + + case('T' ); var = elem%state%T (:,:,:,nt) + case('u' ); var = elem%state%v (:,:,1,:,nt) + case('v' ); var = elem%state%v (:,:,2,:,nt) + case('p' ); var = elem%state%dp3d(:,:,:,nt) + case('ps' ); forall(k=1:nlev) var(:,:,k) = elem%state%ps_v(:,:,nt) + + case('Q' ); var = elem%state%Q(:,:,:,1) + case('Q2' ); qi=2; if(qsize_d>1) var = elem%state%Q(:,:,:,qi) + case('Q3' ); qi=3; if(qsize_d>2) var = elem%state%Q(:,:,:,qi) + case('Q4' ); qi=4; if(qsize_d>3) var = elem%state%Q(:,:,:,qi) + case('Q5' ); qi=5; if(qsize_d>4) var = elem%state%Q(:,:,:,qi) + + case('phi'); var = elem%derived%phi + case('geo'); var = elem%derived%phi + case('omega');var = elem%derived%omega + + case default + print *,'name = ',trim(name) + call abortmp('ERROR: get_field name not supported in this model') + end select + + ! interpolate each column from vertical gll nodes to vertical ouput levels + do i=1,np + do j=1,np + field(i,j,:) = v_interpolate(var(i,j,:),nlev) + enddo + enddo + end subroutine + + !_____________________________________________________________________________ + function get_vector_field(elem, name, n0, ni) result(vfield) + + ! Get vector-field data by name + + character*(*), intent(in) :: name + type(element_t), intent(in) :: elem + integer, intent(in) :: n0 ! time level index + integer, intent(in) :: ni ! num vertical interp pts + + real(rl) :: vfield(np,np,2,ni) + real(rl) :: var(np,np,2,nlev) + integer :: i,j + select case(name) ! switch on short variable name + + case('v'); + var = elem%state%v(:,:,:,:,n0) + + case default; var = unset ! assign special "missing" value + endselect + + ! interpolate each column + do i=1,np + do j=1,np + vfield(i,j,1,:) = v_interpolate(var(i,j,1,:),ni) + vfield(i,j,2,:) = v_interpolate(var(i,j,2,:),ni) + enddo + enddo + + end function + + !_____________________________________________________________________ + subroutine set_state(u,v,w,T,ps,phis,p,dp,zm,g,i,j,k,elem,n0,n1) + + ! set state variables at node(i,j,k) at layer midpoints + + real(real_kind), intent(in) :: u,v,w,T,ps,phis,p,dp,zm,g + integer, intent(in) :: i,j,k,n0,n1 + type(element_t), intent(inout) :: elem + + ! set prognostic state variables at level midpoints + elem%state%v (i,j,1,k,n0:n1) = u + elem%state%v (i,j,2,k,n0:n1) = v + elem%state%T (i,j,k,n0:n1) = T + elem%state%dp3d(i,j,k,n0:n1) = dp + elem%state%ps_v(i,j,n0:n1) = ps + elem%state%phis(i,j) = phis + + ! set some diagnostic variables + elem%derived%dp(i,j,k) = dp + elem%derived%phi(i,j,k) = g*zm + + end subroutine + + !_____________________________________________________________________ + subroutine set_thermostate(elem,temperature,hvcoord,n0,n0_q) + + implicit none + + type (element_t), intent(inout) :: elem + real (kind=real_kind), intent(in) :: temperature(np,np,nlev) + type (hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct + integer :: n0,n0_q + + elem%state%T(:,:,:,n0)=temperature(:,:,:) + + end subroutine set_thermostate + + !_____________________________________________________________________ + subroutine copy_state(elem,nin,nout) + implicit none + type (element_t), intent(inout) :: elem + integer, intent(in) :: nin,nout + + elem%state%v (:,:,:,:,nout)= elem%state%v (:,:,:,:,nin) + elem%state%T (:,:,:, nout)= elem%state%T (:,:,:, nin) + elem%state%dp3d(:,:,:, nout)= elem%state%dp3d(:,:,:, nin) + elem%state%ps_v(:,:, nout)= elem%state%ps_v(:,:, nin) + + end subroutine copy_state + + !_____________________________________________________________________ + subroutine apply_vertical_dss(elem, nt) + + ! apply vertical dss to all prognostics and tracers + + type (element_t), intent(inout), target :: elem + integer, intent(in) :: nt + integer::l + + call vertical_dss( elem%state%T (:,:, :,nt) ) + call vertical_dss( elem%state%v (:,:,1,:,nt) ) + call vertical_dss( elem%state%v (:,:,2,:,nt) ) + + do l=1,qsize_d + call vertical_dss(elem%state%Qdp(:,:,:,l,nt)) + enddo + + end subroutine + + !_____________________________________________________________________ + subroutine pack_edge_data(buffer, elem, nt, ie) + + ! pack prognostics into edge buffer + + type (EdgeBuffer_t), intent(inout) :: buffer ! buffer for edge-data exchange + type (element_t), intent(in), target :: elem ! array of element_t structures + integer, intent(in) :: nt, ie ! time level and element index + + integer :: i + i = 0 + call edgeVpack(buffer, elem%state%ps_v(:,:, nt), 1 , i, ie); i=i+1 + call edgeVpack(buffer, elem%state%T (:,:,:, nt), nlev , i, ie); i=i+nlev + call edgeVpack(buffer, elem%state%v (:,:,:,:,nt), 2*nlev, i, ie); i=i+2*nlev + + end subroutine + + !_____________________________________________________________________ + subroutine unpack_edge_data(buffer, elem, nt, ie) + + ! unpack prognostics from edge buffer + + type (EdgeBuffer_t), intent(inout) :: buffer ! buffer for edge-data exchange + type (element_t), intent(inout), target :: elem ! array of element_t structures + integer, intent(in) :: nt, ie ! time level and element index + + integer :: i + i = 0 + call edgeVunpack(buffer, elem%state%ps_v(:,:, nt), 1 , i, ie); i=i+1 + call edgeVunpack(buffer, elem%state%T (:,:,:, nt), nlev , i, ie); i=i+nlev + call edgeVunpack(buffer, elem%state%v (:,:,:,:,nt), 2*nlev, i, ie); i=i+2*nlev + + end subroutine + + !_____________________________________________________________________ + subroutine apply_map(M, elem, nt) + + ! apply pointwise 2d map to each prognostic + + real(rl), intent(in) :: M(np,np) + type (element_t), intent(inout), target :: elem ! array of element_t structures + integer, intent(in) :: nt ! time level + + integer :: k + elem%state%ps_v(:,:, nt) = elem%state%ps_v(:,:,nt)*M + do k=1,nlev + elem%state%T(:,:,k, nt) = elem%state%T(:,:, k,nt)*M + elem%state%v(:,:,1,k,nt) = elem%state%v(:,:,1,k,nt)*M + elem%state%v(:,:,2,k,nt) = elem%state%v(:,:,2,k,nt)*M + enddo + + end subroutine + + !_____________________________________________________________________ + subroutine display_max_and_min(elem, hybrid, ie, nt, count) + + ! display verbose diagnostics + + type (element_t), intent(inout), target :: elem ! element + type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct + integer, intent(in) :: ie ! element number + integer, intent(in) :: nt ! time level to display + integer, intent(in) :: count ! iteration counter + + real(rl):: maxu,minu,maxT,minT,maxv,minv,maxps,minps + + if( mod(count, statefreq)==0 ) then + + maxu = maxval(elem%state%v (:,:,1,:,nt)) + minu = minval(elem%state%v (:,:,1,:,nt)) + maxv = maxval(elem%state%v (:,:,2,:,nt)) + minv = minval(elem%state%v (:,:,2,:,nt)) + maxps= maxval(elem%state%ps_v(:,:, nt)) + minps= minval(elem%state%ps_v(:,:, nt)) + maxT = maxval(elem%state%T (:,:,:, nt)) + minT = minval(elem%state%T (:,:,:, nt)) + + if (hybrid%masterthread) then + print *,"nt =",nt, "ie =",ie, " max u =",maxu," max v =",maxv," max T =",maxT," max ps=",maxps + endif + + if (any(isnan(elem%state%v(:,:,1,:,nt)))) stop 'detected NaN in u' + if (any(isnan(elem%state%v(:,:,2,:,nt)))) stop 'detected NaN in v' + if (any(isnan(elem%state%T(:,:,:,nt)))) stop 'detected NaN in T' + if (any(isnan(elem%state%ps_v(:,:,nt)))) stop 'detected NaN in ps_v' + + endif + + end subroutine + +end module + diff --git a/components/homme/src/pese/element_state.F90 b/components/homme/src/pese/element_state.F90 new file mode 100644 index 000000000000..27de21b6c8e2 --- /dev/null +++ b/components/homme/src/pese/element_state.F90 @@ -0,0 +1,144 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +module element_state + + use kinds, only: real_kind, long_kind, int_kind + use dimensions_mod, only: np, npsq, nlev, nlevp, qsize_d + + implicit none + private + integer, public, parameter :: timelevels = 3 + +! =========== PRIMITIVE-EQUATION DATA-STRUCTURES ===================== + + type, public :: elem_state_t + + ! prognostic variables for pese solver + + ! prognostics must match those in prim_restart_mod.F90 + ! vertically-lagrangian code advects dp3d instead of ps_v + ! tracers Q, Qdp always use 2 level time scheme + + real (kind=real_kind) :: v (np,np,2,nlev,timelevels) ! velocity 1 + real (kind=real_kind) :: T (np,np,nlev,timelevels) ! temperature 2 + real (kind=real_kind) :: dp3d(np,np,nlev,timelevels) ! delta p on levels 8 + real (kind=real_kind) :: ps_v(np,np,timelevels) ! surface pressure 4 + real (kind=real_kind) :: phis(np,np) ! surface geopotential (prescribed) 5 + real (kind=real_kind) :: Q (np,np,nlev,qsize_d) ! Tracer concentration 6 + real (kind=real_kind) :: Qdp (np,np,nlev,qsize_d,2) ! Tracer mass 7 + + end type elem_state_t + + !___________________________________________________________________ + type, public :: derived_state_t + + ! diagnostic variables for pese solver + + real (kind=real_kind) :: omega(np,np,nlev) ! dp/dt + + ! storage for subcycling tracers/dynamics + + real (kind=real_kind) :: vn0 (np,np,2,nlev) ! velocity for SE tracer advection + real (kind=real_kind) :: vstar(np,np,2,nlev) ! velocity on Lagrangian surfaces + real (kind=real_kind) :: dpdiss_biharmonic(np,np,nlev) ! mean dp dissipation tendency, if nu_p>0 + real (kind=real_kind) :: dpdiss_ave(np,np,nlev) ! mean dp used to compute psdiss_tens + + ! diagnostics for explicit timestep + real (kind=real_kind) :: phi(np,np,nlev) ! geopotential + real (kind=real_kind) :: omega_p(np,np,nlev) ! vertical tendency (derived) + real (kind=real_kind) :: eta_dot_dpdn(np,np,nlevp) ! mean vertical flux from dynamics + real (kind=real_kind) :: eta_dot_dpdn_prescribed(np,np,nlevp) ! prescribed wind test cases + + ! semi-implicit diagnostics: computed in explict-component, reused in Helmholtz-component. + real (kind=real_kind) :: grad_lnps(np,np,2) ! gradient of log surface pressure + real (kind=real_kind) :: zeta(np,np,nlev) ! relative vorticity + real (kind=real_kind) :: div(np,np,nlev,timelevels) ! divergence + + ! tracer advection fields used for consistency and limiters + real (kind=real_kind) :: dp(np,np,nlev) ! for dp_tracers at physics timestep + real (kind=real_kind) :: divdp(np,np,nlev) ! divergence of dp + real (kind=real_kind) :: divdp_proj(np,np,nlev) ! DSSed divdp + + ! forcing terms for CAM + real (kind=real_kind) :: FQ(np,np,nlev,qsize_d) ! tracer forcing + real (kind=real_kind) :: FM(np,np,2,nlev) ! momentum forcing + real (kind=real_kind) :: FT(np,np,nlev) ! temperature forcing + real (kind=real_kind) :: pecnd(np,np,nlev) ! pressure perturbation from condensate + real (kind=real_kind) :: FQps(np,np) ! forcing of FQ on ps_v + + end type derived_state_t + + + !___________________________________________________________________ + type, public :: elem_accum_t + +#ifdef ENERGY_DIAGNOSTICS + + ! Energy equation: + ! KE_t = T1 + T2 + D1 + Err + vertical & horizontal advection terms + ! IE_t = S1 + D2 + vertical & horizontal advection terms + ! PE_t = S2 + ! + ! KEvert* = KE net vertical advection (should be zero) + ! KEhoriz* = KE net horizonatl advection (should be zero) + ! IEvert* = IE net vertical advection (should be zero) + ! IEhoriz* = IE net horizonatl advection (should be zero) + ! + ! With leapfrog, energy equations are all exact except KE + ! (has an Err term that goes to zero as dt**2) + ! + ! Transfer terms: + ! T1 = -< dp/dn u, RT_v/p grad_p > KE<->IE: T1 + T2-T2_s = S1 + ! T2 = -< dp/dn u, grad_phi > KE<->PE: T2_s = S2 + ! T2_s = -< dp/dn u, grad_phis > + ! S1 = < Cp_star dp/dn , RT omega_p/Cp_star > + ! S2 = -< div (u dp/dn), phis > + + real (kind=real_kind) :: KEvert1(np,np) ! term from continuity equ + real (kind=real_kind) :: KEvert2(np,np) ! term from momentum equ + real (kind=real_kind) :: IEvert1(np,np) ! term from continuity equ + real (kind=real_kind) :: IEvert2(np,np) ! term from T equ + real (kind=real_kind) :: IEvert1_wet(np,np) ! wet term from continuity equ + real (kind=real_kind) :: IEvert2_wet(np,np) ! wet term from T equ + + real (kind=real_kind) :: KEhorz1(np,np) ! at time t + real (kind=real_kind) :: KEhorz2(np,np) ! after calling time_advance, these will be at time t-1 + real (kind=real_kind) :: IEhorz1(np,np) + real (kind=real_kind) :: IEhorz2(np,np) + real (kind=real_kind) :: IEhorz1_wet(np,np) + real (kind=real_kind) :: IEhorz2_wet(np,np) + + real (kind=real_kind) :: T1(np,np) + real (kind=real_kind) :: T2(np,np) + real (kind=real_kind) :: T2_s(np,np) + real (kind=real_kind) :: S1(np,np) + real (kind=real_kind) :: S1_wet(np,np) + real (kind=real_kind) :: S2(np,np) + + ! the KE conversion term and diffusion term + real (kind=real_kind) :: DIFF(np,np,2,nlev) ! net hypervis term + real (kind=real_kind) :: DIFFT(np,np,nlev) ! net hypervis term + real (kind=real_kind) :: CONV(np,np,2,nlev) ! dpdn u dot CONV = T1 + T2 +#endif + + ! the "4" timelevels represents data computed at: + ! 1 t-.5 + ! 2 t+.5 after dynamics + ! 3 t+.5 after forcing + ! 4 t+.5 after Robert + ! after calling TimeLevelUpdate, all times above decrease by 1.0 + + real (kind=real_kind) :: KEner(np,np,4) + real (kind=real_kind) :: PEner(np,np,4) + real (kind=real_kind) :: IEner(np,np,4) + real (kind=real_kind) :: IEner_wet(np,np,4) + real (kind=real_kind) :: Qvar(np,np,qsize_d,4) ! Q variance at half time levels + real (kind=real_kind) :: Qmass(np,np,qsize_d,4) ! Q mass at half time levels + real (kind=real_kind) :: Q1mass(np,np,qsize_d) ! Q mass at full time levels + + end type elem_accum_t + +contains +end module diff --git a/components/homme/src/pese/prim_advance_mod.F90 b/components/homme/src/pese/prim_advance_mod.F90 new file mode 100644 index 000000000000..3b71aca9af37 --- /dev/null +++ b/components/homme/src/pese/prim_advance_mod.F90 @@ -0,0 +1,335 @@ +! +! PESE Dynamics: Primtive Equations, Vertical Spectral Elements +!_______________________________________________________________________ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +module prim_advance_mod + + use bndry_mod, only: bndry_exchangev + use control_mod, only: qsplit, prescribed_wind, use_moisture + use derivative_mod, only: derivative_t, divergence_sphere, gradient_sphere + use dimensions_mod, only: np, nlev, nlevp, qsize + use edge_mod, only: initEdgeBuffer + use edgetype_mod, only: edgebuffer_t + use element_mod, only: element_t + use element_state, only: elem_state_t, derived_state_t + use element_ops, only: pack_edge_data, unpack_edge_data, apply_map, apply_vertical_dss, display_max_and_min + use hybrid_mod, only: hybrid_t + use hybvcoord_mod, only: hvcoord_t + use kinds, only: rl => real_kind, real_kind + use parallel_mod, only: abortmp, parallel_t + use physical_constants, only : cp, cpwater_vapor, Rgas, kappa, p0, g, Rwater_vapor + use time_mod, only: timeLevel_t, timelevel_update, timelevel_qdp, nsplit + use vertical_se, only: npv, solver_args, pack_solver_args, eta_derivative, advection2,& + self_advection2, eta_integral_from_n,eta_integral_from_1, make_vertical_mesh + use viscosity_mod, only: apply_hyperviscosity + +#ifndef CAM + use test_mod, only: set_test_prescribed_wind +#endif + + implicit none + + integer,parameter :: n_rhs = 1 + 3*nlev ! ps + T + u + v ! num levels in rhs and edge-buffer + logical,parameter :: verbose = .false. ! verbose output flag + integer :: count = 1 + type (edgebuffer_t) :: edge_buffer + +contains + + !_____________________________________________________________________ + subroutine prim_advance_init(par, elem, integration) + implicit none + + type (parallel_t), intent(in) :: par + type (element_t), intent(inout), target :: elem(:) + character(len=*) , intent(in) :: integration + integer :: i, ie + + ! allocate space for edge-data exchange + call initEdgeBuffer(par,edge_buffer,elem,n_rhs) + + end subroutine prim_advance_init + + !_____________________________________________________________________ + subroutine prim_advance_init2(elem, nets, nete, hybrid, hvcoord) + + type (element_t), intent(inout), target :: elem(:) ! array of element_t structures + integer, intent(in) :: nets,nete ! start and end element indices + type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct + type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coord data struct + + if (hybrid%masterthread) print *,"initializing PESE dynamics solver" + + ! initialize vertical operators and coordinates + call make_vertical_mesh(hybrid, hvcoord) + + end subroutine + + !_____________________________________________________________________ + subroutine prim_advance_exp(elem,deriv,hvcoord,hybrid,dt,tl,nets,nete,compute_diagnostics) + + type (element_t), intent(inout), target :: elem(:) + type (derivative_t),intent(in) :: deriv + type (hvcoord_t) :: hvcoord + type (hybrid_t), intent(in) :: hybrid + real (rl), intent(in) :: dt + type (TimeLevel_t), intent(in) :: tl + integer, intent(in) :: nets, nete + logical, intent(in) :: compute_diagnostics + + integer :: ie, t, q,k,i,j,n, qn0 + integer :: nm1,n0,np1,nstep + real (rl) :: eta_ave_w + + qn0 = 1 + + nm1 = tl%nm1 + n0 = tl%n0 + np1 = tl%np1 + + nstep = tl%nstep + eta_ave_w = 1d0/qsplit ! set q subcycle time averaging weight + +#ifndef CAM + ! if test uses prescribed wind, set dynamic variables analytically and return + if (prescribed_wind ==1 ) then + call set_test_prescribed_wind(elem,deriv,hybrid,hvcoord,dt,tl,nets,nete) + return + endif +#endif + + ! integrate dynamics in time using RK3 + + call compute_and_apply_rhs(np1,n0,n0,qn0,dt/3,elem,hvcoord,hybrid,deriv,nets,nete,compute_diagnostics,0d0) + ! u2 = u0 + dt/2 RHS(u1) + call compute_and_apply_rhs(np1,n0,np1,qn0,dt/2,elem,hvcoord,hybrid,deriv,nets,nete,.false.,0d0) + ! u3 = u0 + dt RHS(u2) + call compute_and_apply_rhs(np1,n0,np1,qn0,dt,elem,hvcoord,hybrid,deriv,nets,nete,.false.,eta_ave_w) + + ! apply viscosity or hyperviscosity to u,v,T,ps + call apply_hyperviscosity(elem,edge_buffer,hvcoord,hybrid,deriv,np1,nets,nete,dt,eta_ave_w) + + end subroutine + + !_____________________________________________________________________ + subroutine compute_and_apply_rhs(np1,nm1,n0,qn0,dt2,elem,hvcoord,hybrid,deriv,nets,nete,compute_diagnostics,eta_ave_w) + + integer, intent(in) :: np1,nm1,n0 ! time indices + integer, intent(in) :: qn0 ! time level used for virtual temperature + real*8, intent(in) :: dt2 + type (element_t), intent(inout), target :: elem(:) ! array of element_t structures + type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coord data struct + type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct + integer, intent(in) :: nets,nete ! start and end element indices + type (derivative_t), intent(in) :: deriv ! horizontal derivative data struct + logical, intent(in) :: compute_diagnostics ! TODO: enable energy diagnostics + real (rl), intent(in) :: eta_ave_w ! TODO: enable qsplit + + type(solver_args) :: a ! solver arguments + integer :: i,k,ie + + count=count+1 + a = pack_solver_args(np1,nm1,n0,qn0,dt2,hvcoord,hybrid,deriv,nets,nete,compute_diagnostics,eta_ave_w) + + do ie=nets,nete + if (verbose) call display_max_and_min(elem(ie),hybrid,ie,n0,count) + call apply_rhs(elem(ie),ie, a) + call apply_vertical_dss(elem(ie),np1) ! perform vertical direct stiffness summation + call apply_map(elem(ie)%spheremp,elem(ie),np1) ! apply sphere-map + call pack_edge_data(edge_buffer,elem(ie),np1,ie) ! pack edge data in buffer + enddo + + call bndry_exchangeV(hybrid, edge_buffer) ! exchange edge-data, perform dss + + do ie=nets,nete + call unpack_edge_data(edge_buffer,elem(ie),np1,ie) ! unpack edge data from buffer + call apply_map(elem(ie)%rspheremp, elem(ie), np1) ! apply inverse sphere-map + enddo + + end subroutine + + !_____________________________________________________________________ + subroutine apply_rhs(e,ie,a) + + type (element_t), intent(inout), target :: e ! element to operate upon + integer, intent(in) :: ie ! element index + type(solver_args),intent(inout) :: a ! solver arguments + + real (rl), dimension(np,np,nlev) :: u,v,T,p,phi,Tv,qv,alpha ! local copies of 3d fields + real (rl), dimension(np,np) :: ps ! local copies of 2d fields + real (rl), dimension(np,np,nlev) :: dp_deta, dT_deta ! vertical hydrostatic pressure gradient + real (rl), dimension(np,np,nlev) :: du_deta, dv_deta ! vertical velocity gradients + real (rl), dimension(np,np,2,nlev):: hflux ! horizontal mass flux + real (rl), dimension(np,np) :: zeros = 0 ! square matrix of zeros + real (rl), dimension(np,np,2,nlev):: grad_p ! gradient of total pressure + real (rl), dimension(np,np,2,nlev):: grad_phi ! gradient of geopotential height + real (rl), dimension(np,np,nlev) :: eta_flux ! vertical mass flux + real (rl), dimension(np,np,nlev) :: eta_dot ! vertical mass flux + real (rl), dimension(np,np,nlev) :: div_hflux ! flux of mass out of the column + real (rl), dimension(np,np,nlev) :: column_flux ! total mass flux into overlying column + real (rl), dimension(np,np,nlev) :: omega ! total time deriv of hydrostatic-pressure + real (rl), dimension(np,np,2,nlev):: f_corilois ! coriolis force + real (rl), dimension(np,np,nlev) :: ddt_T,ddt_u,ddt_v ! total time derivatives + real (rl), dimension(np,np,nlev) :: adv_T ! advective terms + real (rl), dimension(np,np,2,nlev):: adv_v ! advective terms 2d + real (rl), dimension(np,np,nlev) :: rhs_T,rhs_u,rhs_v ! partial time derivatives + real (rl), dimension(np,np) :: rhs_ps + real (rl), dimension(np,np,nlev) :: CpStar + type (elem_state_t), pointer :: s ! pointer to element state variables + + integer :: i,j,k ! loop indices + + ! make local copies of variables for readability + s => e%state + u = s%v (:,:,1,:, a%n0) + v = s%v (:,:,2,:, a%n0) + T = s%T (:,:,:, a%n0) + ps = s%ps_v(:,:, a%n0) + qv = s%Q (:,:,:,1) ! water vapor mixing ratio + + ! get hydrostatic-pressure at eta levels + forall(k=1:nlev) p(:,:,k)= a%hvcoord%hyam(k)*p0 + a%hvcoord%hybm(k)*ps + + ! get gradient of hydrostatic pressure + do k=1,nlev; grad_p (:,:,:,k) = gradient_sphere(p (:,:,k), a%deriv, e%Dinv); enddo + + if (.not. use_moisture ) then + Tv = T + CpStar = Cp + else + ! get virtual temperature (TODO) + Tv = T *(1.0_rl + (Rwater_vapor/Rgas - 1.0_rl)*qv) + CpStar = Cp*(1.0_rl + (Cpwater_vapor/Cp - 1.0_rl)*qv) + endif + + ! get moist density from virtual temperature and pressure + alpha = rgas*Tv/p + + ! get vertical derivatives + dp_deta = eta_derivative(p) + du_deta = eta_derivative(u) + dv_deta = eta_derivative(v) + dT_deta = eta_derivative(T) + + ! get geopotential-height by integrating hydrostatic balance from bottom + phi = eta_integral_from_n( -dp_deta*alpha, s%phis ) + + ! get horizontal gradient of geopotential-height + do k=1,nlev; grad_phi(:,:,:,k) = gradient_sphere(phi(:,:,k),a%deriv,e%Dinv); enddo + + ! get horizontal mass-flux + hflux(:,:,1,:) = dp_deta*u + hflux(:,:,2,:) = dp_deta*v + + ! get divergence of horizontal mass flux + do k=1,nlev; + div_hflux(:,:,k)= divergence_sphere(hflux(:,:,:,k), a%deriv, e); + enddo + + ! get net mass-flux out of column, integrated from the top + column_flux = eta_integral_from_1( div_hflux, zeros) + + ! get vertical mass-flux + forall(k=1:nlev) eta_flux(:,:,k) = a%hvcoord%hybm(k)*column_flux(:,:,nlev)-column_flux(:,:,k) + eta_flux(:,:,1 ) = 0.0d0 ! enforce no flux at top + eta_flux(:,:,nlev) = 0.0d0 ! enforce no flux at bottom + + ! get vertical velocity from vertical flux + eta_dot = eta_flux/dp_deta + + ! get time derivative of hydrostatic-pressure + omega = u*grad_p(:,:,1,:)+v*grad_p(:,:,2,:)-column_flux(:,:,:) + + ! get coriolis force + do k=1,nlev + f_corilois(:,:,1,k)= -e%fcor * v(:,:,k) + f_corilois(:,:,2,k)= +e%fcor * u(:,:,k) + enddo + + ! get total time-derivs + ddt_u = -grad_p(:,:,1,:)*alpha -grad_phi(:,:,1,:) -f_corilois(:,:,1,:) + ddt_v = -grad_p(:,:,2,:)*alpha -grad_phi(:,:,2,:) -f_corilois(:,:,2,:) + ddt_T = omega*alpha/CpStar + + ! get advection terms + adv_T = advection2(T,dT_deta,u,v,eta_dot,a,e) + adv_v = self_advection2(u,v,du_deta,dv_deta,eta_dot,a,e) + + ! get partial-time derivative of each prognostic variable + rhs_ps = -column_flux(:,:,nlev) + rhs_T = -adv_T + ddt_T + rhs_u = -adv_v(:,:,1,:) + ddt_u + rhs_v = -adv_v(:,:,2,:) + ddt_v + + ! apply right-hand-side to prognostics + s%ps_v(:,:, a%np1) = s%ps_v(:,: ,a%nm1) + a%dt * rhs_ps + s%T (:,:,:, a%np1) = s%T (:,:,: ,a%nm1) + a%dt * rhs_T + s%v (:,:,1,:,a%np1) = s%v (:,:,1,:,a%nm1) + a%dt * rhs_u + s%v (:,:,2,:,a%np1) = s%v (:,:,2,:,a%nm1) + a%dt * rhs_v + + ! compute derived quantities for output + !s%dp3d = 1 + !e%derived%omega_p = omega/p + e%derived%phi = phi + e%derived%omega = omega + + end subroutine + + !_____________________________________________________________________ + subroutine applyCAMforcing(elem,hvcoord,np1,np1_qdp,dt_q,nets,nete) + + use physical_constants, only: Cp + + ! implicit none + type (element_t), intent(inout) :: elem(:) + real (kind=real_kind), intent(in) :: dt_q + type (hvcoord_t), intent(in) :: hvcoord + integer, intent(in) :: np1,nets,nete,np1_qdp + + ! local + integer :: ie,q + real(rl) :: dp(np,np,nlev) + + do ie=nets,nete + + ! apply forcing to dynamics + elem(ie)%state%T(:,:,:, np1) = elem(ie)%state%T(:,:,:,np1) + dt_q*elem(ie)%derived%FT + elem(ie)%state%v(:,:,:,:,np1) = elem(ie)%state%v(:,:,:,:,np1) + dt_q*elem(ie)%derived%FM + +! TODO: enable tracer forcing for PESE + + ! apply forcing to tracer mass + !dp = elem(ie)%state%dp3d(:,:,:,np1) + !do q=1,qsize + ! elem(ie)%state%Qdp(:,:,:,q,np1_qdp) = elem(ie)%state%Qdp(:,:,:,q,np1_qdp)+ dt_q * elem(ie)%derived%FQ(:,:,:,q) + ! elem(ie)%state%Q(:,:,:,q) = elem(ie)%state%Qdp(:,:,:,q,np1_qdp)/dp + !enddo + enddo + end subroutine applyCAMforcing + + !___________________________________________________________________ + subroutine applyCAMforcing_dynamics(elem,hvcoord,np1,dt_q,nets,nete) + + use hybvcoord_mod, only: hvcoord_t + + implicit none + type (element_t) , intent(inout) :: elem(:) + real (kind=real_kind), intent(in) :: dt_q + type (hvcoord_t), intent(in) :: hvcoord + integer, intent(in) :: np1,nets,nete + + integer :: ie,q + real (kind=real_kind) :: v1,dp + + do ie=nets,nete + elem(ie)%state%T(:,:,:,np1) = elem(ie)%state%T(:,:,:,np1) + dt_q*elem(ie)%derived%FT(:,:,:) + elem(ie)%state%v(:,:,:,:,np1) = elem(ie)%state%v(:,:,:,:,np1) + dt_q*elem(ie)%derived%FM(:,:,:,:) + enddo + + end subroutine applyCAMforcing_dynamics + +end module prim_advance_mod + diff --git a/components/homme/src/pese/prim_advection_mod.F90 b/components/homme/src/pese/prim_advection_mod.F90 new file mode 100644 index 000000000000..a21559dbd441 --- /dev/null +++ b/components/homme/src/pese/prim_advection_mod.F90 @@ -0,0 +1,283 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +module prim_advection_mod + + use prim_advection_base, only: prim_advec_init1_rk2, prim_advec_tracers_remap_rk2 + + use bndry_mod, only: bndry_exchangev + use derivative_mod, only: derivative_t, gradient_sphere + use dimensions_mod, only: np, nlev, nlevp, qsize,qsize_d, nelemd + use edge_mod, only: initEdgeBuffer, edgevpack, edgevunpack + use edgetype_mod, only: edgebuffer_t + use element_mod, only: element_t + use element_state, only: elem_state_t, derived_state_t + use hybrid_mod, only: hybrid_t + use hybvcoord_mod, only: hvcoord_t + use parallel_mod, only: parallel_t + use perf_mod, only: t_startf, t_stopf + use physical_constants, only : p0 + use kinds, only: rl => real_kind, real_kind, dd => longdouble_kind + use time_mod, only: timelevel_t, timelevel_update, timelevel_qdp, nsplit + use vertical_se, only: eta_derivative, vertical_dss + + implicit none + + type(EdgeBuffer_t) :: edge_buffer ! buffer for edge-data exchange + + ! q(np,np,nlev,qsize,nstage,nelemd) + real(rl), allocatable :: q(:,:,:,:,:,:) ! tracer sub-stage storage + + ! rhs(np,np,nlev,qsize,nelemd) + real(rl), allocatable :: rhs(:,:,:,:,:) ! right-hand-side storage + + type (derivative_t) :: deriv + type (hybrid_t) :: hybrid ! mpi/omp data struct + type (hvcoord_t) :: hv ! hybrid vertical coord data struct + integer :: nets,nete ! start and end element indices + + CONTAINS + + !_____________________________________________________________________ + subroutine prim_advect_tracers(elem, deriv_, hybrid_, nets_, nete_, hvcoord_, dt, tl) + + type (element_t), intent(inout), target :: elem(:) ! array of element_t structures + type (derivative_t), intent(in) :: deriv_ + type (hybrid_t), intent(in) :: hybrid_ ! mpi/omp data struct + integer, intent(in) :: nets_,nete_ ! start and end element indices + type (hvcoord_t), intent(in) :: hvcoord_ ! hybrid vertical coord data struct + real(rl), intent(in) :: dt ! tracer timestep + type (TimeLevel_t), intent(inout) :: tl ! time-level + + real(rl) :: dp(np,np,nlev), dp_dn + + logical :: initialized = .false. + integer :: ie,iq,k + integer :: n0_qdp, np1_qdp + + call t_startf("advect_tracers") + + ! store arguments at module level + deriv = deriv_; hybrid=hybrid_; nets=nets_; nete=nete_; hv=hvcoord_ + + ! allocate edge-buff and rhs-vector based on qsize + if(.not. initialized) then; initialized = .true. + call initEdgeBuffer(hybrid%par, edge_buffer, elem, qsize*nlev) ! allocate edge-data buffer + allocate( rhs(np,np,nlev,qsize,nelemd)) ! allocate memmory for euler sub-steps + endif + + ! advance using 3 stage SSP Runge-Kutta from [Spiteri and Ruuth 2002] + call RK_SSP3(elem,dt,tl) + !call euler_forward(elem,dt,tl) + + ! compute Qdp (it is used by many routines in preqx) + do ie=nets,nete + + dp = elem(ie)%state%dp3d(:,:,:,tl%n0) + + do iq=1,qsize + elem(ie)%state%Qdp(:,:,:,iq,1) = elem(ie)%state%Q(:,:,:,iq)*dp + elem(ie)%state%Qdp(:,:,:,iq,2) = elem(ie)%state%Q(:,:,:,iq)*dp + enddo + enddo + + call t_stopf("advect_tracers") + + end subroutine + + !_____________________________________________________________________ + subroutine euler_forward(elem,dt,tl) + + ! simplest time-stepping solver: take one explicit step forward + + type (element_t), intent(inout), target :: elem(:) ! array of element_t structures + real*8, intent(in) :: dt ! tracer timestep + type (TimeLevel_t), intent(in) :: tl ! time-level + logical :: initialized = .false. + integer :: ie + + ! allocate space to store sub-stages + if(.not. initialized) then; initialized=.true. + allocate( q(np,np,nlev,qsize,1,nelemd) ); q=0.0d0 + endif + + do ie=nets,nete + q(:,:,:,:,1,ie) = elem(ie)%state%Q(:,:,:,1:qsize) + enddo + + call euler_step(elem, 1,1,1 ,dt, tl) ! q1 = q1 + dt RHS[q1] + + do ie=nets,nete ! q = q1 + elem(ie)%state%Q(:,:,:,1:qsize) = 1.0*q(:,:,:,:,1,ie) + enddo + + end subroutine + !_____________________________________________________________________ + subroutine RK3(elem,dt,tl) + + type (element_t), intent(inout), target :: elem(:) ! array of element_t structures + real*8, intent(in) :: dt ! tracer timestep + type (TimeLevel_t), intent(in) :: tl ! time-level + logical :: initialized = .false. + integer :: ie + + ! allocate space to store sub-stages + if(.not. initialized) then; initialized=.true. + allocate( q(np,np,nlev,qsize,4,nelemd) ); q=0 + endif + + ! copy tracer data into sub-stage 1 + do ie=nets,nete + q(:,:,:,:,1,ie) = elem(ie)%state%Q(:,:,:,1:qsize) + enddo + + ! RK 3 stage + call euler_step(elem, 1,1,2 ,dt/3.0d0, tl) ! q2 = q1 + dt/3 RHS[q1] + call euler_step(elem, 1,2,3 ,dt/2.0d0, tl) ! q3 = q1 + dt/2 RHS[q2] + call euler_step(elem, 1,3,4, dt/1.0d0, tl) ! q4 = q1 + dt RHS[q3] + + do ie=nets,nete ! q = q4 + elem(ie)%state%Q(:,:,:,1:qsize) = 1.0*q(:,:,:,:,4,ie) + enddo + + end subroutine + + !_____________________________________________________________________ + subroutine RK_SSP3(elem,dt,tl) + + type (element_t), intent(inout), target :: elem(:) ! array of element_t structures + real*8, intent(in) :: dt ! tracer timestep + type (TimeLevel_t), intent(in) :: tl ! time-level + logical :: initialized = .false. + integer :: ie + + ! allocate space to store sub-stages + if(.not. initialized) then; initialized=.true. + allocate( q(np,np,nlev,qsize,2,nelemd) ); q=0 + endif + + ! copy tracer data into sub-stage 1 + do ie=nets,nete + q(:,:,:,:,1,ie) = elem(ie)%state%Q(:,:,:,1:qsize) + enddo + + ! RK-SSP 3 stage from Spiteri and Ruuth 2002 + call euler_step(elem, 1,1,2 ,dt/2.0d0, tl) ! q2 = q1 + dt/2 RHS[q1] + call euler_step(elem, 2,2,2 ,dt/2.0d0, tl) ! q2 = q2 + dt/2 RHS[q2] + call euler_step(elem, 2,2,2 ,dt/2.0d0, tl) ! q2 = q2 + dt/2 RHS[q2] + + do ie=nets,nete; ! q = 1/3*q + 2/3 q2 + elem(ie)%state%Q(:,:,:,1:qsize) = ( q(:,:,:,:,1,ie)+2.0d0*q(:,:,:,:,2,ie) )/3.0d0 + enddo + + end subroutine + + !_____________________________________________________________________ + subroutine euler_step(elem,n0,n1,n2,dt,tl) + + ! solve dq/dt = -u dot grad q -eta_dot dq/dn + + type (element_t), intent(inout), target :: elem(:) ! element array + integer, intent(in) :: n0,n1,n2 ! substep indices: q(n2) = q(n0) + dt * RHS(q(n1)) + real*8, intent(in) :: dt ! euler forward timestep + type (TimeLevel_t), intent(in) :: tl ! time-level + + integer :: k,ie,iq ! loop indices + + real(rl) :: rhs(np,np,nlev,qsize) + real(rl), dimension(np,np) :: ps + real(rl), dimension(np,np,nlev) :: p, eta_dot, vflux + real(rl), dimension(np,np,nlev) :: dp_dn, div_hflux, ddn_vflux + real(rl), dimension(np,np,2,nlev):: hflux, V + real(rl), dimension(np,np,2,nlev):: grad_q + real(rl), dimension(np,np,nlev) :: dq_dn + real(rl), dimension(np,np,nlev) :: h_adv, v_adv + + do ie=nets,nete + + V = elem(ie)%state%v(:,:,:,:,tl%np1) + ps = elem(ie)%state%ps_v(:,:,tl%np1) + + ! get eta_dot from vertical mass flux + forall(k=1:nlev) p(:,:,k)= hv%hyam(k)*p0 + hv%hybm(k)*ps + dp_dn = eta_derivative(p) + eta_dot = elem(ie)%derived%eta_dot_dpdn(:,:,2:nlevp) / dp_dn + + ! ensure no flux at top and bottom + eta_dot(:,:,1) =0.0 + eta_dot(:,:,nlev) =0.0 + + do iq=1,qsize + + ! get horizontal advection term + do k=1,nlev; grad_q(:,:,:,k) = gradient_sphere( q(:,:,k,iq,n1,ie),deriv, elem(ie)%Dinv); enddo + h_adv = V(:,:,1,:)*grad_q(:,:,1,:)+V(:,:,2,:)*grad_q(:,:,2,:) + + ! get vertical advection term + dq_dn = eta_derivative( q(:,:,:,iq,n1,ie) ) + v_adv = eta_dot*dq_dn + + rhs(:,:,:,iq) =-h_adv -v_adv + enddo + + ! compute euler-forward step + q(:,:,:,:,n2,ie) = q(:,:,:,:,n0,ie) + dt * rhs + + do iq=1,qsize + + ! apply vertical dss + call vertical_dss(q(:,:,:,iq,n2,ie)) + + do k=1,nlev + ! apply mass matrix + q(:,:,k,iq,n2,ie) = q(:,:,k,iq,n2,ie)*elem(ie)%spheremp + enddo + enddo + + ! pack data into edge buffer + call edgeVpack(edge_buffer, q(:,:,:,:,n2,ie), qsize*nlev, 0, ie); + enddo + + ! exchange edge data and perform direct stiffness summation + call bndry_exchangeV(hybrid, edge_buffer) + + do ie=nets,nete + + ! unpack data from edge buffer + call edgeVunpack(edge_buffer, q(:,:,:,:,n2,ie), qsize*nlev, 0, ie) + + do iq=1,qsize + do k=1,nlev + ! apply inverse mass matrix + q(:,:,k,iq,n2,ie) = q(:,:,k,iq,n2,ie)*elem(ie)%rspheremp + enddo + enddo + enddo + + end subroutine + + !_____________________________________________________________________ + subroutine prim_advec_init1(par, elem, n_domains) + + type(parallel_t) :: par + integer, intent(in) :: n_domains + type (element_t) :: elem(:) + + call prim_advec_init1_rk2(par, elem, n_domains) + + end subroutine prim_advec_init1 + + !_____________________________________________________________________ + subroutine prim_advec_init2(elem,hvcoord,hybrid) + + use element_mod , only : element_t + use hybvcoord_mod , only : hvcoord_t + implicit none + type(element_t) , intent(in) :: elem(:) + type(hvcoord_t) , intent(in) :: hvcoord + type (hybrid_t) , intent(in) :: hybrid + end subroutine prim_advec_init2 + + +end module prim_advection_mod diff --git a/components/homme/src/pese/prim_driver_mod.F90 b/components/homme/src/pese/prim_driver_mod.F90 new file mode 100644 index 000000000000..73a0c68e77c4 --- /dev/null +++ b/components/homme/src/pese/prim_driver_mod.F90 @@ -0,0 +1,264 @@ +module prim_driver_mod + + use prim_driver_base, only:& + deriv1,& + prim_init1,& + prim_init2 ,& + prim_finalize,& + smooth_topo_datasets + + use control_mod, only: energy_fixer, statefreq, ftype, qsplit, rsplit + use dimensions_mod, only: np, nlev, nlevp, nelem, nelemd, qsize + use element_mod, only: element_t, allocate_element_desc + use element_state, only: timelevels + use hybrid_mod, only: hybrid_t + use hybvcoord_mod, only: hvcoord_t + use kinds, only: real_kind, iulog + use parallel_mod, only: abortmp + use perf_mod, only: t_startf, t_stopf + use prim_advance_mod, only: compute_and_apply_rhs + use prim_advection_mod, only: prim_advect_tracers + use prim_state_mod, only: prim_printstate, prim_diag_scalars, prim_energy_halftimes + use time_mod, only: timeLevel_t, timelevel_update, timelevel_qdp + +#ifndef CAM + use test_mod, only: compute_test_forcing +#endif + + implicit none + contains + +! customized: prim_run_subcycle , prim_step + +!_____________________________________________________________________ + subroutine apply_forcing(elem,hybrid,hvcoord,tl,dt_remap,nets,nete, compute_diagnostics, compute_energy) + + ! apply forcing from CAM or HOMME stand-alone tests + + use prim_advance_mod, only: ApplyCAMForcing, ApplyCAMForcing_dynamics + + implicit none + type(element_t), intent(inout) :: elem(:) + type(hybrid_t), intent(in) :: hybrid + type(hvcoord_t), intent(in) :: hvcoord + type (TimeLevel_t), intent(inout) :: tl + real(kind=real_kind), intent(in) :: dt_remap + integer, intent(in) :: nets,nete + logical, intent(in) :: compute_diagnostics + logical, intent(in) :: compute_energy + + integer :: n0_qdp + + call TimeLevel_Qdp(tl, qsplit, n0_qdp) + +#ifndef CAM + ! Get HOMME test case forcing + call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete) +#endif + + ! Apply CAM Physics forcing + + ! ftype= 2: Q was adjusted by physics, but apply u,T forcing here + ! ftype= 1: forcing was applied time-split in CAM coupling layer + ! ftype= 0: apply all forcing here + ! ftype=-1: do not apply forcing + + call TimeLevel_Qdp(tl, qsplit, n0_qdp) + + if (ftype==0) then + call t_startf("ApplyCAMForcing") + call ApplyCAMForcing(elem, hvcoord,tl%n0,n0_qdp, dt_remap,nets,nete) + call t_stopf("ApplyCAMForcing") + + elseif (ftype==2) then + call t_startf("ApplyCAMForcing_dynamics") + call ApplyCAMForcing_dynamics(elem, hvcoord,tl%n0,dt_remap,nets,nete) + call t_stopf("ApplyCAMForcing_dynamics") + endif + + ! E(1) Energy after CAM forcing + if (compute_energy) then + call t_startf("prim_energy_halftimes") + call prim_energy_halftimes(elem,hvcoord,tl,1,.true.,nets,nete) + call t_stopf("prim_energy_halftimes") + endif + + ! qmass and variance, using Q(n0),Qdp(n0) + if (compute_diagnostics) then + call t_startf("prim_diag_scalars") + call prim_diag_scalars(elem,hvcoord,tl,1,.true.,nets,nete) + call t_stopf("prim_diag_scalars") + endif + + end subroutine + +!_______________________________________________________________________ +subroutine prim_run_subcycle(elem,hybrid,nets,nete,dt,tl,hvcoord,nsubstep) + + type (element_t) , intent(inout) :: elem(:) + type (hybrid_t), intent(in) :: hybrid ! distributed parallel structure (shared) + type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coordinate struct + integer, intent(in) :: nets ! starting thread element number (private) + integer, intent(in) :: nete ! ending thread element number (private) + real(kind=real_kind), intent(in) :: dt ! "timestep dependent" timestep + type (TimeLevel_t), intent(inout) :: tl + integer, intent(in) :: nsubstep ! nsubstep = 1 .. nsplit + + logical :: compute_diagnostics, compute_energy + real(kind=real_kind) :: dt_q, dt_remap + integer :: n0_qdp,np1_qdp,r,nstep_end + + + ! compute dt_q, dt_remp, and nstep_end + call get_subcycle_stepsize(dt_q,dt_remap,nstep_end,dt,tl) + + ! enable/disable diagnositics for this time-step + call set_diagnostic_state(compute_diagnostics,compute_energy,tl,nstep_end) + + ! compute scalar diagnostics if currently active + if (compute_diagnostics) then + call t_startf("prim_diag_scalars") + call prim_diag_scalars(elem,hvcoord,tl,3,.true.,nets,nete) + call t_stopf("prim_diag_scalars") + + call t_startf("prim_energy_halftimes") + call prim_energy_halftimes(elem,hvcoord,tl,3,.true.,nets,nete) + call t_stopf("prim_energy_halftimes") + endif + + ! apply cam forcing or stand-alone test forcing to state variables + call apply_forcing(elem,hybrid,hvcoord,tl,dt_remap,nets,nete, compute_diagnostics, compute_energy) + + ! get E(1): energy after CAM forcing + if (compute_energy) call prim_energy_halftimes(elem,hvcoord,tl,1,.true.,nets,nete) + + ! get qmass and variance, using Q(n0),Qdp(n0) + if (compute_diagnostics) call prim_diag_scalars(elem,hvcoord,tl,1,.true.,nets,nete) + + ! loop over rsplit vertically lagrangian timesteps + call prim_step(elem, hybrid,nets,nete, dt, tl, hvcoord,compute_diagnostics,1) + do r=2,rsplit + call TimeLevel_update(tl,"leapfrog") + call prim_step(elem, hybrid,nets,nete,dt,tl,hvcoord,.false.,r) + enddo + + ! compute scalar diagnostics if currently active + if (compute_diagnostics) then + ! E(1): Energy after CAM forcing + call t_startf("prim_energy_halftimes") + call prim_energy_halftimes(elem,hvcoord,tl,1,.true.,nets,nete) + call t_stopf("prim_energy_halftimes") + + ! qmass and variance, using Q(n0),Qdp(n0) + call t_startf("prim_diag_scalars") + call prim_diag_scalars(elem,hvcoord,tl,1,.true.,nets,nete) + call t_stopf("prim_diag_scalars") + endif + + ! update dynamics time level pointers + call TimeLevel_update(tl,"leapfrog") + + ! print some diagnostic information + if (compute_diagnostics) call prim_printstate(elem, tl, hybrid,hvcoord,nets,nete) + + end subroutine prim_run_subcycle + +!_______________________________________________________________________ +subroutine prim_step(elem,hybrid,nets,nete,dt,tl,hvcoord,compute_diagnostics,rstep) + + use control_mod, only: statefreq, integration, ftype, qsplit, nu_p, rsplit + use control_mod, only: use_semi_lagrange_transport + use hybvcoord_mod, only: hvcoord_t + use parallel_mod, only: abortmp + use prim_advance_mod, only: prim_advance_exp + use time_mod, only: timelevel_t, timelevel_update + + type(element_t), intent(inout) :: elem(:) + type(hybrid_t), intent(in) :: hybrid ! distributed parallel structure (shared) + type(hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct + integer, intent(in) :: nets ! starting thread element number (private) + integer, intent(in) :: nete ! ending thread element number (private) + real(kind=real_kind), intent(in) :: dt ! "timestep dependent" timestep + type(TimeLevel_t), intent(inout) :: tl + logical, intent(in) :: compute_diagnostics + integer, intent(in) :: rstep ! vertical remap subcycling step + + real(kind=real_kind) :: st, st1, dp, dt_q + integer :: ie, t, q,k,i,j,n, n_Q + real (kind=real_kind) :: maxcflx, maxcfly + real (kind=real_kind) :: dp_np1(np,np) + + dt_q = dt*qsplit + + ! clear derived quantities + + call t_startf("prim_step_init") + dt_q = dt*qsplit + + do ie=nets,nete + if (nu_p>0) then + elem(ie)%derived%dpdiss_ave = 0 + elem(ie)%derived%dpdiss_biharmonic = 0 + endif + enddo + call t_stopf("prim_step_init") + + ! take qsplit dynamics steps, followed by one tracer step + do n=1,qsplit + call prim_advance_exp(elem, deriv1, hvcoord, hybrid, dt, tl, nets, nete, compute_diagnostics) + if(n 0) call prim_advect_tracers(elem, deriv1, hybrid, nets, nete, hvcoord, dt_q, tl) + + end subroutine prim_step + + !_____________________________________________________________________ + subroutine get_subcycle_stepsize(dt_q,dt_remap,nstep_end,dt,tl) + + ! compute timesteps for tracer transport and vertical remap + + use control_mod, only: qsplit, rsplit + + real(kind=real_kind), intent(out) :: dt_q, dt_remap + integer, intent(out) :: nstep_end + real(kind=real_kind), intent(in) :: dt + type (TimeLevel_t), intent(in) :: tl + + dt_q = dt*qsplit + dt_remap = dt_q + nstep_end = tl%nstep + qsplit + + if (rsplit>0) then + dt_remap = dt_q*rsplit ! rsplit=0 means use eulerian code, not vert. lagrange + nstep_end = tl%nstep + qsplit*rsplit ! nstep at end of this routine + endif + + end subroutine + + !_____________________________________________________________________ + subroutine set_diagnostic_state(compute_diagnostics,compute_energy,tl,nstep_end) + + ! enable or disable scalar and energy diagnostics for this timestep + + use control_mod, only: energy_fixer, disable_diagnostics + + logical, intent(out) :: compute_diagnostics, compute_energy + type (TimeLevel_t), intent(in) :: tl + integer, intent(in) :: nstep_end + + ! activate energy diagnostics if using an energy fixer + compute_energy = energy_fixer > 0 + + ! activate diagnostics periodically for display to stdout + compute_diagnostics = .false. + if (MODULO(nstep_end,statefreq)==0 .or. nstep_end==tl%nstep0) then + compute_diagnostics= .true. + compute_energy = .true. + endif + if(disable_diagnostics) compute_diagnostics= .false. + + end subroutine + +end module diff --git a/components/homme/src/pese/sl_advection.F90 b/components/homme/src/pese/sl_advection.F90 new file mode 100644 index 000000000000..c9ce3eac889f --- /dev/null +++ b/components/homme/src/pese/sl_advection.F90 @@ -0,0 +1,915 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + + +module sl_advection +! +! classic semi-lagrange advection +! with optional global iteration for pseudo-local mass conservation +! +! Author: James Overfelt 3/2015 +! + use kinds, only : real_kind + use dimensions_mod, only : nlev, nlevp, np, qsize + use physical_constants, only : rgas, Rwater_vapor, kappa, g, rearth, rrearth, cp + use derivative_mod, only : derivative_t, gradient_sphere, divergence_sphere + use element_mod, only : element_t + use hybvcoord_mod, only : hvcoord_t + use time_mod, only : TimeLevel_t, TimeLevel_Qdp + use control_mod, only : integration, test_case, hypervis_order, use_semi_lagrange_transport + use edge_mod, only : edgevpack, edgevunpack, initedgebuffer,& + initghostbuffer3D, ghostVpack_unoriented, ghostVunpack_unoriented + use edgetype_mod, only : EdgeDescriptor_t, EdgeBuffer_t, ghostbuffer3D_t + use hybrid_mod, only : hybrid_t + use bndry_mod, only : bndry_exchangev + use viscosity_mod, only : biharmonic_wk_scalar, neighbor_minmax, & + neighbor_minmax_start, neighbor_minmax_finish + use perf_mod, only : t_startf, t_stopf, t_barrierf ! _EXTERNAL + use parallel_mod, only : abortmp, parallel_t + + implicit none + + private + save + + type (ghostBuffer3D_t) :: ghostbuf_tr + type (EdgeBuffer_t) :: edgeveloc + + public :: Prim_Advec_Tracers_remap_ALE, sl_init1 + +contains + + +!=================================================================================================! + subroutine sl_init1(par, elem, n_domains) + use interpolate_mod, only : interpolate_tracers_init + type(parallel_t) :: par + integer, intent(in) :: n_domains + type (element_t) :: elem(:) + + if (use_semi_lagrange_transport) then + call initEdgeBuffer(par,edgeveloc,elem,2*nlev) + call initghostbuffer3D(ghostbuf_tr,nlev*qsize,np) + call interpolate_tracers_init() + endif + + end subroutine + + +subroutine Prim_Advec_Tracers_remap_ALE( elem , deriv , hybrid , dt , tl , nets , nete ) + use coordinate_systems_mod, only : cartesian3D_t, cartesian2D_t + use dimensions_mod, only : max_neigh_edges + use bndry_mod, only : ghost_exchangevfull + use interpolate_mod, only : interpolate_tracers, minmax_tracers + use control_mod , only : qsplit + use global_norms_mod, only : wrap_repro_sum + use parallel_mod, only: global_shared_buf, global_shared_sum, syncmp + use control_mod, only : use_semi_lagrange_transport_local_conservation + + + + implicit none + type (element_t) , intent(inout) :: elem(:) + type (derivative_t) , intent(in ) :: deriv + type (hybrid_t) , intent(in ) :: hybrid + real(kind=real_kind) , intent(in ) :: dt + type (TimeLevel_t) , intent(in ) :: tl + integer , intent(in ) :: nets + integer , intent(in ) :: nete + + + type(cartesian3D_t) :: dep_points (np,np) + integer :: elem_indexes(np,np) + type(cartesian2D_t) :: para_coords (np,np) + real(kind=real_kind) :: Que (np,np,nlev,qsize,nets:nete) + real(kind=real_kind) :: Que_t (np,np,nlev,qsize,nets:nete) + real(kind=real_kind) :: minq (np,np,nlev,qsize,nets:nete) + real(kind=real_kind) :: maxq (np,np,nlev,qsize,nets:nete) + real(kind=real_kind) :: f (qsize) + real(kind=real_kind) :: g (qsize) + real(kind=real_kind) :: mass (nlev,qsize) + real(kind=real_kind) :: elem_mass (nlev,qsize,nets:nete) + real(kind=real_kind) :: rho (np,np,nlev, nets:nete) + + real(kind=real_kind) :: neigh_q (np,np,qsize,max_neigh_edges+1) + real(kind=real_kind) :: u (np,np,qsize) + + integer :: i,j,k,l,n,q,ie,kptr, n0_qdp, np1_qdp + integer :: num_neighbors + + call t_barrierf('Prim_Advec_Tracers_remap_ALE', hybrid%par%comm) + call t_startf('Prim_Advec_Tracers_remap_ALE') + + call TimeLevel_Qdp( tl, qsplit, n0_qdp, np1_qdp) + + ! compute displacements for departure grid + ! store in elem%derived%vstar + call ALE_RKdss (elem, nets, nete, hybrid, deriv, dt, tl) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! run ghost exchange to get global ID of all neighbors +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! + call t_startf('Prim_Advec_Tracers_remap_ALE_ghost_exchange') + do ie=nets,nete + kptr=0 + do k=1,nlev + do q=1,qsize + ! note: pack so that tracers per level are contiguous so we can unpack into + ! array neigh_q() + elem(ie)%state%Q(:,:,k,q) = elem(ie)%state%Qdp(:,:,k,q,n0_qdp) / elem(ie)%derived%dp(:,:,k) + call ghostVpack_unoriented(ghostbuf_tr, elem(ie)%state%Q(:,:,k,q),np,1,kptr,elem(ie)%desc) + kptr=kptr+1 + enddo + enddo + end do + + call t_startf('pat_remap_ale_gexchV') + call ghost_exchangeVfull(hybrid%par,hybrid%ithr,ghostbuf_tr) + call t_stopf('pat_remap_ale_gexchV') + + do ie=nets,nete + num_neighbors = elem(ie)%desc%actual_neigh_edges+1 + do k=1,nlev + + ! find departure points + call ALE_departure_from_gll (dep_points, elem(ie)%derived%vstar(:,:,:,k), elem(ie), dt) + + ! find element containing departure point + call ALE_elems_with_dep_points (elem_indexes, dep_points, num_neighbors, elem(ie)%desc%neigh_corners) + + ! compute the parametric points + call ALE_parametric_coords (para_coords, elem_indexes, dep_points, num_neighbors, elem(ie)%desc%neigh_corners) + + ! for each level k, unpack all tracer neighbor data on that level + kptr=(k-1)*qsize + neigh_q=0 + u(:,:,:) = elem(ie)%state%Q(:,:,k,1:qsize) + call ghostVunpack_unoriented (ghostbuf_tr, neigh_q, np, qsize, kptr, elem(ie)%desc, elem(ie)%GlobalId, u) + + do i=1,np + do j=1,np + ! interpolate tracers to deperature grid + call interpolate_tracers (para_coords(i,j), neigh_q(:,:,:,elem_indexes(i,j)),f) + elem(ie)%state%Q(i,j,k,:) = f +! call minmax_tracers (para_coords(i,j), neigh_q(:,:,:,elem_indexes(i,j)),f,g) + do q=1,qsize + f(q) = MINVAL(neigh_q(:,:,q,elem_indexes(i,j))) + g(q) = MAXVAL(neigh_q(:,:,q,elem_indexes(i,j))) + end do + minq(i,j,k,:,ie) = f + maxq(i,j,k,:,ie) = g + enddo + enddo + + end do + end do + + + call t_stopf('Prim_Advec_Tracers_remap_ALE_ghost_exchange') + ! compute original mass, at tl_1%n0 + elem_mass = 0 + do ie=nets,nete + n=0 + do k=1,nlev + do q=1,qsize + n=n+1 + global_shared_buf(ie,n) = 0 + do j=1,np + global_shared_buf(ie,n) = global_shared_buf(ie,n) + DOT_PRODUCT(elem(ie)%state%Qdp(:,j,k,q,n0_qdp),elem(ie)%spheremp(:,j)) + elem_mass(k,q,ie) = elem_mass(k,q,ie) + DOT_PRODUCT(elem(ie)%state%Qdp(:,j,k,q,n0_qdp),elem(ie)%spheremp(:,j)) + end do + end do + end do + end do + call wrap_repro_sum(nvars=n, comm=hybrid%par%comm) + n=0 + do k=1,nlev + do q=1,qsize + n=n+1 + mass(k,q) = global_shared_sum(n) + enddo + enddo + + + do ie=nets,nete + do k=1,nlev + rho(:,:,k,ie) = elem(ie)%spheremp(:,:)*elem(ie)%state%dp3d(:,:,k,tl%np1) + end do + end do + + do ie=nets,nete + Que_t(:,:,:,1:qsize,ie) = elem(ie)%state%Q(:,:,:,1:qsize) + end do + call t_startf('Prim_Advec_Tracers_remap_ALE_Cobra') + if (use_semi_lagrange_transport_local_conservation) then + call Cobra_Elem (Que, Que_t, rho, minq, maxq, elem_mass, hybrid, nets, nete) + else + call Cobra_SLBQP(Que, Que_t, rho, minq, maxq, mass, hybrid, nets, nete) + end if + call t_stopf('Prim_Advec_Tracers_remap_ALE_Cobra') + + do ie=nets,nete + elem(ie)%state%Q(:,:,:,1:qsize) = Que(:,:,:,1:qsize,ie) + end do + + + do ie=nets,nete + do k=1,nlev + do q=1,qsize + ! note: pack so that tracers per level are contiguous so we can unpack into + ! array neigh_q() + elem(ie)%state%Qdp(:,:,k,q,np1_qdp) = elem(ie)%state%Q(:,:,k,q) * elem(ie)%state%dp3d(:,:,k,tl%np1) + enddo + enddo + end do + + +! do ie=nets,nete +! do q = 1 , qsize +! do k = 1 , nlev ! Potential loop inversion (AAM) +! elem(ie)%state%Qdp(:,:,k,q,np1_qdp) = elem(ie)%spheremp(:,:)* elem(ie)%state%Qdp(:,:,k,q,np1_qdp) +! enddo +! enddo +! call edgeVpack(edgeAdv , elem(ie)%state%Qdp(:,:,:,:,np1_qdp) , nlev*qsize , 0 , elem(ie)%desc ) +! enddo +! call bndry_exchangeV( hybrid , edgeAdv ) +! do ie = nets , nete +! call edgeVunpack( edgeAdv , elem(ie)%state%Qdp(:,:,:,:,np1_qdp) , nlev*qsize , 0 , elem(ie)%desc ) +! do q = 1 , qsize +! do k = 1 , nlev ! Potential loop inversion (AAM) +! elem(ie)%state%Qdp(:,:,k,q,np1_qdp) = elem(ie)%rspheremp(:,:) * elem(ie)%state%Qdp(:,:,k,q,np1_qdp) +! enddo +! enddo +! enddo + call t_stopf('Prim_Advec_Tracers_remap_ALE') + + +end subroutine Prim_Advec_Tracers_remap_ALE + +subroutine VDOT(rp,Que,rho,mass,hybrid,nets,nete) + use parallel_mod, only: global_shared_buf, global_shared_sum + use global_norms_mod, only: wrap_repro_sum + + implicit none + integer , intent(in) :: nets + integer , intent(in) :: nete + real(kind=real_kind), intent(out) :: rp (nlev,qsize) + real(kind=real_kind), intent(in) :: Que (np*np,nlev,qsize,nets:nete) + real(kind=real_kind), intent(in) :: rho (np*np,nlev, nets:nete) + real(kind=real_kind), intent(in) :: mass (nlev,qsize) + type (hybrid_t) , intent(in) :: hybrid + + integer :: k,n,q,ie + + global_shared_buf = 0 + do ie=nets,nete + n=0 + do q=1,qsize + do k=1,nlev + n=n+1 + global_shared_buf(ie,n) = global_shared_buf(ie,n) + DOT_PRODUCT(Que(:,k,q,ie), rho(:,k,ie)) + end do + end do + end do + + call wrap_repro_sum(nvars=n, comm=hybrid%par%comm) + + n=0 + do q=1,qsize + do k=1,nlev + n=n+1 + rp(k,q) = global_shared_sum(n) - mass(k,q) + enddo + enddo + +end subroutine VDOT + +subroutine Cobra_SLBQP(Que, Que_t, rho, minq, maxq, mass, hybrid, nets, nete) + + use parallel_mod, only: global_shared_buf, global_shared_sum + use global_norms_mod, only: wrap_repro_sum + + implicit none + integer , intent(in) :: nets + integer , intent(in) :: nete + real(kind=real_kind), intent(out) :: Que (np*np,nlev,qsize,nets:nete) + real(kind=real_kind), intent(in) :: Que_t (np*np,nlev,qsize,nets:nete) + real(kind=real_kind), intent(in) :: rho (np*np,nlev, nets:nete) + real(kind=real_kind), intent(in) :: minq (np*np,nlev,qsize,nets:nete) + real(kind=real_kind), intent(in) :: maxq (np*np,nlev,qsize,nets:nete) + real(kind=real_kind), intent(in) :: mass (nlev,qsize) + type (hybrid_t) , intent(in) :: hybrid + + integer, parameter :: max_clip = 50 + real(kind=real_kind), parameter :: eta = 1D-08 + real(kind=real_kind), parameter :: hfd = 1D-10 + real(kind=real_kind) :: lambda_p (nlev,qsize) + real(kind=real_kind) :: lambda_c (nlev,qsize) + real(kind=real_kind) :: rp (nlev,qsize) + real(kind=real_kind) :: rc (nlev,qsize) + real(kind=real_kind) :: rd (nlev,qsize) + real(kind=real_kind) :: alpha (nlev,qsize) + integer :: j,k,n,q,ie + integer :: nclip + + nclip = 0 + + Que(:,:,:,:) = Que_t(:,:,:,:) + + Que = MIN(MAX(Que,minq),maxq) + + call VDOT(rp,Que,rho,mass,hybrid,nets,nete) + nclip = nclip + 1 + + if (MAXVAL(ABS(rp)).lt.eta) return + + do ie=nets,nete + do q=1,qsize + do k=1,nlev + Que(:,k,q,ie) = hfd * rho(:,k,ie) + Que_t(:,k,q,ie) + enddo + enddo + enddo + + Que = MIN(MAX(Que,minq),maxq) + + call VDOT(rc,Que,rho,mass,hybrid,nets,nete) + + rd = rc-rp + if (MAXVAL(ABS(rd)).eq.0) return + + alpha = 0 + WHERE (rd.ne.0) alpha = hfd / rd + + lambda_p = 0 + lambda_c = -alpha*rp + + do while (MAXVAL(ABS(rc)).gt.eta .and. nclip.lt.max_clip) + + do ie=nets,nete + do q=1,qsize + do k=1,nlev + Que(:,k,q,ie) = (lambda_c(k,q) + hfd) * rho(:,k,ie) + Que_t(:,k,q,ie) + enddo + enddo + enddo + Que = MIN(MAX(Que,minq),maxq) + + call VDOT(rc,Que,rho,mass,hybrid,nets,nete) + nclip = nclip + 1 + + rd = rp-rc + + if (MAXVAL(ABS(rd)).eq.0) exit + + alpha = 0 + WHERE (rd.ne.0) alpha = (lambda_p - lambda_c) / rd + + rp = rc + lambda_p = lambda_c + + lambda_c = lambda_c - alpha * rc + + enddo +end subroutine Cobra_SLBQP + + +subroutine Cobra_Elem(Que, Que_t, rho, minq, maxq, mass, hybrid, nets, nete) + + use parallel_mod, only: global_shared_buf, global_shared_sum + use global_norms_mod, only: wrap_repro_sum + + implicit none + integer , intent(in) :: nets + integer , intent(in) :: nete + real(kind=real_kind), intent(out) :: Que (np*np,nlev,qsize,nets:nete) + real(kind=real_kind), intent(in) :: Que_t (np*np,nlev,qsize,nets:nete) + real(kind=real_kind), intent(in) :: rho (np*np,nlev, nets:nete) + real(kind=real_kind), intent(in) :: minq (np*np,nlev,qsize,nets:nete) + real(kind=real_kind), intent(in) :: maxq (np*np,nlev,qsize,nets:nete) + real(kind=real_kind), intent(in) :: mass (nlev,qsize,nets:nete) + type (hybrid_t) , intent(in) :: hybrid + + integer, parameter :: max_clip = 50 + real(kind=real_kind), parameter :: eta = 1D-10 + real(kind=real_kind), parameter :: hfd = 1D-08 + real(kind=real_kind) :: lambda_p (nlev,qsize,nets:nete) + real(kind=real_kind) :: lambda_c (nlev,qsize,nets:nete) + real(kind=real_kind) :: rp (nlev,qsize,nets:nete) + real(kind=real_kind) :: rc (nlev,qsize,nets:nete) + real(kind=real_kind) :: rd (nlev,qsize,nets:nete) + real(kind=real_kind) :: alpha (nlev,qsize,nets:nete) + integer :: j,k,n,q,ie + integer :: nclip + integer :: mloc(3) + + nclip = 1 + + Que(:,:,:,:) = Que_t(:,:,:,:) + + Que = MIN(MAX(Que,minq),maxq) + + do ie=nets,nete + do q=1,qsize + do k=1,nlev + rp(k,q,ie) = DOT_PRODUCT(Que(:,k,q,ie), rho(:,k,ie)) - mass(k,q,ie) + end do + end do + end do + + if (MAXVAL(ABS(rp)).lt.eta) return + + do ie=nets,nete + do q=1,qsize + do k=1,nlev + Que(:,k,q,ie) = hfd * rho(:,k,ie) + Que_t(:,k,q,ie) + enddo + enddo + enddo + + Que = MIN(MAX(Que,minq),maxq) + + do ie=nets,nete + do q=1,qsize + do k=1,nlev + rc(k,q,ie) = DOT_PRODUCT(Que(:,k,q,ie), rho(:,k,ie)) - mass(k,q,ie) + end do + end do + end do + + rd = rc-rp + if (MAXVAL(ABS(rd)).eq.0) return + + alpha = 0 + WHERE (rd.ne.0) alpha = hfd / rd + + lambda_p = 0 + lambda_c = -alpha*rp + +! if (hybrid%par%masterproc) print *,__FILE__,__LINE__," mass(20,1,4):",mass(20,1,4) +! do k=1,np*np +! if (hybrid%par%masterproc) print *,__FILE__,__LINE__," maxq(k,20,1,4):", & +! maxq(k,20,1,4) ,minq(k,20,1,4),maxq(k,20,1,4)-minq(k,20,1,4) +! enddo +! do k=1,np*np +! if (hybrid%par%masterproc) print *,__FILE__,__LINE__," Que(k,20,1,4):",Que(k,20,1,4) ,rho(k,20,4) +! enddo + do while (MAXVAL(ABS(rc)).gt.eta .and. nclip.lt.max_clip) + nclip = nclip + 1 + + do ie=nets,nete + do q=1,qsize + do k=1,nlev +! Que(:,k,q,ie) = (lambda_c(k,q,ie) + hfd) * rho(:,k,ie) + Que_t(:,k,q,ie) + Que(:,k,q,ie) = lambda_c(k,q,ie) * rho(:,k,ie) + Que_t(:,k,q,ie) + enddo + enddo + enddo + +! do ie=nets,nete +! do q=1,qsize +! do k=1,nlev +! rc(k,q,ie) = DOT_PRODUCT(Que(:,k,q,ie), rho(:,k,ie)) - mass(k,q,ie) +! end do +! end do +! end do +! if (hybrid%par%masterproc) print *,__FILE__,__LINE__," rc(20,1,4):",rc(20,1,4), DOT_PRODUCT(Que(:,20,1,4), rho(:,20,4)) + + Que = MIN(MAX(Que,minq),maxq) + + do ie=nets,nete + do q=1,qsize + do k=1,nlev + rc(k,q,ie) = DOT_PRODUCT(Que(:,k,q,ie), rho(:,k,ie)) - mass(k,q,ie) + end do + end do + end do + + + mloc = MAXLOC(ABS(rc)) +! if (hybrid%par%masterproc) print *,__FILE__,__LINE__," MAXVAL(ABS(rc)):",MAXVAL(ABS(rc)), mloc, nclip + + rd = rp-rc + +! if (MAXVAL(ABS(rd)).eq.0) exit + + alpha = 0 + WHERE (rd.ne.0) alpha = (lambda_p - lambda_c) / rd +! WHERE (alpha.eq.0.and.MAXVAL(ABS(rc)).gt.eta) alpha=10; + + rp = rc + lambda_p = lambda_c + + lambda_c = lambda_c - alpha * rc + +! if (hybrid%par%masterproc) print *,__FILE__,__LINE__," rc(20,1,4):",rc(20,1,4) +! if (hybrid%par%masterproc) print *,__FILE__,__LINE__," rd(20,1,4):",rd(20,1,4) +! if (hybrid%par%masterproc) print *,__FILE__,__LINE__," lambda_p(20,1,4):",lambda_p(20,1,4) +! if (hybrid%par%masterproc) print *,__FILE__,__LINE__," lambda_c(20,1,4):",lambda_c(20,1,4) +! if (hybrid%par%masterproc) print *,__FILE__,__LINE__," alpha(20,1,4):",alpha(20,1,4) +! if (hybrid%par%masterproc) print * + enddo +! if (hybrid%par%masterproc) print *,__FILE__,__LINE__," MAXVAL(ABS(rc)):",MAXVAL(ABS(rc)),eta," nclip:",nclip +end subroutine Cobra_Elem + +! ----------------------------------------------------------------------------------! +!SUBROUTINE ALE_RKDSS +! AUTHOR: CHRISTOPH ERATH, MARK TAYLOR, 06. December 2012 +! +! DESCRIPTION: ! create a runge kutta taylor serios mixture to calculate the departure grid +! +! CALLS: +! INPUT: +! +! OUTPUT: +!-----------------------------------------------------------------------------------! + +! this will calculate the velocity at time t+1/2 along the trajectory s(t) given the velocities +! at the GLL points at time t and t+1 using a second order time accurate formulation. + + +subroutine ALE_RKdss(elem, nets, nete, hy, deriv, dt, tl) + + use derivative_mod, only : derivative_t, ugradv_sphere + use edgetype_mod, only : EdgeBuffer_t + use bndry_mod, only : bndry_exchangev + use kinds, only : real_kind + use hybrid_mod, only : hybrid_t + use element_mod, only : element_t + use dimensions_mod, only : np, nlev + + implicit none + + type (element_t) , intent(inout) :: elem(:) + integer , intent(in ) :: nets + integer , intent(in ) :: nete + type (hybrid_t) , intent(in) :: hy ! distributed parallel structure (shared) + type (derivative_t) , intent(in) :: deriv ! derivative struct + real (kind=real_kind), intent(in) :: dt ! timestep + type (TimeLevel_t) , intent(in) :: tl + + integer :: ie, k + real (kind=real_kind), dimension(np,np,2) :: vtmp + integer :: np1 + + np1 = tl%np1 + + +! RK-SSP 2 stage 2nd order: +! x*(t+1) = x(t) + U(x(t),t) dt +! x(t+1) = x(t) + 1/2 ( U(x*(t+1),t+1) + U(x(t),t) ) dt +! apply taylor series: +! U(x*(t+1),t+1) = U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) +! +! x(t+1) = x(t) + 1/2 ( U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) + U(x(t),t) ) dt +! (x(t+1) - x(t)) / dt = 1/2 ( U(x(t),t+1) + (x*(t+1)-x(t)) gradU(x(t),t+1) + U(x(t),t) ) +! (x(t+1) - x(t)) / dt = 1/2 ( U(x(t),t+1) + U(x(t),t) + (x*(t+1)-x(t)) gradU(x(t),t+1) ) +! (x(t+1) - x(t)) / dt = 1/2 ( U(x(t),t+1) + U(x(t),t) + U(x(t),t) dt gradU(x(t),t+1) ) +! +! +! (x(t+1)-x(t))/dt = 1/2(U(x(t),t+1) + U(x(t),t) + dt U(x(t),t) gradU(x(t),t+1)) +! +! suppose dt = -ts (we go backward) +! (x(t-ts)-x(t))/-ts = 1/2( U(x(t),t-ts)+U(x(t),t)) - ts 1/2 U(x(t),t) gradU(x(t),t-ts) +! +! x(t-ts) = x(t)) -ts * [ 1/2( U(x(t),t-ts)+U(x(t),t)) - ts 1/2 U(x(t),t) gradU(x(t),t-ts) ] +! +! !------------------------------------------------------------------------------------ + + do ie=nets,nete + ! vstarn0 = U(x,t) + ! vstar = U(x,t+1) + do k=1,nlev + vtmp(:,:,:)=ugradv_sphere(elem(ie)%state%v(:,:,:,k,np1), elem(ie)%derived%vstar(:,:,:,k),deriv,elem(ie)) + + elem(ie)%derived%vstar(:,:,:,k) = & + (elem(ie)%state%v(:,:,:,k,np1) + elem(ie)%derived%vstar(:,:,:,k))/2 - dt*vtmp(:,:,:)/2 + + elem(ie)%derived%vstar(:,:,1,k) = elem(ie)%derived%vstar(:,:,1,k)*elem(ie)%spheremp(:,:) + elem(ie)%derived%vstar(:,:,2,k) = elem(ie)%derived%vstar(:,:,2,k)*elem(ie)%spheremp(:,:) + enddo + call edgeVpack(edgeveloc,elem(ie)%derived%vstar,2*nlev,0,ie) + enddo + + call t_startf('ALE_RKdss_bexchV') + call bndry_exchangeV(hy,edgeveloc) + call t_stopf('ALE_RKdss_bexchV') + + do ie=nets,nete + call edgeVunpack(edgeveloc,elem(ie)%derived%vstar,2*nlev,0,ie) + do k=1, nlev + elem(ie)%derived%vstar(:,:,1,k) = elem(ie)%derived%vstar(:,:,1,k)*elem(ie)%rspheremp(:,:) + elem(ie)%derived%vstar(:,:,2,k) = elem(ie)%derived%vstar(:,:,2,k)*elem(ie)%rspheremp(:,:) + end do + end do +end subroutine ALE_RKdss + +! ----------------------------------------------------------------------------------! +!SUBROUTINE FVM_DEP_FROM_GLL----------------------------------------------CE-for FVM! +! AUTHOR: CHRISTOPH ERATH, MARK TAYLOR 14. December 2011 ! +! DESCRIPTION: calculates the deparute grid for fvm coming from the gll points ! +! ! +! CALLS: +! INPUT: +! +! OUTPUT: +!-----------------------------------------------------------------------------------! +subroutine ALE_departure_from_gll(acart, vstar, elem, dt) + use physical_constants, only : rearth + use coordinate_systems_mod, only : spherical_polar_t, cartesian3D_t, change_coordinates + use time_mod, only : timelevel_t + use element_mod, only : element_t + use kinds, only : real_kind + use dimensions_mod, only : np + + implicit none + + type(cartesian3D_t) ,intent(out) :: acart(np,np) + real (kind=real_kind) ,intent(in) :: vstar(np,np,2) + type (element_t) ,intent(in) :: elem + real (kind=real_kind) ,intent(in) :: dt + + integer :: i,j + + real (kind=real_kind) :: uxyz (np,np,3) + + ! convert velocity from lat/lon to cartesian 3D + + do i=1,3 + ! Summing along the third dimension is a sum over components for each point. + ! (This is just a faster way of doing a dot product for each grid point, + ! since reindexing the inputs to use the intrinsic effectively would be + ! just asking for trouble.) + uxyz(:,:,i)=sum( elem%vec_sphere2cart(:,:,i,:)*vstar(:,:,:) ,3) + end do + ! interpolate velocity to fvm nodes + ! compute departure point + ! crude, 1st order accurate approximation. to be improved + do i=1,np + do j=1,np + acart(i,j) = change_coordinates(elem%spherep(i,j)) + acart(i,j)%x = acart(i,j)%x - dt*uxyz(i,j,1)/rearth + acart(i,j)%y = acart(i,j)%y - dt*uxyz(i,j,2)/rearth + acart(i,j)%z = acart(i,j)%z - dt*uxyz(i,j,3)/rearth + enddo + enddo + +end subroutine ALE_departure_from_gll + + + + +subroutine ALE_elems_with_dep_points (elem_indexes, dep_points, num_neighbors, ngh_corners) + + use element_mod, only : element_t + use dimensions_mod, only : np + use coordinate_systems_mod, only : cartesian3D_t, change_coordinates + use interpolate_mod, only : point_inside_quad + + implicit none + + ! The ngh_corners array is a list of corners of both elem and all of it's + ! neighor elements all sorted by global id. + integer , intent(in) :: num_neighbors + type(cartesian3D_t),intent(in) :: ngh_corners(4,num_neighbors) + integer , intent(out) :: elem_indexes(np,np) + type(cartesian3D_t) , intent(in) :: dep_points(np,np) + + integer :: i,j,n + logical :: inside + + elem_indexes = -1 + do i=1,np + do j=1,np +! Just itererate the neighbors in global id order to get the same result on every processor. + do n = 1, num_neighbors +! Mark Taylor's handy dandy point_inside_gc check. + inside = point_inside_quad (ngh_corners(:,n), dep_points(i,j)) + if (inside) then + elem_indexes(i,j) = n + exit + end if + end do + end do + end do + + if (MINVAL(elem_indexes(:,:))==-1) then + write (*,*) __FILE__,__LINE__,"Aborting because point not found in neighbor list. Info:" + do i=1,np + do j=1,np + if (elem_indexes(i,j)==-1) then + write (*,*) " departure point ",dep_points(i,j) + do n = 1, num_neighbors + write (*,*) " quad checked ",ngh_corners(1,n) + write (*,*) " ",ngh_corners(2,n) + write (*,*) " ",ngh_corners(3,n) + write (*,*) " ",ngh_corners(4,n) + write (*,*) + end do + exit + end if + end do + end do + call abortmp("ERROR elems_with_dep_points: Can't find departure grid. Time step too long?") + end if +end subroutine ALE_elems_with_dep_points + +function shape_fcn_deriv(pc) result(dNds) + real (kind=real_kind), intent(in) :: pc(2) + real (kind=real_kind) :: dNds(4,2) + + dNds(1, 1) = - 0.25 * (1.0 - pc(2)) + dNds(1, 2) = - 0.25 * (1.0 - pc(1)) + + dNds(2, 1) = 0.25 * (1.0 - pc(2)) + dNds(2, 2) = - 0.25 * (1.0 + pc(1)) + + dNds(3, 1) = 0.25 * (1.0 + pc(2)) + dNds(3, 2) = 0.25 * (1.0 + pc(1)) + + dNds(4, 1) = - 0.25 * (1.0 + pc(2)) + dNds(4, 2) = 0.25 * (1.0 - pc(1)) +end function + +function inv_2x2(A) result(A_inv) + real (kind=real_kind), intent(in) :: A (2,2) + real (kind=real_kind) :: A_inv(2,2) + real (kind=real_kind) :: det, denom + + det = A(1,1) * A(2,2) - A(2,1) * A(1,2) + denom = 1/det + ! inverse: + A_inv(1,1) = denom * A(2,2) ! dxidx + A_inv(2,1) = -denom * A(2,1) ! detadx + A_inv(1,2) = -denom * A(1,2) ! dxidy + A_inv(2,2) = denom * A(1,1) ! detady +end function + +function INV(dxds) result(dsdx) + + real (kind=real_kind), intent(in) :: dxds(3,2) + + real (kind=real_kind) :: dsdx(2,3) + real (kind=real_kind) :: ata(2,2) + real (kind=real_kind) :: ata_inv(2,2) + + + ! dxds = | dxdxi dxdeta | + ! | dydxi dydeta | + ! | dzdxi dzdeta | + ata = MATMUL(TRANSPOSE(dxds), dxds) + ata_inv = inv_2x2(ata) + dsdx = MATMUL(ata_inv, TRANSPOSE(dxds)) + ! dsdx = | dxidx dxidy dxidz | + ! | detadx detady detadz | + +end function + +subroutine shape_fcn(N, pc) + real (kind=real_kind), intent(out) :: N(4) + real (kind=real_kind), intent(in) :: pc(2) + + ! shape function for each node evaluated at param_coords + N(1) = 0.25 * (1.0 - pc(1)) * (1.0 - pc(2)) + N(2) = 0.25 * (1.0 + pc(1)) * (1.0 - pc(2)) + N(3) = 0.25 * (1.0 + pc(1)) * (1.0 + pc(2)) + N(4) = 0.25 * (1.0 - pc(1)) * (1.0 + pc(2)) +end subroutine + + +function F(coords, pc) result(x) + real (kind=real_kind), intent(in) :: pc(2), coords(4,3) + + real (kind=real_kind) :: N(4), x(3) + call shape_fcn(N,pc) + x = MATMUL(TRANSPOSE(coords), N) + x = x/SQRT(DOT_PRODUCT(x,x)) +end function + +function DF(coords, pc) result(dxds) + real (kind=real_kind), intent(in) :: coords(4,3) + real (kind=real_kind), intent(in) :: pc(2) + + real (kind=real_kind) :: dxds(3,2) + real (kind=real_kind) :: dNds(4,2) + real (kind=real_kind) :: dds(3,2) + real (kind=real_kind) :: c(2) + real (kind=real_kind) :: x(3) + real (kind=real_kind) :: xc(3,2) + real (kind=real_kind) :: nx, nx2 + integer :: i,j + + dNds = shape_fcn_deriv (pc) + dds = MATMUL(TRANSPOSE(coords), dNds) + + x = F(coords, pc) + nx2 = DOT_PRODUCT(x,x) + nx = SQRT(nx2) + c = MATMUL(TRANSPOSE(dds), x) + do j=1,2 + do i=1,3 + xc(i,j) = x(i)*c(j) + end do + end do + dxds = nx2*dds - xc + dxds = dxds/(nx*nx2) +end function + + +function cartesian_parametric_coordinates(sphere, corners3D) result (ref) + use coordinate_systems_mod, only : cartesian2d_t, cartesian3D_t, spherical_polar_t, spherical_to_cart + implicit none + type (spherical_polar_t), intent(in) :: sphere + type (cartesian3D_t) , intent(in) :: corners3D(4) !x,y,z coords of element corners + + type (cartesian2D_t) :: ref + + integer, parameter :: MAXIT = 20 + real (kind=real_kind), parameter :: TOL = 1.0E-13 + integer, parameter :: n = 3 + + type (cartesian3D_t) :: cart + real (kind=real_kind) :: coords(4,3), dxds(3,2), dsdx(2,3) + real (kind=real_kind) :: p(3), pc(2), dx(3), x(3), ds(2) + real (kind=real_kind) :: dist, step + + integer :: i,j,k,iter + do i=1,4 + coords(i,1) = corners3D(i)%x + coords(i,2) = corners3D(i)%y + coords(i,3) = corners3D(i)%z + end do + + pc = 0 + p = 0 + cart = spherical_to_cart(sphere) + + p(1) = cart%x + p(2) = cart%y + p(3) = cart%z + + dx = 0 + ds = 0 + dsdx = 0 + dxds = 0 + + !*-------------------------------------------------------------------------*! + + ! Initial guess, center of element + dist = 9999999. + step = 9999999. + iter = 0 + + do while (TOL*TOL.lt.dist .and. iter.lt.MAXIT .and. TOL*TOL.lt.step) + iter = iter + 1 + + dxds = DF (coords, pc) + x = F (coords, pc) + dsdx = INV (dxds) + + dx = x - p + dist = DOT_PRODUCT(dx,dx) + ds = MATMUL(dsdx, dx) + pc = pc - ds + step = DOT_PRODUCT(ds,ds) + enddo + + ref%x = pc(1) + ref%y = pc(2) +end function + + +subroutine ALE_parametric_coords (parametric_coord, elem_indexes, dep_points, num_neighbors, ngh_corners) + use coordinate_systems_mod, only : cartesian2d_t, cartesian3D_t, spherical_polar_t, change_coordinates, distance + use interpolate_mod, only : parametric_coordinates + use dimensions_mod, only : np + + implicit none + + type(cartesian2D_t) , intent(out) :: parametric_coord(np,np) + type(cartesian3D_t) , intent(in) :: dep_points(np,np) + integer , intent(in) :: elem_indexes(np,np) + integer , intent(in) :: num_neighbors + type(cartesian3D_t) , intent(in) :: ngh_corners(4,num_neighbors) + + type (spherical_polar_t) :: sphere(np,np) + integer :: i,j,n + type(cartesian2D_t) :: parametric_test(np,np) + real(kind=real_kind) :: d + + do j=1,np + sphere(:,j) = change_coordinates(dep_points(:,j)) + end do + + do i=1,np + do j=1,np + n = elem_indexes(i,j) + parametric_coord(i,j)= parametric_coordinates(sphere(i,j),ngh_corners(:,n)) + end do + end do +end subroutine ALE_parametric_coords + + +end module diff --git a/components/homme/src/pese/solver_init_mod.F90 b/components/homme/src/pese/solver_init_mod.F90 new file mode 100644 index 000000000000..43bbfdd7c909 --- /dev/null +++ b/components/homme/src/pese/solver_init_mod.F90 @@ -0,0 +1,8 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +module solver_init_mod + use solver_init_base, only: solver_init2 + implicit none +end module solver_init_mod diff --git a/components/homme/src/pese/vertremap_mod.F90 b/components/homme/src/pese/vertremap_mod.F90 new file mode 100644 index 000000000000..04d501bc797c --- /dev/null +++ b/components/homme/src/pese/vertremap_mod.F90 @@ -0,0 +1,10 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +module vertremap_mod + use vertremap_mod_base, only: vertical_remap + implicit none + private + public :: vertical_remap +end module vertremap_mod diff --git a/components/homme/src/pese/viscosity_mod.F90 b/components/homme/src/pese/viscosity_mod.F90 new file mode 100644 index 000000000000..c1471b45d931 --- /dev/null +++ b/components/homme/src/pese/viscosity_mod.F90 @@ -0,0 +1,167 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +module viscosity_mod + +use viscosity_base, only: compute_zeta_C0, compute_div_C0, compute_zeta_C0_contra, compute_div_C0_contra, make_c0, make_c0_vector, neighbor_minmax, biharmonic_wk_scalar, neighbor_minmax_start,neighbor_minmax_finish, smooth_phis + + use kinds, only: rl => real_kind, dd => longdouble_kind + use dimensions_mod, only: np, nlev, nlevp, nelem, nelemd + use hybrid_mod, only: hybrid_t + use element_mod, only: element_t + use element_state, only: elem_state_t, derived_state_t + use derivative_mod, only: derivative_t, laplace_sphere_wk, vlaplace_sphere_wk + use edge_mod, only: initEdgeBuffer, edgevpack, edgevunpack + use edgetype_mod, only: EdgeBuffer_t + use bndry_mod, only: bndry_exchangev + use hybvcoord_mod, only: hvcoord_t + use control_mod, only: nu, nu_s, nu_p, nu_div, nu_top, hypervis_order, hypervis_subcycle, psurf_vis + use element_ops, only: pack_edge_data, unpack_edge_data, apply_vertical_dss, apply_map + use physical_constants, only: Cp, cpwater_vapor, Rgas, kappa, p0, g + use vertical_se, only: vertical_dss + + implicit none + + CONTAINS + + !_____________________________________________________________________ + subroutine apply_laplacian(T,v,elem,edge_buffer,hybrid,deriv,nets,nete,apply_map) + + real(rl), intent(inout) :: T(np,np,nlev,nelemd) + real(rl), intent(inout) :: v(np,np,2,nlev,nelemd) + type (element_t), intent(inout), target :: elem(:) ! array of element_t structures + type (EdgeBuffer_t), intent(inout) :: edge_buffer + type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct + type (derivative_t), intent(in) :: deriv ! horizontal derivative data struct + integer, intent(in) :: nets,nete ! start and end element indices + logical, intent(in) :: apply_map + + type (elem_state_t), pointer :: s ! ptr to element state variables + integer :: ie,i,k ! loop indicies + + do ie=nets,nete + s => elem(ie)%state + + ! apply laplace operator + do k=1,nlev + T(:,:,k,ie) = laplace_sphere_wk(T(:,:,k,ie) ,deriv,elem(ie),var_coef=.false.) + v(:,:,:,k,ie) = vlaplace_sphere_wk(v(:,:,:,k,ie),deriv,elem(ie),var_coef=.false.) + enddo!k + + ! apply mass matrix + if(apply_map) then + do k=1,nlev + T(:,:,k,ie) = T(:,:,k,ie) *elem(ie)%spheremp + v(:,:,1,k,ie) = v(:,:,1,k,ie)*elem(ie)%spheremp + v(:,:,2,k,ie) = v(:,:,2,k,ie)*elem(ie)%spheremp + enddo!k + endif + + ! pack data into edge buffer + i = 0 + call edgeVpack(edge_buffer, T(:,:,:,ie) , nlev , i, ie); i=i+nlev + call edgeVpack(edge_buffer, v(:,:,:,:,ie), 2*nlev, i, ie); i=i+2*nlev + + enddo!ie + + ! exchange edge data + call bndry_exchangeV(hybrid, edge_buffer) ! exchange horizontal edge data and perform dss + + do ie=nets,nete + s => elem(ie)%state + + ! unpack data + i = 0 + call edgeVunpack(edge_buffer, T(:,:,:,ie) , nlev , i, ie); i=i+nlev + call edgeVunpack(edge_buffer, v(:,:,:,:,ie), 2*nlev, i, ie); i=i+2*nlev + + ! apply inverse mass matrix + do k=1,nlev + T(:,:,k,ie) = T(:,:,k,ie) *elem(ie)%rspheremp + v(:,:,1,k,ie) = v(:,:,1,k,ie)*elem(ie)%rspheremp + v(:,:,2,k,ie) = v(:,:,2,k,ie)*elem(ie)%rspheremp + enddo!k + + ! perform stiffness summation + call vertical_dss( T(:,:,:,ie) ) + call vertical_dss( v(:,:,1,:,ie) ) + call vertical_dss( v(:,:,2,:,ie) ) + + enddo!ie + end subroutine + + !_____________________________________________________________________ + subroutine apply_hyperviscosity(elem,edge_buffer,hvcoord,hybrid,deriv,nt,nets,nete,dt2,eta_ave_w) + + ! apply artificial horizontal diffusion to combat spurrious energy build-up + + type (element_t), intent(inout), target :: elem(:) ! array of element_t structures + type (EdgeBuffer_t), intent(inout) :: edge_buffer + type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coord data struct + type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct + type (derivative_t), intent(in) :: deriv ! horizontal derivative data struct + integer, intent(in) :: nt ! time index + integer, intent(in) :: nets,nete ! start and end element indices + real*8, intent(in) :: dt2 + real (rl), intent(in) :: eta_ave_w + + real(rl), dimension(np,np,nlev,nelemd) :: T_diff + real(rl), dimension(np,np,2,nlev,nelemd) :: v_diff + + type (elem_state_t), pointer :: s ! ptr to element state variables + real(rl):: dt + integer :: ic,ie,i,k ! loop indicies + + ! Exit if hypervis coefficients are all zero + if (nu_s == 0 .and. nu == 0 .and. nu_p==0 ) return; + ! get subcycled time-step + dt=dt2/hypervis_subcycle + + ! subcycle hyperviscosity if needed + do ic=1,hypervis_subcycle ! subcycle if needed + + ! copy data to diff-fields + do ie=nets,nete + s => elem(ie)%state + T_diff(:,:,:,ie) = s%T(:,:,:,nt) + v_diff(:,:,:,:,ie) = s%v(:,:,:,:,nt) + enddo + + ! apply 1st-order hyperviscosity + if (hypervis_order == 1) then + + ! apply laplacian once + call apply_laplacian(T_diff,v_diff,elem,edge_buffer,hybrid,deriv,nets,nete,.true.) + + do ie=nets,nete + s => elem(ie)%state + s%T(:,:,:,nt) =s%T (:,:,:,nt) + dt * nu_s * T_diff(:,:,:,ie) + s%v(:,:,:,:,nt) =s%v (:,:,:,:,nt)+ dt * nu * v_diff(:,:,:,:,ie) + call apply_vertical_dss(elem(ie), nt) + enddo!ie + + endif + + ! apply 2nd-order hyperviscosity + if (hypervis_order == 2) then + + ! apply laplacian twice + call apply_laplacian(T_diff,v_diff,elem,edge_buffer, hybrid,deriv,nets,nete,.false.) + call apply_laplacian(T_diff,v_diff,elem,edge_buffer, hybrid,deriv,nets,nete,.false.) + + do ie=nets,nete + s => elem(ie)%state + s%T(:,:,:,nt) =s%T (:,:,:,nt) - dt * nu_s * T_diff(:,:,:,ie) + s%v(:,:,:,:,nt) =s%v (:,:,:,:,nt)- dt * nu * v_diff(:,:,:,:,ie) + call apply_vertical_dss(elem(ie), nt) + enddo!ie + endif + enddo ! ic + + end subroutine + +end module + + + diff --git a/components/homme/src/preqx/CMakeLists.txt b/components/homme/src/preqx/CMakeLists.txt index e32dd1b91c9e..8615f12f267a 100644 --- a/components/homme/src/preqx/CMakeLists.txt +++ b/components/homme/src/preqx/CMakeLists.txt @@ -12,22 +12,20 @@ SET(UTILS_TIMING_DIR ${CMAKE_BINARY_DIR}/utils/cime/share/timing) # Make INCLUDE_DIRS global so the tests can access it SET (EXEC_INCLUDE_DIRS ${PIO_INCLUDE_DIRS} ${UTILS_TIMING_DIR} ) -# Find F90 files in target directory -FILE(GLOB TARGET_F90 ${TARGET_DIR}/*.F90) +# Find all F90 files in specific directories +FILE(GLOB TARGET_F90 ${TARGET_DIR}/*.F90) +FILE(GLOB TEST_SRC_F90 ${SRC_DIR}/test_src/*.F90) +FILE(GLOB SB81_SRC_F90 ${SRC_SHARE_DIR}/sb81/*.F90) SET(PREQX_SRCS_F90 ${TARGET_F90} - ${SRC_DIR}/test_src/asp_tests.F90 - ${SRC_DIR}/test_src/baroclinic_inst_mod.F90 - ${SRC_DIR}/test_src/dcmip2012_test1_2_3.F90 - ${SRC_DIR}/test_src/dcmip2012_test4.F90 - ${SRC_DIR}/test_src/dcmip2012_test5.F90 + ${TEST_SRC_F90} + ${SB81_SRC_F90} ${SRC_DIR}/checksum_mod.F90 ${SRC_DIR}/common_io_mod.F90 ${SRC_DIR}/common_movie_mod.F90 - ${SRC_DIR}/forcing_mod.F90 - ${SRC_DIR}/interpolate_driver_mod.F90 ${SRC_DIR}/interp_movie_mod.F90 + ${SRC_DIR}/interpolate_driver_mod.F90 ${SRC_DIR}/netcdf_io_mod.F90 ${SRC_DIR}/pio_io_mod.F90 ${SRC_DIR}/prim_main.F90 @@ -38,62 +36,58 @@ SET(PREQX_SRCS_F90 ${SRC_DIR}/surfaces_mod.F90 ${SRC_DIR}/test_mod.F90 ${SRC_SHARE_DIR}/bndry_mod_base.F90 - ${SRC_SHARE_DIR}/derivative_mod_base.F90 - ${SRC_SHARE_DIR}/edge_mod_base.F90 - ${SRC_SHARE_DIR}/prim_advection_mod_base.F90 - ${SRC_SHARE_DIR}/solver_init_mod_base.F90 - ${SRC_SHARE_DIR}/vertremap_mod_base.F90 - ${SRC_SHARE_DIR}/viscosity_mod_base.F90 - ${SRC_SHARE_DIR}/kinds.F90 - ${SRC_SHARE_DIR}/dimensions_mod.F90 + ${SRC_SHARE_DIR}/cg_mod.F90 ${SRC_SHARE_DIR}/control_mod.F90 - ${SRC_SHARE_DIR}/physical_constants.F90 ${SRC_SHARE_DIR}/coordinate_systems_mod.F90 - ${SRC_SHARE_DIR}/params_mod.F90 - ${SRC_SHARE_DIR}/gridgraph_mod.F90 - ${SRC_SHARE_DIR}/element_mod.F90 - ${SRC_SHARE_DIR}/parallel_mod.F90 - ${SRC_SHARE_DIR}/hybrid_mod.F90 - ${SRC_SHARE_DIR}/reduction_mod.F90 - ${SRC_SHARE_DIR}/quadrature_mod.F90 - ${SRC_SHARE_DIR}/global_norms_mod.F90 - ${SRC_SHARE_DIR}/ll_mod.F90 - ${SRC_SHARE_DIR}/metagraph_mod.F90 - ${SRC_SHARE_DIR}/schedule_mod.F90 - ${SRC_SHARE_DIR}/schedtype_mod.F90 - ${SRC_SHARE_DIR}/edgetype_mod.F90 - ${SRC_SHARE_DIR}/thread_mod.F90 - ${SRC_SHARE_DIR}/unit_tests_mod.F90 - ${SRC_SHARE_DIR}/spacecurve_mod.F90 ${SRC_SHARE_DIR}/cube_mod.F90 - ${SRC_SHARE_DIR}/interpolate_mod.F90 + ${SRC_SHARE_DIR}/derivative_mod_base.F90 + ${SRC_SHARE_DIR}/dimensions_mod.F90 ${SRC_SHARE_DIR}/dof_mod.F90 - ${SRC_SHARE_DIR}/time_mod.F90 - ${SRC_SHARE_DIR}/physics_mod.F90 - ${SRC_SHARE_DIR}/hybvcoord_mod.F90 ${SRC_SHARE_DIR}/domain_mod.F90 - ${SRC_SHARE_DIR}/cg_mod.F90 + ${SRC_SHARE_DIR}/edge_mod_base.F90 + ${SRC_SHARE_DIR}/edgetype_mod.F90 + ${SRC_SHARE_DIR}/element_mod.F90 + ${SRC_SHARE_DIR}/global_norms_mod.F90 + ${SRC_SHARE_DIR}/gridgraph_mod.F90 + ${SRC_SHARE_DIR}/hybrid_mod.F90 + ${SRC_SHARE_DIR}/hybvcoord_mod.F90 + ${SRC_SHARE_DIR}/interpolate_mod.F90 + ${SRC_SHARE_DIR}/kinds.F90 ${SRC_SHARE_DIR}/linear_algebra_mod.F90 - ${SRC_SHARE_DIR}/solver_mod.F90 - ${SRC_SHARE_DIR}/namelist_mod.F90 + ${SRC_SHARE_DIR}/ll_mod.F90 ${SRC_SHARE_DIR}/mass_matrix_mod.F90 + ${SRC_SHARE_DIR}/mesh_mod.F90 + ${SRC_SHARE_DIR}/metagraph_mod.F90 + ${SRC_SHARE_DIR}/metis_mod.F90 + ${SRC_SHARE_DIR}/namelist_mod.F90 + ${SRC_SHARE_DIR}/parallel_mod.F90 + ${SRC_SHARE_DIR}/params_mod.F90 + ${SRC_SHARE_DIR}/physical_constants.F90 + ${SRC_SHARE_DIR}/physics_mod.F90 + ${SRC_SHARE_DIR}/prim_advection_base.F90 + ${SRC_SHARE_DIR}/prim_driver_base.F90 ${SRC_SHARE_DIR}/prim_si_mod.F90 - ${SRC_SHARE_DIR}/prim_state_mod.F90 ${SRC_SHARE_DIR}/prim_si_ref_mod.F90 - ${SRC_SHARE_DIR}/prim_advance_mod.F90 - ${SRC_SHARE_DIR}/metis_mod.F90 - ${SRC_SHARE_DIR}/prim_driver_mod.F90 - ${SRC_SHARE_DIR}/mesh_mod.F90 + ${SRC_SHARE_DIR}/prim_state_mod.F90 + ${SRC_SHARE_DIR}/quadrature_mod.F90 + ${SRC_SHARE_DIR}/reduction_mod.F90 + ${SRC_SHARE_DIR}/schedtype_mod.F90 + ${SRC_SHARE_DIR}/schedule_mod.F90 + ${SRC_SHARE_DIR}/solver_init_base.F90 + ${SRC_SHARE_DIR}/solver_mod.F90 + ${SRC_SHARE_DIR}/spacecurve_mod.F90 + ${SRC_SHARE_DIR}/thread_mod.F90 + ${SRC_SHARE_DIR}/time_mod.F90 + ${SRC_SHARE_DIR}/unit_tests_mod.F90 + ${SRC_SHARE_DIR}/vertremap_base.F90 + ${SRC_SHARE_DIR}/viscosity_base.F90 + ${UTILS_SHARE_DIR}/shr_const_mod.F90 + ${UTILS_SHARE_DIR}/shr_file_mod.F90 ${UTILS_SHARE_DIR}/shr_kind_mod.F90 ${UTILS_SHARE_DIR}/shr_mpi_mod.F90 - ${UTILS_SHARE_DIR}/shr_sys_mod.F90 - ${UTILS_SHARE_DIR}/shr_file_mod.F90 - ${UTILS_SHARE_DIR}/shr_vmath_mod.F90 - ${UTILS_SHARE_DIR}/shr_const_mod.F90 ${UTILS_SHARE_DIR}/shr_spfn_mod.F90 - ${PHYSICS_DIR}/heldsuarez/held_suarez_mod.F90 - ${PHYSICS_DIR}/column_model_mod.F90 - ${PHYSICS_DIR}/column_types_mod.F90 + ${UTILS_SHARE_DIR}/shr_sys_mod.F90 + ${UTILS_SHARE_DIR}/shr_vmath_mod.F90 ) SET(PREQX_SRCS_C diff --git a/components/homme/src/preqx/dcmip_tests.F90 b/components/homme/src/preqx/dcmip_tests.F90 index 16f1f560483e..5c3453f919ee 100644 --- a/components/homme/src/preqx/dcmip_tests.F90 +++ b/components/homme/src/preqx/dcmip_tests.F90 @@ -362,8 +362,8 @@ subroutine dcmip2012_test2_x_forcing(elem,hybrid,hvcoord,nets,nete,n,dt) ! apply sponge layer forcing to momentum terms do ie=nets,nete do k=1,nlev - elem(ie)%derived%FM(:,:,1,k,n) = -f_d(k)/tau * ( elem(ie)%state%v(:,:,1,k,n) - u0(:,:,k,ie) ) - elem(ie)%derived%FM(:,:,2,k,n) = -f_d(k)/tau * ( elem(ie)%state%v(:,:,2,k,n) - v0(:,:,k,ie) ) + elem(ie)%derived%FM(:,:,1,k) = -f_d(k)/tau * ( elem(ie)%state%v(:,:,1,k,n) - u0(:,:,k,ie) ) + elem(ie)%derived%FM(:,:,2,k) = -f_d(k)/tau * ( elem(ie)%state%v(:,:,2,k,n) - v0(:,:,k,ie) ) enddo enddo diff --git a/components/homme/src/preqx/element_ops.F90 b/components/homme/src/preqx/element_ops.F90 new file mode 100644 index 000000000000..60c7806183de --- /dev/null +++ b/components/homme/src/preqx/element_ops.F90 @@ -0,0 +1,177 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +! +! getter & setter functions that must be provided by each model +! +! IMPORTANT NOTE: For vertically lagrangian models, these +! routines should ONLY be used outside the timestepping loop +! on reference levels. To compute these fields on floating levels +! the model should do that directly (or we need to modify the interface) +! +! get_field() +! returns temperature, potential temperature, phi, etc.. +! +! These should be unified to a single interface: +! set_thermostate() +! initial condition interface used by DCMIP 2008 tests +! +! set_state() +! initial condition interface used by DCMIP 2012 tests +! +! +! +module element_ops + + use dimensions_mod, only: np, nlev, nlevp, nelemd, qsize, max_corner_elem + use element_mod, only: element_t + use element_state, only: timelevels + use hybrid_mod, only: hybrid_t + use hybvcoord_mod, only: hvcoord_t + use kinds, only: real_kind, iulog + use perf_mod, only: t_startf, t_stopf, t_barrierf, t_adj_detailf ! _EXTERNAL + use parallel_mod, only: abortmp + use physical_constants, only : kappa, p0 + + implicit none + +contains + + + subroutine get_field(elem,name,field,hvcoord,nt,ntQ) + implicit none + type (element_t), intent(in) :: elem + character(len=*), intent(in) :: name + real (kind=real_kind), intent(out) :: field(np,np,nlev) + type (hvcoord_t), intent(in) :: hvcoord + integer, intent(in) :: nt + integer, intent(in) :: ntQ + + integer :: k + + select case(name) + case ('temperature') + call get_temperature(elem,field,hvcoord,nt,ntQ) + case ('pottemp') + call get_pottemp(elem,field,hvcoord,nt,ntQ) + case ('phi') + field = elem%derived%phi(:,:,:) + case ('omega') + do k=1,nlev + field(:,:,k)=elem%derived%omega_p(:,:,k)*& + (hvcoord%hyam(k)*hvcoord%ps0 + hvcoord%hybm(k)*elem%state%ps_v(:,:,nt)) + end do + case default + print *,'name = ',trim(name) + call abortmp('ERROR: get_field name not supported in this model') + end select + + end subroutine + + + + subroutine get_pottemp(elem,pottemp,hvcoord,nt,ntQ) + implicit none + + type (element_t), intent(in) :: elem + real (kind=real_kind), intent(out) :: pottemp(np,np,nlev) + type (hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct + integer, intent(in) :: nt + integer, intent(in) :: ntQ + + ! local + real (kind=real_kind) :: pfull(np,np,nlev) + integer :: k + + +#if (defined COLUMN_OPENMP) + !$omp parallel do default(shared), private(k) +#endif + do k=1,nlev + pfull(:,:,k) = hvcoord%hyam(k)*hvcoord%ps0 & + + hvcoord%hybm(k)*elem%state%ps_v(:,:,nt) + pottemp(:,:,k)=elem%state%T(:,:,k,nt)* & + (pfull(:,:,k)/p0)**(-kappa) + enddo + + end subroutine get_pottemp + + + + subroutine get_temperature(elem,temperature,hvcoord,nt,ntQ) + implicit none + + type (element_t), intent(in) :: elem + real (kind=real_kind), intent(out) :: temperature(np,np,nlev) + type (hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct + integer, intent(in) :: nt + integer, intent(in) :: ntQ + + ! local + real (kind=real_kind) :: p(np,np,nlev) + real (kind=real_kind) :: dp(np,np,nlev) + real (kind=real_kind) :: kappa_star(np,np,nlev) + real (kind=real_kind) :: Qt(np,np,nlev) + integer :: k + + temperature = elem%state%T(:,:,:,nt) + + end subroutine get_temperature + + + + subroutine copy_state(elem,nin,nout) + implicit none + + type (element_t), intent(inout) :: elem + integer :: nin,nout + + elem%state%v(:,:,:,:,nout)=elem%state%v(:,:,:,:,nin) + elem%state%T(:,:,:,nout) =elem%state%T(:,:,:,nin) + elem%state%dp3d(:,:,:,nout)=elem%state%dp3d(:,:,:,nin) + elem%state%ps_v(:,:,nout) =elem%state%ps_v(:,:,nin) + end subroutine copy_state + + + + subroutine set_thermostate(elem,temperature,hvcoord,n0,n0_q) + implicit none + + type (element_t), intent(inout) :: elem + real (kind=real_kind), intent(in) :: temperature(np,np,nlev) + type (hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct + integer :: n0,n0_q + + elem%state%T(:,:,:,n0)=temperature(:,:,:) + + end subroutine set_thermostate + + +subroutine set_state(u,v,w,T,ps,phis,p,dp,zm, g, i,j,k,elem,n0,n1) + + ! set state variables at node(i,j,k) at layer midpoints + + real(real_kind), intent(in) :: u,v,w,T,ps,phis,p,dp,zm,g + integer, intent(in) :: i,j,k,n0,n1 + type(element_t), intent(inout) :: elem + + + ! set prognostic state variables at level midpoints + elem%state%v (i,j,1,k,n0:n1) = u + elem%state%v (i,j,2,k,n0:n1) = v + elem%state%T (i,j,k,n0:n1) = T + elem%state%dp3d(i,j,k,n0:n1) = dp + elem%state%ps_v(i,j,n0:n1) = ps + elem%state%phis(i,j) = phis + + ! set some diagnostic variables + elem%derived%dp(i,j,k) = dp + elem%derived%phi(i,j,k) = g*zm + +end subroutine + + + +end module + diff --git a/components/homme/src/preqx/element_state.F90 b/components/homme/src/preqx/element_state.F90 index d2fb853a990a..cb550d64e9e7 100644 --- a/components/homme/src/preqx/element_state.F90 +++ b/components/homme/src/preqx/element_state.F90 @@ -31,8 +31,6 @@ module element_state end type elem_state_t - integer(kind=int_kind),public,parameter::StateComponents=7! num prognistics variables (for prim_restart_mod.F90) - !___________________________________________________________________ type, public :: derived_state_t @@ -61,37 +59,12 @@ module element_state real (kind=real_kind) :: divdp(np,np,nlev) ! divergence of dp real (kind=real_kind) :: divdp_proj(np,np,nlev) ! DSSed divdp -#ifdef CAM ! forcing terms for CAM - real (kind=real_kind) :: FQ(np,np,nlev,qsize_d, 1) ! tracer forcing - real (kind=real_kind) :: FM(np,np,2,nlev, 1) ! momentum forcing - real (kind=real_kind) :: FT(np,np,nlev, 1) ! temperature forcing - real (kind=real_kind) :: etadot_prescribed(np,np,nlevp) ! prescribed vertical tendency - real (kind=real_kind) :: u_met(np,np,nlev) ! zonal component of prescribed meteorology winds - real (kind=real_kind) :: dudt_met(np,np,nlev) ! rate of change of zonal component of prescribed meteorology winds - real (kind=real_kind) :: v_met(np,np,nlev) ! meridional component of prescribed meteorology winds - real (kind=real_kind) :: dvdt_met(np,np,nlev) ! rate of change of meridional component of prescribed meteorology winds - real (kind=real_kind) :: T_met(np,np,nlev) ! prescribed meteorology temperature - real (kind=real_kind) :: dTdt_met(np,np,nlev) ! rate of change of prescribed meteorology temperature - real (kind=real_kind) :: ps_met(np,np) ! surface pressure of prescribed meteorology - real (kind=real_kind) :: dpsdt_met(np,np) ! rate of change of surface pressure of prescribed meteorology - real (kind=real_kind) :: nudge_factor(np,np,nlev) ! nudging factor (prescribed) - real (kind=real_kind) :: Utnd(npsq,nlev) ! accumulated U tendency due to nudging towards prescribed met - real (kind=real_kind) :: Vtnd(npsq,nlev) ! accumulated V tendency due to nudging towards prescribed met - real (kind=real_kind) :: Ttnd(npsq,nlev) ! accumulated T tendency due to nudging towards prescribed met - -#else - ! forcing terms for HOMME - real (kind=real_kind) :: FQ(np,np,nlev,qsize_d, timelevels) ! tracer forcing - real (kind=real_kind) :: FM(np,np,2,nlev, timelevels) ! momentum forcing - real (kind=real_kind) :: FT(np,np,nlev, timelevels) ! temperature forcing -#endif - - ! forcing terms for both CAM and HOMME - ! FQps for conserving dry mass in the presence of precipitation - + real (kind=real_kind) :: FQ(np,np,nlev,qsize_d) ! tracer forcing + real (kind=real_kind) :: FM(np,np,2,nlev) ! momentum forcing + real (kind=real_kind) :: FT(np,np,nlev) ! temperature forcing real (kind=real_kind) :: pecnd(np,np,nlev) ! pressure perturbation from condensate - real (kind=real_kind) :: FQps(np,np,timelevels) ! forcing of FQ on ps_v + real (kind=real_kind) :: FQps(np,np) ! forcing of FQ on ps_v end type derived_state_t diff --git a/components/homme/src/preqx/prim_advection_mod.F90 b/components/homme/src/preqx/prim_advection_mod.F90 index d46594145c7f..e9fd5da5a8d4 100644 --- a/components/homme/src/preqx/prim_advection_mod.F90 +++ b/components/homme/src/preqx/prim_advection_mod.F90 @@ -14,7 +14,7 @@ module prim_advection_mod use hybrid_mod, only : hybrid_t use control_mod, only : use_semi_lagrange_transport use sl_advection, only : prim_advec_tracers_remap_ALE, sl_init1 - use prim_advection_mod_base, only: prim_advec_init1_rk2, prim_advec_tracers_remap_rk2 + use prim_advection_base, only: prim_advec_init1_rk2, prim_advec_tracers_remap_rk2 implicit none diff --git a/components/homme/src/preqx/prim_driver_mod.F90 b/components/homme/src/preqx/prim_driver_mod.F90 new file mode 100644 index 000000000000..69e06f67915e --- /dev/null +++ b/components/homme/src/preqx/prim_driver_mod.F90 @@ -0,0 +1,355 @@ +module prim_driver_mod + + use prim_driver_base, only:& + deriv1,& + prim_init1,& + prim_init2 ,& + prim_finalize,& + smooth_topo_datasets + + use control_mod, only: energy_fixer, statefreq, ftype, qsplit, rsplit + use dimensions_mod, only: np, nlev, nlevp, nelem, nelemd, qsize + use element_mod, only: element_t, allocate_element_desc + use element_state, only: timelevels + use hybrid_mod, only: hybrid_t + use hybvcoord_mod, only: hvcoord_t + use kinds, only: real_kind, iulog + use parallel_mod, only: abortmp + use perf_mod, only: t_startf, t_stopf + use prim_advection_mod, only: prim_advec_tracers_remap + use prim_state_mod, only: prim_printstate, prim_diag_scalars, prim_energy_halftimes + use time_mod, only: timeLevel_t, timelevel_update, timelevel_qdp + +#ifndef CAM + use test_mod, only: compute_test_forcing +#endif + + implicit none + contains + + subroutine prim_run_subcycle(elem, hybrid,nets,nete, dt, tl, hvcoord,nsubstep) + + ! advance dynamic variables and tracers (u,v,T,ps,Q,C) from time t to t + dt_q + ! + ! input: + ! tl%nm1 not used + ! tl%n0 data at time t + ! tl%np1 new values at t+dt_q + ! + ! then we update timelevel pointers: + ! tl%nm1 = tl%n0 + ! tl%n0 = tl%np1 + ! so that: + ! tl%nm1 tracers: t dynamics: t+(qsplit-1)*dt + ! tl%n0 time t + dt_q + + use control_mod, only: statefreq, ftype, qsplit, rsplit, disable_diagnostics + use hybvcoord_mod, only: hvcoord_t + use parallel_mod, only: abortmp + use prim_advance_mod, only: applycamforcing, applycamforcing_dynamics + use prim_state_mod, only: prim_printstate, prim_diag_scalars, prim_energy_halftimes + use vertremap_mod, only: vertical_remap + use reduction_mod, only: parallelmax + use time_mod, only: TimeLevel_t, timelevel_update, timelevel_qdp, nsplit + +#if USE_OPENACC + use openacc_utils_mod, only: copy_qdp_h2d, copy_qdp_d2h +#endif + + type (element_t) , intent(inout) :: elem(:) + type (hybrid_t), intent(in) :: hybrid ! distributed parallel structure (shared) + type (hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct + integer, intent(in) :: nets ! starting thread element number (private) + integer, intent(in) :: nete ! ending thread element number (private) + real(kind=real_kind), intent(in) :: dt ! "timestep dependent" timestep + type (TimeLevel_t), intent(inout) :: tl + integer, intent(in) :: nsubstep ! nsubstep = 1 .. nsplit + + real(kind=real_kind) :: dp, dt_q, dt_remap + real(kind=real_kind) :: dp_np1(np,np) + integer :: ie,i,j,k,n,q,t + integer :: n0_qdp,np1_qdp,r,nstep_end + logical :: compute_diagnostics, compute_energy + + ! compute timesteps for tracer transport and vertical remap + dt_q = dt*qsplit + dt_remap = dt_q + nstep_end = tl%nstep + qsplit + if (rsplit>0) then + dt_remap = dt_q*rsplit ! rsplit=0 means use eulerian code, not vert. lagrange + nstep_end = tl%nstep + qsplit*rsplit ! nstep at end of this routine + endif + + ! activate diagnostics periodically for display to stdout + compute_energy = .false. + compute_diagnostics = .false. + if (MODULO(nstep_end,statefreq)==0 .or. nstep_end==tl%nstep0) then + compute_diagnostics= .true. + compute_energy = .true. + endif + if(disable_diagnostics) compute_diagnostics= .false. + + ! compute scalar diagnostics if currently active + if (compute_diagnostics) then + call t_startf("prim_diag_scalars") + call prim_diag_scalars(elem,hvcoord,tl,4,.true.,nets,nete) + call t_stopf("prim_diag_scalars") + endif + +#ifndef CAM + ! Get HOMME test case forcing + call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete) +#endif + + ! Apply CAM Physics forcing + + ! ftype= 2: Q was adjusted by physics, but apply u,T forcing here + ! ftype= 1: forcing was applied time-split in CAM coupling layer + ! ftype= 0: apply all forcing here + ! ftype=-1: do not apply forcing + + call TimeLevel_Qdp(tl, qsplit, n0_qdp) + + if (ftype==0) then + call t_startf("ApplyCAMForcing") + call ApplyCAMForcing(elem, hvcoord,tl%n0,n0_qdp, dt_remap,nets,nete) + call t_stopf("ApplyCAMForcing") + + elseif (ftype==2) then + call t_startf("ApplyCAMForcing_dynamics") + call ApplyCAMForcing_dynamics(elem, hvcoord,tl%n0,dt_remap,nets,nete) + call t_stopf("ApplyCAMForcing_dynamics") + endif + + ! E(1) Energy after CAM forcing + if (compute_energy) then + call t_startf("prim_energy_halftimes") + call prim_energy_halftimes(elem,hvcoord,tl,1,.true.,nets,nete) + call t_stopf("prim_energy_halftimes") + endif + + ! qmass and variance, using Q(n0),Qdp(n0) + if (compute_diagnostics) then + call t_startf("prim_diag_scalars") + call prim_diag_scalars(elem,hvcoord,tl,1,.true.,nets,nete) + call t_stopf("prim_diag_scalars") + endif + + ! initialize dp3d from ps + do ie=nets,nete + do k=1,nlev + elem(ie)%state%dp3d(:,:,k,tl%n0)=& + ( hvcoord%hyai(k+1) - hvcoord%hyai(k) )*hvcoord%ps0 + & + ( hvcoord%hybi(k+1) - hvcoord%hybi(k) )*elem(ie)%state%ps_v(:,:,tl%n0) + enddo + enddo + +#if (USE_OPENACC) + call TimeLevel_Qdp( tl, qsplit, n0_qdp, np1_qdp) + call t_startf("copy_qdp_h2d") + call copy_qdp_h2d( elem , n0_qdp ) + call t_stopf("copy_qdp_h2d") +#endif + + ! Loop over rsplit vertically lagrangian timesteps + call t_startf("prim_step_rX") + call prim_step(elem, hybrid,nets,nete, dt, tl, hvcoord,compute_diagnostics,1) + call t_stopf("prim_step_rX") + + do r=2,rsplit + call TimeLevel_update(tl,"leapfrog") + call t_startf("prim_step_rX") + call prim_step(elem, hybrid,nets,nete, dt, tl, hvcoord,.false.,r) + call t_stopf("prim_step_rX") + enddo + ! defer final timelevel update until after remap and diagnostics + +#if (USE_OPENACC) + call TimeLevel_Qdp( tl, qsplit, n0_qdp, np1_qdp) + call t_startf("copy_qdp_h2d") + call copy_qdp_d2h( elem , np1_qdp ) + call t_stopf("copy_qdp_h2d") +#endif + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! apply vertical remap + ! always for tracers + ! if rsplit>0: also remap dynamics and compute reference level ps_v + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !compute timelevels for tracers (no longer the same as dynamics) + call TimeLevel_Qdp( tl, qsplit, n0_qdp, np1_qdp) + call vertical_remap(hybrid,elem,hvcoord,dt_remap,tl%np1,np1_qdp,nets,nete) + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! time step is complete. update some diagnostic variables: + ! Q (mixing ratio) + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + call t_startf("prim_run_subcyle_diags") + do ie=nets,nete +#if (defined COLUMN_OPENMP) + !$omp parallel do default(shared), private(k,q,dp_np1) +#endif + do k=1,nlev ! Loop inversion (AAM) + dp_np1(:,:) = ( hvcoord%hyai(k+1) - hvcoord%hyai(k) )*hvcoord%ps0 + & + ( hvcoord%hybi(k+1) - hvcoord%hybi(k) )*elem(ie)%state%ps_v(:,:,tl%np1) + !dir$ simd + do q=1,qsize + elem(ie)%state%Q(:,:,k,q)=elem(ie)%state%Qdp(:,:,k,q,np1_qdp)/dp_np1(:,:) + enddo + enddo + enddo + call t_stopf("prim_run_subcyle_diags") + + ! now we have: + ! u(nm1) dynamics at t+dt_remap - 2*dt + ! u(n0) dynamics at t+dt_remap - dt + ! u(np1) dynamics at t+dt_remap + ! + ! Q(1) Q at t+dt_remap + if (compute_diagnostics) then + call t_startf("prim_diag_scalars") + call prim_diag_scalars(elem,hvcoord,tl,2,.false.,nets,nete) + call t_stopf("prim_diag_scalars") + endif + if (compute_energy) then + call t_startf("prim_energy_halftimes") + call prim_energy_halftimes(elem,hvcoord,tl,2,.false.,nets,nete) + call t_stopf("prim_energy_halftimes") + endif + + if (compute_diagnostics) then + call t_startf("prim_diag_scalars") + call prim_diag_scalars(elem,hvcoord,tl,3,.false.,nets,nete) + call t_stopf("prim_diag_scalars") + + call t_startf("prim_energy_halftimes") + call prim_energy_halftimes(elem,hvcoord,tl,3,.false.,nets,nete) + call t_stopf("prim_energy_halftimes") + endif + + ! ================================= + ! update dynamics time level pointers + ! ================================= + call TimeLevel_update(tl,"leapfrog") + ! now we have: + ! u(nm1) dynamics at t+dt_remap - dt + ! u(n0) dynamics at t+dt_remap + ! u(np1) undefined + + ! ============================================================ + ! Print some diagnostic information + ! ============================================================ + if (compute_diagnostics) then + call prim_printstate(elem, tl, hybrid,hvcoord,nets,nete) + end if + + end subroutine prim_run_subcycle + + !_____________________________________________________________________ + subroutine prim_step(elem, hybrid,nets,nete, dt, tl, hvcoord, compute_diagnostics,rstep) + + ! Take qsplit dynamics steps and one tracer step + ! for vertically lagrangian option, this subroutine does only the horizontal step + ! + ! input: + ! tl%nm1 not used + ! tl%n0 data at time t + ! tl%np1 new values at t+dt_q + ! + ! then we update timelevel pointers: + ! tl%nm1 = tl%n0 + ! tl%n0 = tl%np1 + ! so that: + ! tl%nm1 tracers: t dynamics: t+(qsplit-1)*dt + ! tl%n0 time t + dt_q + + use control_mod, only: statefreq, integration, ftype, qsplit, nu_p, rsplit + use control_mod, only: use_semi_lagrange_transport + use hybvcoord_mod, only : hvcoord_t + use parallel_mod, only: abortmp + use prim_advance_mod, only: prim_advance_exp + use prim_advection_mod, only: prim_advec_tracers_remap + use reduction_mod, only: parallelmax + use time_mod, only: time_at,TimeLevel_t, timelevel_update, nsplit + + type(element_t), intent(inout) :: elem(:) + type(hybrid_t), intent(in) :: hybrid ! distributed parallel structure (shared) + type(hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct + integer, intent(in) :: nets ! starting thread element number (private) + integer, intent(in) :: nete ! ending thread element number (private) + real(kind=real_kind), intent(in) :: dt ! "timestep dependent" timestep + type(TimeLevel_t), intent(inout) :: tl + integer, intent(in) :: rstep ! vertical remap subcycling step + + real(kind=real_kind) :: st, st1, dp, dt_q + real(kind=real_kind) :: maxcflx, maxcfly + real(kind=real_kind) :: dp_np1(np,np) + logical :: compute_diagnostics + integer :: ie, t, q,k,i,j,n, n_Q + + dt_q = dt*qsplit + + ! =============== + ! initialize mean flux accumulation variables and save some variables at n0 + ! for use by advection + ! =============== + do ie=nets,nete + elem(ie)%derived%eta_dot_dpdn=0 ! mean vertical mass flux + elem(ie)%derived%vn0=0 ! mean horizontal mass flux + elem(ie)%derived%omega_p=0 + if (nu_p>0) then + elem(ie)%derived%dpdiss_ave=0 + elem(ie)%derived%dpdiss_biharmonic=0 + endif + if (use_semi_lagrange_transport) then + elem(ie)%derived%vstar=elem(ie)%state%v(:,:,:,:,tl%n0) + end if + elem(ie)%derived%dp(:,:,:)=elem(ie)%state%dp3d(:,:,:,tl%n0) + enddo + + ! =============== + ! Dynamical Step + ! =============== + call t_startf("prim_step_dyn") + n_Q = tl%n0 ! n_Q = timelevel of FV tracers at time t. need to save this + ! FV tracers still carry 3 timelevels + ! SE tracers only carry 2 timelevels + call prim_advance_exp(elem, deriv1, hvcoord, & + hybrid, dt, tl, nets, nete, compute_diagnostics) + do n=2,qsplit + call TimeLevel_update(tl,"leapfrog") + call prim_advance_exp(elem, deriv1, hvcoord,hybrid, dt, tl, nets, nete, .false.) + ! defer final timelevel update until after Q update. + enddo + call t_stopf("prim_step_dyn") + + ! current dynamics state variables: + ! derived%dp = dp at start of timestep + ! derived%vstar = velocity at start of tracer timestep + ! derived%vn0 = mean horiz. flux: U*dp + ! state%dp3d(:,:,:,np1) = dp3d + ! rsplit=0 + ! state%v(:,:,:,np1) = velocity on reference levels + ! rsplit>0 + ! state%v(:,:,:,np1) = velocity on lagrangian levels + ! + ! Tracer Advection. + ! in addition, this routine will apply the DSS to: + ! derived%eta_dot_dpdn = mean vertical velocity (used for remap below) + ! derived%omega = + ! Tracers are always vertically lagrangian. + ! For rsplit=0: + ! if tracer scheme needs v on lagrangian levels it has to vertically interpolate + ! if tracer scheme needs dp3d, it needs to derive it from ps_v + + call t_startf("prim_step_advec") + if (qsize > 0) then + call t_startf("PAT_remap") + call Prim_Advec_Tracers_remap(elem, deriv1,hvcoord,hybrid,dt_q,tl,nets,nete) + call t_stopf("PAT_remap") + end if + call t_stopf("prim_step_advec") + + end subroutine prim_step + +end module diff --git a/components/homme/src/preqx/solver_init_mod.F90 b/components/homme/src/preqx/solver_init_mod.F90 index 90efb25eadf8..43bbfdd7c909 100644 --- a/components/homme/src/preqx/solver_init_mod.F90 +++ b/components/homme/src/preqx/solver_init_mod.F90 @@ -3,6 +3,6 @@ #endif module solver_init_mod - use solver_init_mod_base, only: solver_init2 + use solver_init_base, only: solver_init2 implicit none end module solver_init_mod diff --git a/components/homme/src/preqx/viscosity_mod.F90 b/components/homme/src/preqx/viscosity_mod.F90 index 07aa91047f18..1bd29c62484f 100644 --- a/components/homme/src/preqx/viscosity_mod.F90 +++ b/components/homme/src/preqx/viscosity_mod.F90 @@ -3,7 +3,7 @@ #endif module viscosity_mod - use viscosity_mod_base, only: biharmonic_wk, compute_zeta_C0, compute_div_C0, compute_zeta_C0_contra, compute_div_C0_contra, make_c0, make_c0_vector, neighbor_minmax - use viscosity_mod_base, only: biharmonic_wk_scalar, neighbor_minmax_start,neighbor_minmax_finish, biharmonic_wk_dp3d + use viscosity_base, only: biharmonic_wk, compute_zeta_C0, compute_div_C0, compute_zeta_C0_contra, compute_div_C0_contra, make_c0, make_c0_vector, neighbor_minmax + use viscosity_base, only: biharmonic_wk_scalar, neighbor_minmax_start,neighbor_minmax_finish, biharmonic_wk_dp3d, smooth_phis implicit none end module viscosity_mod diff --git a/components/homme/src/preqx_acc/CMakeLists.txt b/components/homme/src/preqx_acc/CMakeLists.txt index 92a659a55d6e..dd3837ba0a78 100644 --- a/components/homme/src/preqx_acc/CMakeLists.txt +++ b/components/homme/src/preqx_acc/CMakeLists.txt @@ -4,6 +4,7 @@ macro(preqx_acc_setup) +SET(TARGET_DIR ${HOMME_SOURCE_DIR}/src/preqx_acc) SET(UTILS_SHARE_DIR ${HOMME_SOURCE_DIR}/utils/csm_share) SET(SRC_DIR ${HOMME_SOURCE_DIR}/src) SET(SRC_SHARE_DIR ${HOMME_SOURCE_DIR}/src/share) @@ -14,89 +15,88 @@ SET(UTILS_TIMING_DIR ${CMAKE_BINARY_DIR}/utils/cime/share/timing) SET (EXEC_INCLUDE_DIRS ${PIO_INCLUDE_DIRS} ${UTILS_TIMING_DIR} ) -SET(TARGET_DIR ${HOMME_SOURCE_DIR}/src/preqx_acc) -FILE(GLOB TARGET_F90 ${TARGET_DIR}/*.F90) +# Find all F90 files in specific directories +FILE(GLOB TARGET_F90 ${TARGET_DIR}/*.F90) +FILE(GLOB TEST_SRC_F90 ${SRC_DIR}/test_src/*.F90) +FILE(GLOB SB81_SRC_F90 ${SRC_SHARE_DIR}/sb81/*.F90) SET(PREQX_SRCS_F90 ${TARGET_F90} + ${TEST_SRC_F90} + ${SB81_SRC_F90} + ${SRC_DIR}/checksum_mod.F90 + ${SRC_DIR}/common_io_mod.F90 + ${SRC_DIR}/common_movie_mod.F90 + ${SRC_DIR}/interp_movie_mod.F90 + ${SRC_DIR}/interpolate_driver_mod.F90 + ${SRC_DIR}/netcdf_io_mod.F90 + ${SRC_DIR}/pio_io_mod.F90 + ${SRC_DIR}/preqx/dcmip_tests.F90 + ${SRC_DIR}/prim_main.F90 + ${SRC_DIR}/prim_movie_mod.F90 + ${SRC_DIR}/prim_restart_mod.F90 + ${SRC_DIR}/repro_sum_mod.F90 + ${SRC_DIR}/restart_io_mod.F90 + ${SRC_DIR}/surfaces_mod.F90 + ${SRC_DIR}/test_mod.F90 ${SRC_DIR}/test_src/asp_tests.F90 ${SRC_DIR}/test_src/baroclinic_inst_mod.F90 ${SRC_DIR}/test_src/dcmip2012_test1_2_3.F90 ${SRC_DIR}/test_src/dcmip2012_test4.F90 ${SRC_DIR}/test_src/dcmip2012_test5.F90 ${SRC_SHARE_DIR}/bndry_mod_base.F90 - ${SRC_SHARE_DIR}/derivative_mod_base.F90 - ${SRC_SHARE_DIR}/edge_mod_base.F90 - ${SRC_SHARE_DIR}/prim_advection_mod_base.F90 - ${SRC_SHARE_DIR}/solver_init_mod_base.F90 - ${SRC_SHARE_DIR}/vertremap_mod_base.F90 - ${SRC_SHARE_DIR}/viscosity_mod_base.F90 - ${SRC_DIR}/preqx/dcmip_tests.F90 - ${SRC_SHARE_DIR}/kinds.F90 - ${SRC_SHARE_DIR}/dimensions_mod.F90 + ${SRC_SHARE_DIR}/cg_mod.F90 ${SRC_SHARE_DIR}/control_mod.F90 - ${SRC_SHARE_DIR}/physical_constants.F90 ${SRC_SHARE_DIR}/coordinate_systems_mod.F90 - ${SRC_SHARE_DIR}/params_mod.F90 - ${SRC_SHARE_DIR}/gridgraph_mod.F90 + ${SRC_SHARE_DIR}/cube_mod.F90 + ${SRC_SHARE_DIR}/derivative_mod_base.F90 + ${SRC_SHARE_DIR}/dimensions_mod.F90 + ${SRC_SHARE_DIR}/dof_mod.F90 + ${SRC_SHARE_DIR}/domain_mod.F90 + ${SRC_SHARE_DIR}/edge_mod_base.F90 + ${SRC_SHARE_DIR}/edgetype_mod.F90 ${SRC_SHARE_DIR}/element_mod.F90 - ${SRC_SHARE_DIR}/parallel_mod.F90 - ${SRC_SHARE_DIR}/hybrid_mod.F90 - ${SRC_SHARE_DIR}/reduction_mod.F90 - ${SRC_SHARE_DIR}/quadrature_mod.F90 ${SRC_SHARE_DIR}/global_norms_mod.F90 + ${SRC_SHARE_DIR}/gridgraph_mod.F90 + ${SRC_SHARE_DIR}/hybrid_mod.F90 + ${SRC_SHARE_DIR}/hybvcoord_mod.F90 + ${SRC_SHARE_DIR}/interpolate_mod.F90 + ${SRC_SHARE_DIR}/kinds.F90 + ${SRC_SHARE_DIR}/linear_algebra_mod.F90 ${SRC_SHARE_DIR}/ll_mod.F90 + ${SRC_SHARE_DIR}/mass_matrix_mod.F90 + ${SRC_SHARE_DIR}/mesh_mod.F90 ${SRC_SHARE_DIR}/metagraph_mod.F90 - ${SRC_SHARE_DIR}/schedule_mod.F90 + ${SRC_SHARE_DIR}/metis_mod.F90 + ${SRC_SHARE_DIR}/namelist_mod.F90 + ${SRC_SHARE_DIR}/parallel_mod.F90 + ${SRC_SHARE_DIR}/params_mod.F90 + ${SRC_SHARE_DIR}/physical_constants.F90 + ${SRC_SHARE_DIR}/physics_mod.F90 + ${SRC_SHARE_DIR}/prim_advection_base.F90 + ${SRC_SHARE_DIR}/prim_driver_base.F90 + ${SRC_SHARE_DIR}/prim_si_mod.F90 + ${SRC_SHARE_DIR}/prim_si_ref_mod.F90 + ${SRC_SHARE_DIR}/prim_state_mod.F90 + ${SRC_SHARE_DIR}/quadrature_mod.F90 + ${SRC_SHARE_DIR}/reduction_mod.F90 ${SRC_SHARE_DIR}/schedtype_mod.F90 - ${SRC_SHARE_DIR}/edgetype_mod.F90 - ${SRC_SHARE_DIR}/thread_mod.F90 - ${SRC_SHARE_DIR}/unit_tests_mod.F90 + ${SRC_SHARE_DIR}/schedule_mod.F90 + ${SRC_SHARE_DIR}/solver_init_base.F90 + ${SRC_SHARE_DIR}/solver_mod.F90 ${SRC_SHARE_DIR}/spacecurve_mod.F90 - ${SRC_SHARE_DIR}/cube_mod.F90 - ${SRC_SHARE_DIR}/interpolate_mod.F90 - ${SRC_SHARE_DIR}/dof_mod.F90 - ${SRC_DIR}/common_io_mod.F90 - ${SRC_DIR}/pio_io_mod.F90 - ${SRC_DIR}/repro_sum_mod.F90 + ${SRC_SHARE_DIR}/thread_mod.F90 ${SRC_SHARE_DIR}/time_mod.F90 - ${SRC_SHARE_DIR}/physics_mod.F90 - ${SRC_SHARE_DIR}/hybvcoord_mod.F90 - ${SRC_DIR}/interp_movie_mod.F90 + ${SRC_SHARE_DIR}/unit_tests_mod.F90 + ${SRC_SHARE_DIR}/vertremap_base.F90 + ${SRC_SHARE_DIR}/viscosity_base.F90 + ${UTILS_SHARE_DIR}/shr_const_mod.F90 + ${UTILS_SHARE_DIR}/shr_file_mod.F90 ${UTILS_SHARE_DIR}/shr_kind_mod.F90 ${UTILS_SHARE_DIR}/shr_mpi_mod.F90 - ${UTILS_SHARE_DIR}/shr_sys_mod.F90 - ${UTILS_SHARE_DIR}/shr_file_mod.F90 - ${UTILS_SHARE_DIR}/shr_vmath_mod.F90 - ${UTILS_SHARE_DIR}/shr_const_mod.F90 ${UTILS_SHARE_DIR}/shr_spfn_mod.F90 - ${SRC_SHARE_DIR}/domain_mod.F90 - ${SRC_DIR}/restart_io_mod.F90 - ${SRC_SHARE_DIR}/cg_mod.F90 - ${SRC_SHARE_DIR}/linear_algebra_mod.F90 - ${SRC_SHARE_DIR}/solver_mod.F90 - ${SRC_DIR}/common_movie_mod.F90 - ${SRC_SHARE_DIR}/namelist_mod.F90 - ${SRC_SHARE_DIR}/mass_matrix_mod.F90 - ${SRC_DIR}/checksum_mod.F90 - ${SRC_SHARE_DIR}/prim_si_mod.F90 - ${SRC_SHARE_DIR}/prim_state_mod.F90 - ${SRC_DIR}/prim_restart_mod.F90 - ${SRC_SHARE_DIR}/prim_si_ref_mod.F90 - ${SRC_SHARE_DIR}/prim_advance_mod.F90 - ${SRC_DIR}/forcing_mod.F90 - ${SRC_SHARE_DIR}/metis_mod.F90 - ${SRC_SHARE_DIR}/prim_driver_mod.F90 - ${SRC_DIR}/prim_movie_mod.F90 - ${SRC_DIR}/surfaces_mod.F90 - ${SRC_DIR}/interpolate_driver_mod.F90 - ${SRC_DIR}/netcdf_io_mod.F90 - ${SRC_SHARE_DIR}/mesh_mod.F90 - ${SRC_DIR}/prim_main.F90 - ${PHYSICS_DIR}/heldsuarez/held_suarez_mod.F90 - ${PHYSICS_DIR}/column_model_mod.F90 - ${PHYSICS_DIR}/column_types_mod.F90 - ${SRC_DIR}/test_mod.F90 + ${UTILS_SHARE_DIR}/shr_sys_mod.F90 + ${UTILS_SHARE_DIR}/shr_vmath_mod.F90 ) SET(PREQX_SRCS_C @@ -142,9 +142,6 @@ SET(COLUMN_OPENMP FALSE ) endmacro(preqx_acc_setup) - - - ############################################################################ # createTestExec(exec_name exec_type NP PLEV USE_PIO USE_ENERGY) ############################################################################ diff --git a/components/homme/src/preqx_acc/element_ops.F90 b/components/homme/src/preqx_acc/element_ops.F90 new file mode 100644 index 000000000000..fef2b86002eb --- /dev/null +++ b/components/homme/src/preqx_acc/element_ops.F90 @@ -0,0 +1,175 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +! +! getter & setter functions that must be provided by each model +! +! IMPORTANT NOTE: For vertically lagrangian models, these +! routines should ONLY be used outside the timestepping loop +! on reference levels. To compute these fields on floating levels +! the model should do that directly (or we need to modify the interface) +! +! get_field() +! returns temperature, potential temperature, phi, etc.. +! +! These should be unified to a single interface: +! set_thermostate() +! initial condition interface used by DCMIP 2008 tests +! +! set_state() +! initial condition interface used by DCMIP 2012 tests +! +! +! +module element_ops + + use dimensions_mod, only: np, nlev, nlevp, nelemd, qsize, max_corner_elem + use element_mod, only: element_t + use element_state, only: timelevels + use hybrid_mod, only: hybrid_t + use hybvcoord_mod, only: hvcoord_t + use kinds, only: real_kind, iulog + use perf_mod, only: t_startf, t_stopf, t_barrierf, t_adj_detailf ! _EXTERNAL + use parallel_mod, only: abortmp + use physical_constants, only : kappa, p0 + + implicit none + +contains + + + subroutine get_field(elem,name,field,hvcoord,nt,ntQ) + implicit none + type (element_t), intent(in) :: elem + character(len=*), intent(in) :: name + real (kind=real_kind), intent(out) :: field(np,np,nlev) + type (hvcoord_t), intent(in) :: hvcoord + integer, intent(in) :: nt + integer, intent(in) :: ntQ + + integer :: k + + select case(name) + case ('temperature') + call get_temperature(elem,field,hvcoord,nt,ntQ) + case ('pottemp') + call get_pottemp(elem,field,hvcoord,nt,ntQ) + case ('phi') + field = elem%derived%phi(:,:,:) + case ('omega') + do k=1,nlev + field(:,:,k)=elem%derived%omega_p(:,:,k)*& + (hvcoord%hyam(k)*hvcoord%ps0 + hvcoord%hybm(k)*elem%state%ps_v(:,:,nt)) + end do + case default + print *,'name = ',trim(name) + call abortmp('ERROR: get_field name not supported in this model') + end select + + end subroutine + + + + subroutine get_pottemp(elem,pottemp,hvcoord,nt,ntQ) + implicit none + + type (element_t), intent(in) :: elem + real (kind=real_kind), intent(out) :: pottemp(np,np,nlev) + type (hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct + integer, intent(in) :: nt + integer, intent(in) :: ntQ + + ! local + real (kind=real_kind) :: pfull(np,np,nlev) + integer :: k + + +#if (defined COLUMN_OPENMP) + !$omp parallel do default(shared), private(k) +#endif + do k=1,nlev + pfull(:,:,k) = hvcoord%hyam(k)*hvcoord%ps0 & + + hvcoord%hybm(k)*elem%state%ps_v(:,:,nt) + pottemp(:,:,k)=elem%state%T(:,:,k,nt)* & + (pfull(:,:,k)/p0)**(-kappa) + enddo + + end subroutine get_pottemp + + + + subroutine get_temperature(elem,temperature,hvcoord,nt,ntQ) + implicit none + + type (element_t), intent(in) :: elem + real (kind=real_kind), intent(out) :: temperature(np,np,nlev) + type (hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct + integer, intent(in) :: nt + integer, intent(in) :: ntQ + + ! local + real (kind=real_kind) :: p(np,np,nlev) + real (kind=real_kind) :: dp(np,np,nlev) + real (kind=real_kind) :: kappa_star(np,np,nlev) + real (kind=real_kind) :: Qt(np,np,nlev) + integer :: k + + temperature = elem%state%T(:,:,:,nt) + + end subroutine get_temperature + + + + subroutine copy_state(elem,nin,nout) + implicit none + + type (element_t), intent(inout) :: elem + integer :: nin,nout + + elem%state%v(:,:,:,:,nout)=elem%state%v(:,:,:,:,nin) + elem%state%T(:,:,:,nout) =elem%state%T(:,:,:,nin) + elem%state%dp3d(:,:,:,nout)=elem%state%dp3d(:,:,:,nin) + elem%state%ps_v(:,:,nout) =elem%state%ps_v(:,:,nin) + end subroutine copy_state + + + + subroutine set_thermostate(elem,temperature,hvcoord,n0,n0_q) + implicit none + + type (element_t), intent(inout) :: elem + real (kind=real_kind), intent(in) :: temperature(np,np,nlev) + type (hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct + integer :: n0,n0_q + + elem%state%T(:,:,:,n0)=temperature(:,:,:) + + end subroutine set_thermostate + + +subroutine set_state(u,v,w,T,ps,phis,p,dp,zm, g, i,j,k,elem,n0,n1) + + ! set state variables at node(i,j,k) at layer midpoints + + real(real_kind), intent(in) :: u,v,w,T,ps,phis,p,dp,zm,g + integer, intent(in) :: i,j,k,n0,n1 + type(element_t), intent(inout) :: elem + + + ! set prognostic state variables at level midpoints + elem%state%v (i,j,1,k,n0:n1) = u + elem%state%v (i,j,2,k,n0:n1) = v + elem%state%T (i,j,k,n0:n1) = T + elem%state%dp3d(i,j,k,n0:n1) = dp + elem%state%ps_v(i,j,n0:n1) = ps + elem%state%phis(i,j) = phis + + ! set some diagnostic variables + elem%derived%dp(i,j,k) = dp + elem%derived%phi(i,j,k) = g*zm + +end subroutine + +end module + diff --git a/components/homme/src/preqx_acc/element_state.F90 b/components/homme/src/preqx_acc/element_state.F90 index 5cd80056800b..e16d7a7d135d 100644 --- a/components/homme/src/preqx_acc/element_state.F90 +++ b/components/homme/src/preqx_acc/element_state.F90 @@ -32,7 +32,6 @@ module element_state real (kind=real_kind), pointer :: Qdp (:,:,:,:,:) ! Tracer mass 7 (np,np,nlev,qsize,2) end type elem_state_t - integer(kind=int_kind),public,parameter::StateComponents=7 ! num prognistics variables (for prim_restart_mod.F90) type, public :: derived_state_t ! diagnostic variables for preqx solver ! storage for subcycling tracers/dynamics @@ -58,36 +57,12 @@ module element_state real (kind=real_kind), pointer :: divdp (:,:,:) ! (np,np,nlev) divergence of dp real (kind=real_kind), pointer :: divdp_proj (:,:,:) ! (np,np,nlev) DSSed divdp -#ifdef CAM ! forcing terms for CAM - real (kind=real_kind) :: FQ(np,np,nlev,qsize_d, 1) ! tracer forcing - real (kind=real_kind) :: FM(np,np,2,nlev, 1) ! momentum forcing - real (kind=real_kind) :: FT(np,np,nlev, 1) ! temperature forcing - real (kind=real_kind) :: etadot_prescribed(np,np,nlevp) ! prescribed vertical tendency - real (kind=real_kind) :: u_met(np,np,nlev) ! zonal component of prescribed meteorology winds - real (kind=real_kind) :: dudt_met(np,np,nlev) ! rate of change of zonal component of prescribed meteorology winds - real (kind=real_kind) :: v_met(np,np,nlev) ! meridional component of prescribed meteorology winds - real (kind=real_kind) :: dvdt_met(np,np,nlev) ! rate of change of meridional component of prescribed meteorology winds - real (kind=real_kind) :: T_met(np,np,nlev) ! prescribed meteorology temperature - real (kind=real_kind) :: dTdt_met(np,np,nlev) ! rate of change of prescribed meteorology temperature - real (kind=real_kind) :: ps_met(np,np) ! surface pressure of prescribed meteorology - real (kind=real_kind) :: dpsdt_met(np,np) ! rate of change of surface pressure of prescribed meteorology - real (kind=real_kind) :: nudge_factor(np,np,nlev) ! nudging factor (prescribed) - real (kind=real_kind) :: Utnd(npsq,nlev) ! accumulated U tendency due to nudging towards prescribed met - real (kind=real_kind) :: Vtnd(npsq,nlev) ! accumulated V tendency due to nudging towards prescribed met - real (kind=real_kind) :: Ttnd(npsq,nlev) ! accumulated T tendency due to nudging towards prescribed met -#else - ! forcing terms for HOMME - real (kind=real_kind) :: FQ(np,np,nlev,qsize_d, timelevels) ! tracer forcing - real (kind=real_kind) :: FM(np,np,2,nlev, timelevels) ! momentum forcing - real (kind=real_kind) :: FT(np,np,nlev, timelevels) ! temperature forcing -#endif - - ! forcing terms for both CAM and HOMME - ! FQps for conserving dry mass in the presence of precipitation - + real (kind=real_kind) :: FQ(np,np,nlev,qsize_d) ! tracer forcing + real (kind=real_kind) :: FM(np,np,2,nlev) ! momentum forcing + real (kind=real_kind) :: FT(np,np,nlev) ! temperature forcing real (kind=real_kind) :: pecnd(np,np,nlev) ! pressure perturbation from condensate - real (kind=real_kind) :: FQps(np,np,timelevels) ! forcing of FQ on ps_v + real (kind=real_kind) :: FQps(np,np) ! forcing of FQ on ps_v end type derived_state_t diff --git a/components/homme/src/preqx_acc/prim_driver_mod.F90 b/components/homme/src/preqx_acc/prim_driver_mod.F90 new file mode 100644 index 000000000000..65d0680367e1 --- /dev/null +++ b/components/homme/src/preqx_acc/prim_driver_mod.F90 @@ -0,0 +1,353 @@ +module prim_driver_mod + + use prim_driver_base, only:& + deriv1,& + prim_init1,& + prim_init2 ,& + prim_finalize,& + smooth_topo_datasets + + use control_mod, only: energy_fixer, statefreq, ftype, qsplit, rsplit + use dimensions_mod, only: np, nlev, nlevp, nelem, nelemd, qsize + use element_mod, only: element_t, allocate_element_desc + use element_state, only: timelevels + use hybrid_mod, only: hybrid_t + use hybvcoord_mod, only: hvcoord_t + use kinds, only: real_kind, iulog + use parallel_mod, only: abortmp + use perf_mod, only: t_startf, t_stopf +! use prim_advance_mod, only: compute_and_apply_rhs + use prim_advection_mod, only: prim_advec_tracers_remap + use prim_state_mod, only: prim_printstate, prim_diag_scalars, prim_energy_halftimes + use time_mod, only: timeLevel_t, timelevel_update, timelevel_qdp + use test_mod, only: compute_test_forcing + + implicit none + contains + + subroutine prim_run_subcycle(elem, hybrid,nets,nete, dt, tl, hvcoord,nsubstep) + + ! advance dynamic variables and tracers (u,v,T,ps,Q,C) from time t to t + dt_q + ! + ! input: + ! tl%nm1 not used + ! tl%n0 data at time t + ! tl%np1 new values at t+dt_q + ! + ! then we update timelevel pointers: + ! tl%nm1 = tl%n0 + ! tl%n0 = tl%np1 + ! so that: + ! tl%nm1 tracers: t dynamics: t+(qsplit-1)*dt + ! tl%n0 time t + dt_q + + use control_mod, only: statefreq, ftype, qsplit, rsplit, disable_diagnostics + use hybvcoord_mod, only: hvcoord_t + use parallel_mod, only: abortmp + use prim_advance_mod, only: applycamforcing, applycamforcing_dynamics + use prim_state_mod, only: prim_printstate, prim_diag_scalars, prim_energy_halftimes + use vertremap_mod, only: vertical_remap + use reduction_mod, only: parallelmax + use time_mod, only: TimeLevel_t, timelevel_update, timelevel_qdp, nsplit + +#if USE_OPENACC + use openacc_utils_mod, only: copy_qdp_h2d, copy_qdp_d2h +#endif + + type (element_t) , intent(inout) :: elem(:) + type (hybrid_t), intent(in) :: hybrid ! distributed parallel structure (shared) + type (hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct + integer, intent(in) :: nets ! starting thread element number (private) + integer, intent(in) :: nete ! ending thread element number (private) + real(kind=real_kind), intent(in) :: dt ! "timestep dependent" timestep + type (TimeLevel_t), intent(inout) :: tl + integer, intent(in) :: nsubstep ! nsubstep = 1 .. nsplit + + real(kind=real_kind) :: dp, dt_q, dt_remap + real(kind=real_kind) :: dp_np1(np,np) + integer :: ie,i,j,k,n,q,t + integer :: n0_qdp,np1_qdp,r,nstep_end + logical :: compute_diagnostics, compute_energy + + ! compute timesteps for tracer transport and vertical remap + dt_q = dt*qsplit + dt_remap = dt_q + nstep_end = tl%nstep + qsplit + if (rsplit>0) then + dt_remap = dt_q*rsplit ! rsplit=0 means use eulerian code, not vert. lagrange + nstep_end = tl%nstep + qsplit*rsplit ! nstep at end of this routine + endif + + ! activate diagnostics periodically for display to stdout + compute_energy = .false. + compute_diagnostics = .false. + if (MODULO(nstep_end,statefreq)==0 .or. nstep_end==tl%nstep0) then + compute_diagnostics= .true. + compute_energy = .true. + endif + if(disable_diagnostics) compute_diagnostics= .false. + + ! compute scalar diagnostics if currently active + if (compute_diagnostics) then + call t_startf("prim_diag_scalars") + call prim_diag_scalars(elem,hvcoord,tl,4,.true.,nets,nete) + call t_stopf("prim_diag_scalars") + endif + +#ifndef CAM + ! Get HOMME test case forcing + call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete) +#endif + + ! Apply CAM Physics forcing + + ! ftype= 2: Q was adjusted by physics, but apply u,T forcing here + ! ftype= 1: forcing was applied time-split in CAM coupling layer + ! ftype= 0: apply all forcing here + ! ftype=-1: do not apply forcing + + call TimeLevel_Qdp(tl, qsplit, n0_qdp) + + if (ftype==0) then + call t_startf("ApplyCAMForcing") + call ApplyCAMForcing(elem, hvcoord,tl%n0,n0_qdp, dt_remap,nets,nete) + call t_stopf("ApplyCAMForcing") + + elseif (ftype==2) then + call t_startf("ApplyCAMForcing_dynamics") + call ApplyCAMForcing_dynamics(elem, hvcoord,tl%n0,dt_remap,nets,nete) + call t_stopf("ApplyCAMForcing_dynamics") + endif + + ! E(1) Energy after CAM forcing + if (compute_energy) then + call t_startf("prim_energy_halftimes") + call prim_energy_halftimes(elem,hvcoord,tl,1,.true.,nets,nete) + call t_stopf("prim_energy_halftimes") + endif + + ! qmass and variance, using Q(n0),Qdp(n0) + if (compute_diagnostics) then + call t_startf("prim_diag_scalars") + call prim_diag_scalars(elem,hvcoord,tl,1,.true.,nets,nete) + call t_stopf("prim_diag_scalars") + endif + + ! initialize dp3d from ps + do ie=nets,nete + do k=1,nlev + elem(ie)%state%dp3d(:,:,k,tl%n0)=& + ( hvcoord%hyai(k+1) - hvcoord%hyai(k) )*hvcoord%ps0 + & + ( hvcoord%hybi(k+1) - hvcoord%hybi(k) )*elem(ie)%state%ps_v(:,:,tl%n0) + enddo + enddo + +#if (USE_OPENACC) + call TimeLevel_Qdp( tl, qsplit, n0_qdp, np1_qdp) + call t_startf("copy_qdp_h2d") + call copy_qdp_h2d( elem , n0_qdp ) + call t_stopf("copy_qdp_h2d") +#endif + + ! Loop over rsplit vertically lagrangian timesteps + call t_startf("prim_step_rX") + call prim_step(elem, hybrid,nets,nete, dt, tl, hvcoord,compute_diagnostics,1) + call t_stopf("prim_step_rX") + + do r=2,rsplit + call TimeLevel_update(tl,"leapfrog") + call t_startf("prim_step_rX") + call prim_step(elem, hybrid,nets,nete, dt, tl, hvcoord,.false.,r) + call t_stopf("prim_step_rX") + enddo + ! defer final timelevel update until after remap and diagnostics + +#if (USE_OPENACC) + call TimeLevel_Qdp( tl, qsplit, n0_qdp, np1_qdp) + call t_startf("copy_qdp_h2d") + call copy_qdp_d2h( elem , np1_qdp ) + call t_stopf("copy_qdp_h2d") +#endif + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! apply vertical remap + ! always for tracers + ! if rsplit>0: also remap dynamics and compute reference level ps_v + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !compute timelevels for tracers (no longer the same as dynamics) + call TimeLevel_Qdp( tl, qsplit, n0_qdp, np1_qdp) + call vertical_remap(hybrid,elem,hvcoord,dt_remap,tl%np1,np1_qdp,nets,nete) + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! time step is complete. update some diagnostic variables: + ! Q (mixing ratio) + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + call t_startf("prim_run_subcyle_diags") + do ie=nets,nete +#if (defined COLUMN_OPENMP) + !$omp parallel do default(shared), private(k,q,dp_np1) +#endif + do k=1,nlev ! Loop inversion (AAM) + dp_np1(:,:) = ( hvcoord%hyai(k+1) - hvcoord%hyai(k) )*hvcoord%ps0 + & + ( hvcoord%hybi(k+1) - hvcoord%hybi(k) )*elem(ie)%state%ps_v(:,:,tl%np1) + !dir$ simd + do q=1,qsize + elem(ie)%state%Q(:,:,k,q)=elem(ie)%state%Qdp(:,:,k,q,np1_qdp)/dp_np1(:,:) + enddo + enddo + enddo + call t_stopf("prim_run_subcyle_diags") + + ! now we have: + ! u(nm1) dynamics at t+dt_remap - 2*dt + ! u(n0) dynamics at t+dt_remap - dt + ! u(np1) dynamics at t+dt_remap + ! + ! Q(1) Q at t+dt_remap + if (compute_diagnostics) then + call t_startf("prim_diag_scalars") + call prim_diag_scalars(elem,hvcoord,tl,2,.false.,nets,nete) + call t_stopf("prim_diag_scalars") + endif + if (compute_energy) then + call t_startf("prim_energy_halftimes") + call prim_energy_halftimes(elem,hvcoord,tl,2,.false.,nets,nete) + call t_stopf("prim_energy_halftimes") + endif + + if (compute_diagnostics) then + call t_startf("prim_diag_scalars") + call prim_diag_scalars(elem,hvcoord,tl,3,.false.,nets,nete) + call t_stopf("prim_diag_scalars") + + call t_startf("prim_energy_halftimes") + call prim_energy_halftimes(elem,hvcoord,tl,3,.false.,nets,nete) + call t_stopf("prim_energy_halftimes") + endif + + ! ================================= + ! update dynamics time level pointers + ! ================================= + call TimeLevel_update(tl,"leapfrog") + ! now we have: + ! u(nm1) dynamics at t+dt_remap - dt + ! u(n0) dynamics at t+dt_remap + ! u(np1) undefined + + ! ============================================================ + ! Print some diagnostic information + ! ============================================================ + if (compute_diagnostics) then + call prim_printstate(elem, tl, hybrid,hvcoord,nets,nete) + end if + + end subroutine prim_run_subcycle + + !_____________________________________________________________________ + subroutine prim_step(elem, hybrid,nets,nete, dt, tl, hvcoord, compute_diagnostics,rstep) + + ! Take qsplit dynamics steps and one tracer step + ! for vertically lagrangian option, this subroutine does only the horizontal step + ! + ! input: + ! tl%nm1 not used + ! tl%n0 data at time t + ! tl%np1 new values at t+dt_q + ! + ! then we update timelevel pointers: + ! tl%nm1 = tl%n0 + ! tl%n0 = tl%np1 + ! so that: + ! tl%nm1 tracers: t dynamics: t+(qsplit-1)*dt + ! tl%n0 time t + dt_q + + use control_mod, only: statefreq, integration, ftype, qsplit, nu_p, rsplit + use control_mod, only: use_semi_lagrange_transport + use hybvcoord_mod, only : hvcoord_t + use parallel_mod, only: abortmp + use prim_advance_mod, only: prim_advance_exp + use prim_advection_mod, only: prim_advec_tracers_remap + use reduction_mod, only: parallelmax + use time_mod, only: time_at,TimeLevel_t, timelevel_update, nsplit + + type(element_t), intent(inout) :: elem(:) + type(hybrid_t), intent(in) :: hybrid ! distributed parallel structure (shared) + type(hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct + integer, intent(in) :: nets ! starting thread element number (private) + integer, intent(in) :: nete ! ending thread element number (private) + real(kind=real_kind), intent(in) :: dt ! "timestep dependent" timestep + type(TimeLevel_t), intent(inout) :: tl + integer, intent(in) :: rstep ! vertical remap subcycling step + + real(kind=real_kind) :: st, st1, dp, dt_q + real(kind=real_kind) :: maxcflx, maxcfly + real(kind=real_kind) :: dp_np1(np,np) + logical :: compute_diagnostics + integer :: ie, t, q,k,i,j,n, n_Q + + dt_q = dt*qsplit + + ! =============== + ! initialize mean flux accumulation variables and save some variables at n0 + ! for use by advection + ! =============== + do ie=nets,nete + elem(ie)%derived%eta_dot_dpdn=0 ! mean vertical mass flux + elem(ie)%derived%vn0=0 ! mean horizontal mass flux + elem(ie)%derived%omega_p=0 + if (nu_p>0) then + elem(ie)%derived%dpdiss_ave=0 + elem(ie)%derived%dpdiss_biharmonic=0 + endif + if (use_semi_lagrange_transport) then + elem(ie)%derived%vstar=elem(ie)%state%v(:,:,:,:,tl%n0) + end if + elem(ie)%derived%dp(:,:,:)=elem(ie)%state%dp3d(:,:,:,tl%n0) + enddo + + ! =============== + ! Dynamical Step + ! =============== + call t_startf("prim_step_dyn") + n_Q = tl%n0 ! n_Q = timelevel of FV tracers at time t. need to save this + ! FV tracers still carry 3 timelevels + ! SE tracers only carry 2 timelevels + call prim_advance_exp(elem, deriv1, hvcoord, & + hybrid, dt, tl, nets, nete, compute_diagnostics) + do n=2,qsplit + call TimeLevel_update(tl,"leapfrog") + call prim_advance_exp(elem, deriv1, hvcoord,hybrid, dt, tl, nets, nete, .false.) + ! defer final timelevel update until after Q update. + enddo + call t_stopf("prim_step_dyn") + + ! current dynamics state variables: + ! derived%dp = dp at start of timestep + ! derived%vstar = velocity at start of tracer timestep + ! derived%vn0 = mean horiz. flux: U*dp + ! state%dp3d(:,:,:,np1) = dp3d + ! rsplit=0 + ! state%v(:,:,:,np1) = velocity on reference levels + ! rsplit>0 + ! state%v(:,:,:,np1) = velocity on lagrangian levels + ! + ! Tracer Advection. + ! in addition, this routine will apply the DSS to: + ! derived%eta_dot_dpdn = mean vertical velocity (used for remap below) + ! derived%omega = + ! Tracers are always vertically lagrangian. + ! For rsplit=0: + ! if tracer scheme needs v on lagrangian levels it has to vertically interpolate + ! if tracer scheme needs dp3d, it needs to derive it from ps_v + + call t_startf("prim_step_advec") + if (qsize > 0) then + call t_startf("PAT_remap") + call Prim_Advec_Tracers_remap(elem, deriv1,hvcoord,hybrid,dt_q,tl,nets,nete) + call t_stopf("PAT_remap") + end if + call t_stopf("prim_step_advec") + + end subroutine prim_step + +end module diff --git a/components/homme/src/preqx_acc/solver_init_mod.F90 b/components/homme/src/preqx_acc/solver_init_mod.F90 index 85dfec78b59a..0034fdc987cc 100644 --- a/components/homme/src/preqx_acc/solver_init_mod.F90 +++ b/components/homme/src/preqx_acc/solver_init_mod.F90 @@ -4,7 +4,7 @@ module solver_init_mod !OVERWRITING: solver_init2 - use solver_init_mod_base, only: + use solver_init_base, only: use dimensions_mod, only: nelemd implicit none private diff --git a/components/homme/src/preqx_acc/viscosity_mod.F90 b/components/homme/src/preqx_acc/viscosity_mod.F90 index 640be444267e..79dc80d9de4e 100644 --- a/components/homme/src/preqx_acc/viscosity_mod.F90 +++ b/components/homme/src/preqx_acc/viscosity_mod.F90 @@ -4,8 +4,8 @@ #endif module viscosity_mod - use viscosity_mod_base, only: biharmonic_wk, compute_zeta_C0, compute_div_C0, compute_zeta_C0_contra, compute_div_C0_contra, make_c0, make_c0_vector - use viscosity_mod_base, only: biharmonic_wk_scalar,neighbor_minmax, neighbor_minmax_start,neighbor_minmax_finish, biharmonic_wk_dp3d + use viscosity_base, only: biharmonic_wk, compute_zeta_C0, compute_div_C0, compute_zeta_C0_contra, compute_div_C0_contra, make_c0, make_c0_vector + use viscosity_base, only: biharmonic_wk_scalar,neighbor_minmax, neighbor_minmax_start,neighbor_minmax_finish, biharmonic_wk_dp3d, smooth_phis use thread_mod, only : omp_get_num_threads use kinds, only : real_kind, iulog use dimensions_mod, only : np, nlev,qsize,nelemd @@ -22,8 +22,7 @@ module viscosity_mod public :: biharmonic_wk, compute_zeta_C0, compute_div_C0, compute_zeta_C0_contra, compute_div_C0_contra, make_c0, make_c0_vector public :: biharmonic_wk_scalar, neighbor_minmax, neighbor_minmax_start,neighbor_minmax_finish, biharmonic_wk_dp3d public :: biharmonic_wk_scalar_openacc - public :: neighbor_minmax_openacc - + public :: neighbor_minmax_openacc, smooth_phis contains diff --git a/components/homme/src/prim/CMakeLists.txt b/components/homme/src/prim/CMakeLists.txt index 2ce2ca94711b..59b1ed82ac79 100644 --- a/components/homme/src/prim/CMakeLists.txt +++ b/components/homme/src/prim/CMakeLists.txt @@ -17,8 +17,12 @@ SET(UTILS_TRILINOS_DIR ${HOMME_SOURCE_DIR}/utils/trilinos) SET (EXEC_INCLUDE_DIRS ${PIO_INCLUDE_DIRS} ${UTILS_TIMING_DIR} ) +FILE(GLOB SB81_SRC_F90 ${SRC_SHARE_DIR}/sb81/*.F90) + # Make SRCS global so the tests can access it -SET(PRIM_SRCS ${SRC_SHARE_DIR}/kinds.F90 +SET(PRIM_SRCS + ${SB81_SRC_F90} + ${SRC_SHARE_DIR}/kinds.F90 ${SRC_DIR}/test_src/asp_tests.F90 ${SRC_DIR}/test_src/baroclinic_inst_mod.F90 ${SRC_SHARE_DIR}/dimensions_mod.F90 @@ -79,9 +83,8 @@ SET(PRIM_SRCS ${SRC_SHARE_DIR}/kinds.F90 ${SRC_SHARE_DIR}/prim_advance_mod.F90 ${SRC_SHARE_DIR}/prim_advection_mod.F90 ${SRC_SHARE_DIR}/prim_implicit_mod.F90 - ${SRC_DIR}/forcing_mod.F90 ${SRC_SHARE_DIR}/metis_mod.F90 - ${SRC_SHARE_DIR}/prim_driver_mod.F90 + ${SRC_SHARE_DIR}/prim_driver_base.F90 ${SRC_DIR}/prim_movie_mod.F90 ${SRC_DIR}/surfaces_mod.F90 ${SRC_DIR}/repro_sum_mod.F90 diff --git a/components/homme/src/prim_main.F90 b/components/homme/src/prim_main.F90 index 054129130c67..78920479f1d5 100644 --- a/components/homme/src/prim_main.F90 +++ b/components/homme/src/prim_main.F90 @@ -24,6 +24,10 @@ program prim_main use restart_io_mod , only: restartheader_t, writerestart use hybrid_mod, only: hybrid_create +#ifdef VERTICAL_INTERPOLATION + use netcdf_interp_mod, only: netcdf_interp_init, netcdf_interp_write, netcdf_interp_finish +#endif + #ifdef PIO_INTERP use interp_movie_mod, only : interp_movie_output, interp_movie_finish, interp_movie_init use interpolate_driver_mod, only : interpolate_driver @@ -178,9 +182,11 @@ program prim_main #endif -#ifdef PIO_INTERP - ! initialize history files. filename constructed with restart time - ! so we have to do this after ReadRestart in prim_init2 above +! initialize history files. filename constructed with restart time +! so we have to do this after ReadRestart in prim_init2 above +#ifdef VERTICAL_INTERPOLATION + call netcdf_interp_init(elem, hybrid, hvcoord) +#elif defined PIO_INTERP call interp_movie_init( elem, par, hvcoord, tl ) #else call prim_movie_init( elem, par, hvcoord, tl ) @@ -189,7 +195,10 @@ program prim_main ! output initial state for NEW runs (not restarts or branch runs) if (runtype == 0 ) then -#ifdef PIO_INTERP + +#ifdef VERTICAL_INTERPOLATION + call netcdf_interp_write(elem, tl, hybrid, hvcoord) +#elif defined PIO_INTERP call interp_movie_output(elem, tl, par, 0d0, hvcoord=hvcoord) #else call prim_movie_output(elem, tl, hvcoord, par) @@ -219,7 +228,9 @@ program prim_main !$OMP END PARALLEL #endif -#ifdef PIO_INTERP +#ifdef VERTICAL_INTERPOLATION + call netcdf_interp_write(elem, tl, hybrid, hvcoord) +#elif defined PIO_INTERP call interp_movie_output(elem, tl, par, 0d0,hvcoord=hvcoord) #else call prim_movie_output(elem, tl, hvcoord, par) @@ -237,7 +248,10 @@ program prim_main if(par%masterproc) print *,"Finished main timestepping loop",tl%nstep call prim_finalize() if(par%masterproc) print *,"closing history files" -#ifdef PIO_INTERP + +#ifdef VERTICAL_INTERPOLATION + call netcdf_interp_finish +#elif defined PIO_INTERP call interp_movie_finish #else call prim_movie_finish diff --git a/components/homme/src/prim_restart_mod.F90 b/components/homme/src/prim_restart_mod.F90 index a3e5606e284f..9b92b805aa89 100644 --- a/components/homme/src/prim_restart_mod.F90 +++ b/components/homme/src/prim_restart_mod.F90 @@ -10,7 +10,7 @@ module prim_restart_mod !------------------ use parallel_mod, only : parallel_t, MPIreal_t, abortmp !------------------ - use element_state, only : elem_state_t, StateComponents + use element_state, only : elem_state_t !------------------ use restart_io_mod, only : nwordsRestartBuffer_t, RestartBuffer, File_elem_t, & StateDesc_t, createstatedescriptor, AddStateField, constructelementfile, & @@ -71,8 +71,8 @@ subroutine initRestartFile(state, par,File) !================================================================ ! Construct the descriptor of the state variable for MPI I/O !================================================================ - NumComponents = StateComponents ! number of variable components in the state buffer + NumComponents = 7 ! THIS NUMBER MUST MATCH NUMBER OF TIMES AddStateField is called below RestDesc = CreateStateDescriptor(NumComponents) type = MPIReal_t ! All the types are Real diff --git a/components/homme/src/share/prim_advection_mod_base.F90 b/components/homme/src/share/prim_advection_base.F90 similarity index 99% rename from components/homme/src/share/prim_advection_mod_base.F90 rename to components/homme/src/share/prim_advection_base.F90 index 571711132e44..614305e786a9 100644 --- a/components/homme/src/share/prim_advection_mod_base.F90 +++ b/components/homme/src/share/prim_advection_base.F90 @@ -35,7 +35,7 @@ ! remap Qdp* to Qdp(t+1) [ dp_star(t+1) -> dp(t+1) ] -module prim_advection_mod_base +module prim_advection_base ! ! total divergence formulation: ! d/dt[dp/dn Q] + div( U dp/dn Q ) + d/dn ( eta_dot dp/dn Q ) = 0 @@ -739,8 +739,4 @@ subroutine advance_hypervis_scalar( edgeAdv , elem , hvcoord , hybrid , deriv , end subroutine advance_hypervis_scalar - - - - -end module prim_advection_mod_base +end module prim_advection_base diff --git a/components/homme/src/share/prim_driver_mod.F90 b/components/homme/src/share/prim_driver_base.F90 similarity index 63% rename from components/homme/src/share/prim_driver_mod.F90 rename to components/homme/src/share/prim_driver_base.F90 index 5c6b19897fe1..a6b2bae5e3cf 100644 --- a/components/homme/src/share/prim_driver_mod.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -2,7 +2,7 @@ #include "config.h" #endif -module prim_driver_mod +module prim_driver_base use cg_mod, only: cg_t use derivative_mod, only: derivative_t, derivinit @@ -20,88 +20,56 @@ module prim_driver_mod use thread_mod, only: nThreadsHoriz, omp_get_num_threads #ifndef CAM - use column_types_mod, only : ColumnModel_t use prim_restart_mod, only : initrestartfile use restart_io_mod , only : RestFile,readrestart - use test_mod, only: set_test_initial_conditions, apply_test_forcing + use test_mod, only: set_test_initial_conditions, compute_test_forcing #endif implicit none private - public :: prim_init1, prim_init2 , prim_run_subcycle, prim_finalize - public :: smooth_topo_datasets + public :: prim_init1, prim_init2, prim_finalize, smooth_topo_datasets type (cg_t), allocatable :: cg(:) ! conjugate gradient struct (nthreads) type (quadrature_t) :: gp ! element GLL points - -#ifndef CAM - type (ColumnModel_t), allocatable :: cm(:) ! (nthreads) -#endif type (ReductionBuffer_ordered_1d_t), save :: red ! reduction buffer (shared) - type (derivative_t) :: deriv1 + type (derivative_t), public :: deriv1 contains subroutine prim_init1(elem, par, dom_mt, Tl) - ! -------------------------------- - use thread_mod, only : nthreads, omp_get_thread_num - ! -------------------------------- - use control_mod, only : runtype, restartfreq, integration, topology, & - partmethod, use_semi_lagrange_transport - ! -------------------------------- - use prim_state_mod, only : prim_printstate_init - ! -------------------------------- - use namelist_mod, only : readnl - ! -------------------------------- - use mesh_mod, only : MeshUseMeshFile - ! -------------------------------- - use time_mod, only : nmax, time_at, timelevel_init, timelevel_t - ! -------------------------------- + use thread_mod, only : nthreads, omp_get_thread_num + use control_mod, only : runtype, restartfreq, integration, topology, partmethod, use_semi_lagrange_transport + use prim_state_mod, only : prim_printstate_init + use namelist_mod, only : readnl + use mesh_mod, only : MeshUseMeshFile + use time_mod, only : nmax, time_at, timelevel_init, timelevel_t use mass_matrix_mod, only : mass_matrix - ! -------------------------------- - use cube_mod, only : cubeedgecount , cubeelemcount, cubetopology - ! -------------------------------- - use mesh_mod, only : MeshSetCoordinates, MeshUseMeshFile, MeshCubeTopology, & - MeshCubeElemCount, MeshCubeEdgeCount - use cube_mod, only : cube_init_atomic, set_corner_coordinates, assign_node_numbers_to_elem - ! -------------------------------- - use metagraph_mod, only : metavertex_t, metaedge_t, localelemcount, initmetagraph, printmetavertex - ! -------------------------------- - use gridgraph_mod, only : gridvertex_t, gridedge_t, allocate_gridvertex_nbrs, deallocate_gridvertex_nbrs - ! -------------------------------- - use schedtype_mod, only : schedule - ! -------------------------------- - use schedule_mod, only : genEdgeSched, PrintSchedule - ! -------------------------------- + use cube_mod, only : cubeedgecount , cubeelemcount, cubetopology + use mesh_mod, only : MeshSetCoordinates, MeshUseMeshFile, MeshCubeTopology, MeshCubeElemCount, MeshCubeEdgeCount + use cube_mod, only : cube_init_atomic, set_corner_coordinates, assign_node_numbers_to_elem + use metagraph_mod, only : metavertex_t, metaedge_t, localelemcount, initmetagraph, printmetavertex + use gridgraph_mod, only : gridvertex_t, gridedge_t, allocate_gridvertex_nbrs, deallocate_gridvertex_nbrs + use schedtype_mod, only : schedule + use schedule_mod, only : genEdgeSched, PrintSchedule use prim_advection_mod, only: prim_advec_init1 - ! -------------------------------- - use prim_advance_mod, only: prim_advance_init - ! -------------------------------- + use prim_advance_mod, only: prim_advance_init + #ifdef TRILINOS use prim_implicit_mod, only : prim_implicit_init #endif - ! -------------------------------- use parallel_mod, only : iam, parallel_t, syncmp, abortmp, global_shared_buf, nrepro_vars #ifdef _MPI use parallel_mod, only : mpiinteger_t, mpireal_t, mpi_max, mpi_sum, haltmp #endif - ! -------------------------------- use metis_mod, only : genmetispart - ! -------------------------------- use spacecurve_mod, only : genspacepart - ! -------------------------------- use dof_mod, only : global_dof, CreateUniqueIndex, SetElemOffset - ! -------------------------------- use params_mod, only : SFCURVE - ! -------------------------------- use domain_mod, only : domain1d_t, decompose - ! -------------------------------- use physical_constants, only : dd_pi - ! -------------------------------- use bndry_mod, only : sort_neighbor_buffer_mapping - ! -------------------------------- #ifndef CAM use repro_sum_mod, only: repro_sum, repro_sum_defaultopts, repro_sum_setopts #else @@ -460,9 +428,7 @@ subroutine prim_init1(elem, par, dom_mt, Tl) nete=nelemd ! set the actual number of threads which will be used in the horizontal nThreadsHoriz = n_domains -#ifndef CAM - allocate(cm(0:n_domains-1)) -#endif + allocate(cg(0:n_domains-1)) call prim_advance_init(par,elem,integration) #ifdef TRILINOS @@ -493,14 +459,14 @@ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) use parallel_mod, only: parallel_t, haltmp, syncmp, abortmp use prim_state_mod, only: prim_printstate, prim_diag_scalars use prim_si_ref_mod, only: prim_si_refstate_init, prim_set_mass + use prim_advance_mod, only: prim_advance_init2 use prim_advection_mod, only: prim_advec_init2 use solver_init_mod, only: solver_init2 use time_mod, only: timelevel_t, tstep, phys_tscale, timelevel_init, nendstep, smooth, nsplit, TimeLevel_Qdp use thread_mod, only: nthreads #ifndef CAM - use column_model_mod, only: InitColumnModel - use control_mod, only: pertlim + use control_mod, only: pertlim #endif #ifdef TRILINOS @@ -652,12 +618,13 @@ end subroutine noxinit #ifndef CAM + ! initialize dynamics solver before setting initial conditions + call prim_advance_init2(elem, nets, nete, hybrid, hvcoord) + ! ================================= ! HOMME stand alone initialization ! ================================= - call InitColumnModel(elem, cm(hybrid%ithr), hvcoord, hybrid, tl,nets,nete,runtype) - if(runtype >= 1) then ! =========================================================== @@ -798,347 +765,12 @@ end subroutine noxinit call prim_printstate(elem, tl, hybrid,hvcoord,nets,nete) call solver_init2(elem(:), deriv1) - call Prim_Advec_Init2(elem(:), hvcoord, hybrid) + call prim_advec_init2(elem(:), hvcoord, hybrid) end subroutine prim_init2 !=======================================================================================================! - - - subroutine prim_run_subcycle(elem, hybrid,nets,nete, dt, tl, hvcoord,nsubstep) - - ! advance dynamic variables and tracers (u,v,T,ps,Q,C) from time t to t + dt_q - ! - ! input: - ! tl%nm1 not used - ! tl%n0 data at time t - ! tl%np1 new values at t+dt_q - ! - ! then we update timelevel pointers: - ! tl%nm1 = tl%n0 - ! tl%n0 = tl%np1 - ! so that: - ! tl%nm1 tracers: t dynamics: t+(qsplit-1)*dt - ! tl%n0 time t + dt_q - - use control_mod, only: statefreq, ftype, qsplit, rsplit, disable_diagnostics - use hybvcoord_mod, only: hvcoord_t - use parallel_mod, only: abortmp - use prim_advance_mod, only: applycamforcing, applycamforcing_dynamics - use prim_state_mod, only: prim_printstate, prim_diag_scalars, prim_energy_halftimes - use vertremap_mod, only: vertical_remap - use reduction_mod, only: parallelmax - use time_mod, only: TimeLevel_t, timelevel_update, timelevel_qdp, nsplit - -#if USE_OPENACC - use openacc_utils_mod, only: copy_qdp_h2d, copy_qdp_d2h -#endif - - type (element_t) , intent(inout) :: elem(:) - type (hybrid_t), intent(in) :: hybrid ! distributed parallel structure (shared) - type (hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct - integer, intent(in) :: nets ! starting thread element number (private) - integer, intent(in) :: nete ! ending thread element number (private) - real(kind=real_kind), intent(in) :: dt ! "timestep dependent" timestep - type (TimeLevel_t), intent(inout) :: tl - integer, intent(in) :: nsubstep ! nsubstep = 1 .. nsplit - - real(kind=real_kind) :: dp, dt_q, dt_remap - real(kind=real_kind) :: dp_np1(np,np) - integer :: ie,i,j,k,n,q,t - integer :: n0_qdp,np1_qdp,r,nstep_end - logical :: compute_diagnostics, compute_energy - - ! compute timesteps for tracer transport and vertical remap - - dt_q = dt*qsplit - dt_remap = dt_q - nstep_end = tl%nstep + qsplit - if (rsplit>0) then - dt_remap = dt_q*rsplit ! rsplit=0 means use eulerian code, not vert. lagrange - nstep_end = tl%nstep + qsplit*rsplit ! nstep at end of this routine - endif - - ! activate diagnostics periodically for display to stdout - compute_energy = .false. - compute_diagnostics = .false. - if (MODULO(nstep_end,statefreq)==0 .or. nstep_end==tl%nstep0) then - compute_diagnostics= .true. - compute_energy = .true. - endif - if(disable_diagnostics) compute_diagnostics= .false. - - ! compute scalar diagnostics if currently active - if (compute_diagnostics) then - call t_startf("prim_diag_scalars") - call prim_diag_scalars(elem,hvcoord,tl,4,.true.,nets,nete) - call t_stopf("prim_diag_scalars") - endif - -#ifdef CAM - - ! Apply CAM Physics forcing - - ! ftype= 2: Q was adjusted by physics, but apply u,T forcing here - ! ftype= 1: forcing was applied time-split in CAM coupling layer - ! ftype= 0: apply all forcing here - ! ftype=-1: do not apply forcing - - call TimeLevel_Qdp(tl, qsplit, n0_qdp) - - if (ftype==0) then - call t_startf("ApplyCAMForcing") - call ApplyCAMForcing(elem, hvcoord,tl%n0,n0_qdp, dt_remap,nets,nete) - call t_stopf("ApplyCAMForcing") - - elseif (ftype==2) then - call t_startf("ApplyCAMForcing_dynamics") - call ApplyCAMForcing_dynamics(elem, hvcoord,tl%n0,dt_remap,nets,nete) - call t_stopf("ApplyCAMForcing_dynamics") - endif -#else - ! Apply HOMME test case forcing - call apply_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete) - -#endif - - ! E(1) Energy after CAM forcing - if (compute_energy) then - call t_startf("prim_energy_halftimes") - call prim_energy_halftimes(elem,hvcoord,tl,1,.true.,nets,nete) - call t_stopf("prim_energy_halftimes") - endif - - ! qmass and variance, using Q(n0),Qdp(n0) - if (compute_diagnostics) then - call t_startf("prim_diag_scalars") - call prim_diag_scalars(elem,hvcoord,tl,1,.true.,nets,nete) - call t_stopf("prim_diag_scalars") - endif - - ! initialize dp3d from ps - do ie=nets,nete - do k=1,nlev - elem(ie)%state%dp3d(:,:,k,tl%n0)=& - ( hvcoord%hyai(k+1) - hvcoord%hyai(k) )*hvcoord%ps0 + & - ( hvcoord%hybi(k+1) - hvcoord%hybi(k) )*elem(ie)%state%ps_v(:,:,tl%n0) - enddo - enddo - - -#if (USE_OPENACC) - call TimeLevel_Qdp( tl, qsplit, n0_qdp, np1_qdp) - call t_startf("copy_qdp_h2d") - call copy_qdp_h2d( elem , n0_qdp ) - call t_stopf("copy_qdp_h2d") -#endif - - ! Loop over rsplit vertically lagrangian timesteps - call t_startf("prim_step_rX") - call prim_step(elem, hybrid,nets,nete, dt, tl, hvcoord,compute_diagnostics,1) - call t_stopf("prim_step_rX") - - do r=2,rsplit - call TimeLevel_update(tl,"leapfrog") - call t_startf("prim_step_rX") - call prim_step(elem, hybrid,nets,nete, dt, tl, hvcoord,.false.,r) - call t_stopf("prim_step_rX") - enddo - ! defer final timelevel update until after remap and diagnostics - -#if (USE_OPENACC) - call TimeLevel_Qdp( tl, qsplit, n0_qdp, np1_qdp) - call t_startf("copy_qdp_h2d") - call copy_qdp_d2h( elem , np1_qdp ) - call t_stopf("copy_qdp_h2d") -#endif - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! apply vertical remap - ! always for tracers - ! if rsplit>0: also remap dynamics and compute reference level ps_v - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !compute timelevels for tracers (no longer the same as dynamics) - call TimeLevel_Qdp( tl, qsplit, n0_qdp, np1_qdp) - call vertical_remap(hybrid,elem,hvcoord,dt_remap,tl%np1,np1_qdp,nets,nete) - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! time step is complete. update some diagnostic variables: - ! Q (mixing ratio) - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - call t_startf("prim_run_subcyle_diags") - do ie=nets,nete -#if (defined COLUMN_OPENMP) - !$omp parallel do default(shared), private(k,q,dp_np1) -#endif - do k=1,nlev ! Loop inversion (AAM) - dp_np1(:,:) = ( hvcoord%hyai(k+1) - hvcoord%hyai(k) )*hvcoord%ps0 + & - ( hvcoord%hybi(k+1) - hvcoord%hybi(k) )*elem(ie)%state%ps_v(:,:,tl%np1) - !dir$ simd - do q=1,qsize - elem(ie)%state%Q(:,:,k,q)=elem(ie)%state%Qdp(:,:,k,q,np1_qdp)/dp_np1(:,:) - enddo - enddo - enddo - call t_stopf("prim_run_subcyle_diags") - - ! now we have: - ! u(nm1) dynamics at t+dt_remap - 2*dt - ! u(n0) dynamics at t+dt_remap - dt - ! u(np1) dynamics at t+dt_remap - ! - ! Q(1) Q at t+dt_remap - if (compute_diagnostics) then - call t_startf("prim_diag_scalars") - call prim_diag_scalars(elem,hvcoord,tl,2,.false.,nets,nete) - call t_stopf("prim_diag_scalars") - endif - if (compute_energy) then - call t_startf("prim_energy_halftimes") - call prim_energy_halftimes(elem,hvcoord,tl,2,.false.,nets,nete) - call t_stopf("prim_energy_halftimes") - endif - - if (compute_diagnostics) then - call t_startf("prim_diag_scalars") - call prim_diag_scalars(elem,hvcoord,tl,3,.false.,nets,nete) - call t_stopf("prim_diag_scalars") - - call t_startf("prim_energy_halftimes") - call prim_energy_halftimes(elem,hvcoord,tl,3,.false.,nets,nete) - call t_stopf("prim_energy_halftimes") - endif - - ! ================================= - ! update dynamics time level pointers - ! ================================= - call TimeLevel_update(tl,"leapfrog") - ! now we have: - ! u(nm1) dynamics at t+dt_remap - dt - ! u(n0) dynamics at t+dt_remap - ! u(np1) undefined - - ! ============================================================ - ! Print some diagnostic information - ! ============================================================ - if (compute_diagnostics) then - call prim_printstate(elem, tl, hybrid,hvcoord,nets,nete) - end if - end subroutine prim_run_subcycle - - - - subroutine prim_step(elem, hybrid,nets,nete, dt, tl, hvcoord, compute_diagnostics,rstep) - ! - ! Take qsplit dynamics steps and one tracer step - ! for vertically lagrangian option, this subroutine does only the horizontal step - ! - ! input: - ! tl%nm1 not used - ! tl%n0 data at time t - ! tl%np1 new values at t+dt_q - ! - ! then we update timelevel pointers: - ! tl%nm1 = tl%n0 - ! tl%n0 = tl%np1 - ! so that: - ! tl%nm1 tracers: t dynamics: t+(qsplit-1)*dt - ! tl%n0 time t + dt_q - ! - ! - use control_mod, only: statefreq, integration, ftype, qsplit, nu_p, rsplit - use control_mod, only: use_semi_lagrange_transport - use hybvcoord_mod, only : hvcoord_t - use parallel_mod, only: abortmp - use prim_advance_mod, only: prim_advance_exp - use prim_advection_mod, only: prim_advec_tracers_remap - use reduction_mod, only: parallelmax - use time_mod, only: time_at,TimeLevel_t, timelevel_update, nsplit - - type(element_t), intent(inout) :: elem(:) - type(hybrid_t), intent(in) :: hybrid ! distributed parallel structure (shared) - type(hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct - integer, intent(in) :: nets ! starting thread element number (private) - integer, intent(in) :: nete ! ending thread element number (private) - real(kind=real_kind), intent(in) :: dt ! "timestep dependent" timestep - type(TimeLevel_t), intent(inout) :: tl - integer, intent(in) :: rstep ! vertical remap subcycling step - - real(kind=real_kind) :: st, st1, dp, dt_q - integer :: ie, t, q,k,i,j,n, n_Q - real (kind=real_kind) :: maxcflx, maxcfly - real (kind=real_kind) :: dp_np1(np,np) - logical :: compute_diagnostics - - dt_q = dt*qsplit - - ! =============== - ! initialize mean flux accumulation variables and save some variables at n0 - ! for use by advection - ! =============== - do ie=nets,nete - elem(ie)%derived%eta_dot_dpdn=0 ! mean vertical mass flux - elem(ie)%derived%vn0=0 ! mean horizontal mass flux - elem(ie)%derived%omega_p=0 - if (nu_p>0) then - elem(ie)%derived%dpdiss_ave=0 - elem(ie)%derived%dpdiss_biharmonic=0 - endif - if (use_semi_lagrange_transport) then - elem(ie)%derived%vstar=elem(ie)%state%v(:,:,:,:,tl%n0) - end if - elem(ie)%derived%dp(:,:,:)=elem(ie)%state%dp3d(:,:,:,tl%n0) - enddo - - ! =============== - ! Dynamical Step - ! =============== - call t_startf("prim_step_dyn") - n_Q = tl%n0 ! n_Q = timelevel of FV tracers at time t. need to save this - ! FV tracers still carry 3 timelevels - ! SE tracers only carry 2 timelevels - call prim_advance_exp(elem, deriv1, hvcoord, & - hybrid, dt, tl, nets, nete, compute_diagnostics) - do n=2,qsplit - call TimeLevel_update(tl,"leapfrog") - call prim_advance_exp(elem, deriv1, hvcoord,hybrid, dt, tl, nets, nete, .false.) - ! defer final timelevel update until after Q update. - enddo - call t_stopf("prim_step_dyn") - - ! current dynamics state variables: - ! derived%dp = dp at start of timestep - ! derived%vstar = velocity at start of tracer timestep - ! derived%vn0 = mean horiz. flux: U*dp - ! state%dp3d(:,:,:,np1) = dp3d - ! rsplit=0 - ! state%v(:,:,:,np1) = velocity on reference levels - ! rsplit>0 - ! state%v(:,:,:,np1) = velocity on lagrangian levels - ! - ! Tracer Advection. - ! in addition, this routine will apply the DSS to: - ! derived%eta_dot_dpdn = mean vertical velocity (used for remap below) - ! derived%omega = - ! Tracers are always vertically lagrangian. - ! For rsplit=0: - ! if tracer scheme needs v on lagrangian levels it has to vertically interpolate - ! if tracer scheme needs dp3d, it needs to derive it from ps_v - call t_startf("prim_step_advec") - if (qsize > 0) then - call t_startf("PAT_remap") - call Prim_Advec_Tracers_remap(elem, deriv1,hvcoord,hybrid,dt_q,tl,nets,nete) - call t_stopf("PAT_remap") - end if - call t_stopf("prim_step_advec") - - end subroutine prim_step - - -!=======================================================================================================! - - subroutine prim_finalize() #ifdef TRILINOS @@ -1157,15 +789,12 @@ end subroutine noxfinish ! ========================== end subroutine prim_finalize - - - subroutine smooth_topo_datasets(phis,sghdyn,sgh30dyn,elem,hybrid,nets,nete) + subroutine smooth_topo_datasets(phis,sghdyn,sgh30dyn,elem,hybrid,nets,nete) use control_mod, only : smooth_phis_numcycle,smooth_sgh_numcycle use hybrid_mod, only : hybrid_t use bndry_mod, only : bndry_exchangev use derivative_mod, only : derivative_t , laplace_sphere_wk - use viscosity_mod, only : biharmonic_wk - use prim_advance_mod, only : smooth_phis + use viscosity_mod, only : smooth_phis implicit none integer , intent(in) :: nets,nete @@ -1193,7 +822,8 @@ subroutine smooth_topo_datasets(phis,sghdyn,sgh30dyn,elem,hybrid,nets,nete) end subroutine smooth_topo_datasets -end module prim_driver_mod + +end module prim_driver_base diff --git a/components/homme/src/share/prim_state_mod.F90 b/components/homme/src/share/prim_state_mod.F90 index 8da7cdcebb88..ca5db3dc7282 100644 --- a/components/homme/src/share/prim_state_mod.F90 +++ b/components/homme/src/share/prim_state_mod.F90 @@ -200,8 +200,8 @@ subroutine prim_printstate(elem, tl,hybrid,hvcoord,nets,nete) umax_local(ie) = MAXVAL(elem(ie)%state%v(:,:,1,:,n0)) vmax_local(ie) = MAXVAL(elem(ie)%state%v(:,:,2,:,n0)) - fumax_local(ie) = MAXVAL(elem(ie)%derived%FM(:,:,1,:,pnm1)) - fvmax_local(ie) = MAXVAL(elem(ie)%derived%FM(:,:,2,:,pnm1)) + fumax_local(ie) = MAXVAL(elem(ie)%derived%FM(:,:,1,:)) + fvmax_local(ie) = MAXVAL(elem(ie)%derived%FM(:,:,2,:)) tmax_local(ie) = MAXVAL(elem(ie)%state%T(:,:,:,n0)) @@ -209,24 +209,24 @@ subroutine prim_printstate(elem, tl,hybrid,hvcoord,nets,nete) dpmax_local(ie) = MAXVAL(elem(ie)%state%dp3d(:,:,:,n0)) psmax_local(ie) = MAXVAL(tmp(:,:,ie)) - ftmax_local(ie) = MAXVAL(elem(ie)%derived%FT(:,:,:,pnm1)) - fqmax_local(ie) = MAXVAL(elem(ie)%derived%FQ(:,:,:,1,pnm1)) + ftmax_local(ie) = MAXVAL(elem(ie)%derived%FT(:,:,:)) + fqmax_local(ie) = MAXVAL(elem(ie)%derived%FQ(:,:,:,1)) omegamax_local(ie) = MAXVAL(elem(ie)%derived%Omega_p(:,:,:)) !====================================================== umin_local(ie) = MINVAL(elem(ie)%state%v(:,:,1,:,n0)) vmin_local(ie) = MINVAL(elem(ie)%state%v(:,:,2,:,n0)) - Fumin_local(ie) = MINVAL(elem(ie)%derived%FM(:,:,1,:,pnm1)) - Fvmin_local(ie) = MINVAL(elem(ie)%derived%FM(:,:,2,:,pnm1)) + Fumin_local(ie) = MINVAL(elem(ie)%derived%FM(:,:,1,:)) + Fvmin_local(ie) = MINVAL(elem(ie)%derived%FM(:,:,2,:)) tmin_local(ie) = MINVAL(elem(ie)%state%T(:,:,:,n0)) if (rsplit>0) & dpmin_local(ie) = MINVAL(elem(ie)%state%dp3d(:,:,:,n0)) - Ftmin_local(ie) = MINVAL(elem(ie)%derived%FT(:,:,:,pnm1)) - Fqmin_local(ie) = MINVAL(elem(ie)%derived%FQ(:,:,:,1,pnm1)) + Ftmin_local(ie) = MINVAL(elem(ie)%derived%FT(:,:,:)) + Fqmin_local(ie) = MINVAL(elem(ie)%derived%FQ(:,:,:,1)) Omegamin_local(ie) = MINVAL(elem(ie)%derived%Omega_p(:,:,:)) @@ -235,8 +235,8 @@ subroutine prim_printstate(elem, tl,hybrid,hvcoord,nets,nete) usum_local(ie) = SUM(elem(ie)%state%v(:,:,1,:,n0)) vsum_local(ie) = SUM(elem(ie)%state%v(:,:,2,:,n0)) - Fusum_local(ie) = SUM(elem(ie)%derived%FM(:,:,1,:,pnm1)) - Fvsum_local(ie) = SUM(elem(ie)%derived%FM(:,:,2,:,pnm1)) + Fusum_local(ie) = SUM(elem(ie)%derived%FM(:,:,1,:)) + Fvsum_local(ie) = SUM(elem(ie)%derived%FM(:,:,2,:)) tsum_local(ie) = SUM(elem(ie)%state%t(:,:,:,n0)) if (rsplit>0) then @@ -247,8 +247,8 @@ subroutine prim_printstate(elem, tl,hybrid,hvcoord,nets,nete) dpsum_local(ie) = 0.0D0 end if - Ftsum_local(ie) = SUM(elem(ie)%derived%FT(:,:,:,pnm1)) - FQsum_local(ie) = SUM(elem(ie)%derived%FQ(:,:,:,1,pnm1)) + Ftsum_local(ie) = SUM(elem(ie)%derived%FT(:,:,:)) + FQsum_local(ie) = SUM(elem(ie)%derived%FQ(:,:,:,1)) Omegasum_local(ie) = SUM(elem(ie)%derived%Omega_p(:,:,:)) pssum_local(ie) = SUM(tmp(:,:,ie)) diff --git a/components/homme/src/share/sb81/README.txt b/components/homme/src/share/sb81/README.txt new file mode 100644 index 000000000000..6c679b4483c8 --- /dev/null +++ b/components/homme/src/share/sb81/README.txt @@ -0,0 +1,2 @@ +This directory contains code using the mimetic finite difference method described in Simmons and Burridge 1981. + diff --git a/components/homme/src/share/prim_advance_mod.F90 b/components/homme/src/share/sb81/prim_advance_mod.F90 similarity index 91% rename from components/homme/src/share/prim_advance_mod.F90 rename to components/homme/src/share/sb81/prim_advance_mod.F90 index 29e7ac919872..f58ecfb37199 100644 --- a/components/homme/src/share/prim_advance_mod.F90 +++ b/components/homme/src/share/sb81/prim_advance_mod.F90 @@ -23,8 +23,7 @@ module prim_advance_mod implicit none private save - public :: prim_advance_exp, prim_advance_init, & - applyCAMforcing_dynamics, applyCAMforcing, smooth_phis + public :: prim_advance_exp, prim_advance_init, prim_advance_init2, applyCAMforcing_dynamics, applyCAMforcing type (EdgeBuffer_t) :: edge1 type (EdgeBuffer_t) :: edge2 @@ -36,15 +35,15 @@ module prim_advance_mod contains + !_____________________________________________________________________ subroutine prim_advance_init(par, elem,integration) use edge_mod, only : initEdgeBuffer implicit none type (parallel_t) :: par type (element_t), intent(inout), target :: elem(:) - character(len=*) , intent(in) :: integration - integer :: i - integer :: ie + character(len=*), intent(in) :: integration + integer :: i, ie call initEdgeBuffer(par,edge3p1,elem,4*nlev) @@ -65,6 +64,18 @@ subroutine prim_advance_init(par, elem,integration) end subroutine prim_advance_init + !_____________________________________________________________________ + subroutine prim_advance_init2(elem, nets, nete, hybrid, hvcoord) + + ! additional solver specific initializations (called from prim_init2) + + type (element_t), intent(inout), target :: elem(:) ! array of element_t structures + integer, intent(in) :: nets,nete ! start and end element indices + type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct + type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coord data struct + + end subroutine + #ifndef CAM !_____________________________________________________________________ subroutine set_prescribed_wind(elem,deriv,hybrid,hv,dt,tl,nets,nete,eta_ave_w) @@ -608,7 +619,7 @@ subroutine applyCAMforcing(elem,hvcoord,np1,np1_qdp,dt_q,nets,nete) do ie=nets,nete ! apply forcing to Qdp - elem(ie)%derived%FQps(:,:,1)=0 + elem(ie)%derived%FQps(:,:)=0 #if (defined COLUMN_OPENMP) !$omp parallel do private(q,k,i,j,v1) #endif @@ -616,7 +627,7 @@ subroutine applyCAMforcing(elem,hvcoord,np1,np1_qdp,dt_q,nets,nete) do k=1,nlev do j=1,np do i=1,np - v1 = dt_q*elem(ie)%derived%FQ(i,j,k,q,1) + v1 = dt_q*elem(ie)%derived%FQ(i,j,k,q) !if (elem(ie)%state%Qdp(i,j,k,q,np1) + v1 < 0 .and. v1<0) then if (elem(ie)%state%Qdp(i,j,k,q,np1_qdp) + v1 < 0 .and. v1<0) then !if (elem(ie)%state%Qdp(i,j,k,q,np1) < 0 ) then @@ -630,7 +641,7 @@ subroutine applyCAMforcing(elem,hvcoord,np1,np1_qdp,dt_q,nets,nete) !elem(ie)%state%Qdp(i,j,k,q,np1) = elem(ie)%state%Qdp(i,j,k,q,np1)+v1 elem(ie)%state%Qdp(i,j,k,q,np1_qdp) = elem(ie)%state%Qdp(i,j,k,q,np1_qdp)+v1 if (q==1) then - elem(ie)%derived%FQps(i,j,1)=elem(ie)%derived%FQps(i,j,1)+v1/dt_q + elem(ie)%derived%FQps(i,j)=elem(ie)%derived%FQps(i,j)+v1/dt_q endif enddo enddo @@ -640,7 +651,7 @@ subroutine applyCAMforcing(elem,hvcoord,np1,np1_qdp,dt_q,nets,nete) if (use_moisture) then ! to conserve dry mass in the precese of Q1 forcing: elem(ie)%state%ps_v(:,:,np1) = elem(ie)%state%ps_v(:,:,np1) + & - dt_q*elem(ie)%derived%FQps(:,:,1) + dt_q*elem(ie)%derived%FQps(:,:) endif #if 0 @@ -690,8 +701,8 @@ subroutine applyCAMforcing(elem,hvcoord,np1,np1_qdp,dt_q,nets,nete) enddo enddo - elem(ie)%state%T(:,:,:,np1) = elem(ie)%state%T(:,:,:,np1) + dt_q*elem(ie)%derived%FT(:,:,:,1) - elem(ie)%state%v(:,:,:,:,np1) = elem(ie)%state%v(:,:,:,:,np1) + dt_q*elem(ie)%derived%FM(:,:,:,:,1) + elem(ie)%state%T(:,:,:,np1) = elem(ie)%state%T(:,:,:,np1) + dt_q*elem(ie)%derived%FT(:,:,:) + elem(ie)%state%v(:,:,:,:,np1) = elem(ie)%state%v(:,:,:,:,np1) + dt_q*elem(ie)%derived%FM(:,:,:,:) enddo end subroutine applyCAMforcing @@ -712,8 +723,8 @@ subroutine applyCAMforcing_dynamics(elem,hvcoord,np1,dt_q,nets,nete) real (kind=real_kind) :: v1,dp do ie=nets,nete - elem(ie)%state%T(:,:,:,np1) = elem(ie)%state%T(:,:,:,np1) + dt_q*elem(ie)%derived%FT(:,:,:,1) - elem(ie)%state%v(:,:,:,:,np1) = elem(ie)%state%v(:,:,:,:,np1) + dt_q*elem(ie)%derived%FM(:,:,:,:,1) + elem(ie)%state%T(:,:,:,np1) = elem(ie)%state%T(:,:,:,np1) + dt_q*elem(ie)%derived%FT(:,:,:) + elem(ie)%state%v(:,:,:,:,np1) = elem(ie)%state%v(:,:,:,:,np1) + dt_q*elem(ie)%derived%FM(:,:,:,:) enddo end subroutine applyCAMforcing_dynamics @@ -1036,20 +1047,19 @@ subroutine compute_and_apply_rhs(np1,nm1,n0,qn0,dt2,elem,hvcoord,hybrid,& ! ! ! =================================== - use kinds, only : real_kind + use kinds, only : real_kind use derivative_mod, only : derivative_t, divergence_sphere, gradient_sphere, vorticity_sphere use derivative_mod, only : subcell_div_fluxes, subcell_dss_fluxes - use edge_mod, only : edgevpack, edgevunpack, edgeDGVunpack - use edgetype_mod, only : edgedescriptor_t - use bndry_mod, only : bndry_exchangev - use control_mod, only : moisture, qsplit, use_cpstar, rsplit, swest - use hybvcoord_mod, only : hvcoord_t - + use edge_mod, only : edgevpack, edgevunpack, edgeDGVunpack + use edgetype_mod, only : edgedescriptor_t + use bndry_mod, only : bndry_exchangev + use control_mod, only : moisture, qsplit, use_cpstar, rsplit, swest + use hybvcoord_mod, only : hvcoord_t use physical_constants, only : cp, cpwater_vapor, Rgas, kappa - use physics_mod, only : virtual_specific_heat, virtual_temperature - use prim_si_mod, only : preq_vertadv, preq_omega_ps, preq_hydrostatic - - use time_mod, only : tevolve + use physics_mod, only : virtual_specific_heat, virtual_temperature + use prim_si_mod, only : preq_vertadv, preq_omega_ps, preq_hydrostatic + use time_mod, only : tevolve + use viscosity_base, only: smooth_phis implicit none integer, intent(in) :: np1,nm1,n0,qn0,nets,nete @@ -1570,163 +1580,7 @@ subroutine compute_and_apply_rhs(np1,nm1,n0,qn0,dt2,elem,hvcoord,hybrid,& end subroutine compute_and_apply_rhs - subroutine smooth_phis(phis,elem,hybrid,deriv,nets,nete,minf,numcycle) - use control_mod, only : smooth_phis_nudt, hypervis_scaling - use hybrid_mod, only : hybrid_t - use edge_mod, only : edgevpack, edgevunpack, edgevunpackmax, edgevunpackmin - use edgetype_mod, only : EdgeBuffer_t - use bndry_mod, only : bndry_exchangev - use derivative_mod, only : derivative_t , laplace_sphere_wk - use time_mod, only : TimeLevel_t - implicit none - - integer :: nets,nete - real (kind=real_kind), dimension(np,np,nets:nete), intent(inout) :: phis - type (hybrid_t) , intent(in) :: hybrid - type (element_t) , intent(inout), target :: elem(:) - type (derivative_t) , intent(in) :: deriv - real (kind=real_kind), intent(in) :: minf - integer, intent(in) :: numcycle - - ! local - real (kind=real_kind), dimension(np,np,nets:nete) :: pstens - real (kind=real_kind), dimension(nets:nete) :: pmin,pmax - real (kind=real_kind) :: mx,mn - integer :: nt,ie,ic,i,j,order,order_max, iuse - logical :: use_var_coef - - - ! compute local element neighbor min/max - do ie=nets,nete - pstens(:,:,ie)=minval(phis(:,:,ie)) - call edgeVpack(edge3p1,pstens(:,:,ie),1,0,ie) - enddo - - call t_startf('smooth_phis_bexchV1') - call bndry_exchangeV(hybrid,edge3p1) - call t_stopf('smooth_phis_bexchV1') - - do ie=nets,nete - call edgeVunpackMin(edge3p1, pstens(:,:,ie), 1, 0, ie) - pmin(ie)=minval(pstens(:,:,ie)) - enddo - do ie=nets,nete - pstens(:,:,ie)=maxval(phis(:,:,ie)) - call edgeVpack(edge3p1,pstens(:,:,ie),1,0,ie) - enddo - - call t_startf('smooth_phis_bexchV2') - call bndry_exchangeV(hybrid,edge3p1) - call t_stopf('smooth_phis_bexchV2') - - do ie=nets,nete - call edgeVunpackMax(edge3p1, pstens(:,:,ie), 1, 0, ie) - pmax(ie)=maxval(pstens(:,:,ie)) - enddo - - ! order = 1 grad^2 laplacian - ! order = 2 grad^4 (need to add a negative sign) - ! order = 3 grad^6 - ! order = 4 grad^8 (need to add a negative sign) - order_max = 1 - - - use_var_coef=.true. - if (hypervis_scaling/=0) then - ! for tensorHV option, we turn off the tensor except for *last* laplace operator - use_var_coef=.false. - if (hypervis_scaling>=3) then - ! with a 3.2 or 4 scaling, assume hyperviscosity - order_max = 2 - endif - endif - - - do ic=1,numcycle - pstens=phis - - do order=1,order_max-1 - - do ie=nets,nete - pstens(:,:,ie)=laplace_sphere_wk(pstens(:,:,ie),deriv,elem(ie),var_coef=use_var_coef) - call edgeVpack(edge3p1,pstens(:,:,ie),1,0,ie) - enddo - - call t_startf('smooth_phis_bexchV3') - call bndry_exchangeV(hybrid,edge3p1) - call t_stopf('smooth_phis_bexchV3') - - do ie=nets,nete - call edgeVunpack(edge3p1, pstens(:,:,ie), 1, 0, ie) - pstens(:,:,ie)=pstens(:,:,ie)*elem(ie)%rspheremp(:,:) - enddo -#ifdef DEBUGOMP -#if (defined HORIZ_OPENMP) -!$OMP BARRIER -#endif -#endif - enddo - do ie=nets,nete - pstens(:,:,ie)=laplace_sphere_wk(pstens(:,:,ie),deriv,elem(ie),var_coef=.true.) - enddo - if (mod(order_max,2)==0) pstens=-pstens - - do ie=nets,nete - ! ps(t+1) = ps(t) + Minv * DSS * M * RHS - ! ps(t+1) = Minv * DSS * M [ ps(t) + RHS ] - ! but output of biharminc_wk is of the form M*RHS. rewrite as: - ! ps(t+1) = Minv * DSS * M [ ps(t) + M*RHS/M ] - ! so we can apply limiter to ps(t) + (M*RHS)/M -#if 1 - mn=pmin(ie) - mx=pmax(ie) - iuse = numcycle+1 ! always apply min/max limiter -#endif - phis(:,:,ie)=phis(:,:,ie) + & - smooth_phis_nudt*pstens(:,:,ie)/elem(ie)%spheremp(:,:) - - - ! remove new extrema. could use conservative reconstruction from advection - ! but no reason to conserve mean PHI. - if (ic < iuse) then - do i=1,np - do j=1,np - if (phis(i,j,ie)>mx) phis(i,j,ie)=mx - if (phis(i,j,ie) real_kind, dd => longdouble_kind + use physical_constants, only : pi => dd_pi + use quadrature_mod, only: quadrature_t, gausslobatto, legendre + + implicit none + + ! dimension for vertical se coordinates + integer, parameter, public :: npv = 5 ! number of points per vertical element +!integer, parameter, public :: npv = 30 ! number of points per vertical element + + integer, parameter, public :: nev = nlev/npv ! number of vertical elements + integer, parameter, public :: n_unique = nev*(npv-1)+1 + + !_____________________________________________________________________ + type, public :: velem_t + + ! data structure for a vertical element in general vertical coordinates s + + integer :: kt, kb ! top and bottom indices + real(rl):: t , b ! top and bottom coordinates + + end type + + !_____________________________________________________________________ + type, public :: solver_args + + ! data structure containing arguments commonly used by solvers + + integer :: np1,nm1,n0 ! time level indices + integer :: qn0 ! time level used for virtual temperature + real(rl) :: dt ! dynamics timestep + type (hybrid_t) :: hybrid ! hybrid omp/mpi structure + type (hvcoord_t) :: hvcoord ! hybrid vertical coordinate structure + integer :: nets,nete ! start and end element indices + type (derivative_t) :: deriv ! horizontal derivative data struct + logical :: compute_diagnostics + real (rl) :: eta_ave_w + end type + + real(rl), dimension(npv,npv) :: ddn,ddn2,ddn3,ddn4 ! derivatives w.r.t eta + real(rl), dimension(npv,npv) :: ddn_1,ddn_n ! eta integration with upper bc, lower bc + real(rl), dimension(npv,npv) :: M,LU_1,LU_n ! mass, LU decomposition matrices + + type (quadrature_t) :: gll ! gll nodes for 1 vertical element + real(rl) :: eta_t, eta_b ! position of bottom and top of the column + real(rl) :: ds_deta, deta_ds ! metric terms for linear map + real(rl) :: eta(nlev) ! array of all eta values in the column + real(rl) :: ddn_hyam(nlev), ddn_hybm(nlev) ! vertical derivatives of hybrid coefficients + real(rl) :: ddn_hyai(nlevp), ddn_hybi(nlevp) ! vertical derivatives of hybrid coefficients + + integer :: ipiv_1(npv),ipiv_n(npv) ! pivot indices for vertical integration + + type(velem_t), allocatable :: ev(:) ! array of vertical element data structures + real(rl) :: elem_height ! height of vertical elements in s coords + + real(rl), allocatable:: M_interp(:,:), L_interp(:) ! vertical interpolation matrix and levels + + CONTAINS + + !_____________________________________________________________________ + function pack_solver_args(np1,nm1,n0,qn0,dt,hvcoord,hybrid,deriv,nets,nete,compute_diagnostics,eta_ave_w) result(a) + + ! convenience routine for packing data into solver_args structure + + integer, intent(in) :: np1,nm1,n0 ! time indices + integer, intent(in) :: qn0 ! time level used for virtual temperature + real*8, intent(in) :: dt ! dynamics timestep + type (hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coord data struct + type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct + integer, intent(in) :: nets,nete ! start and end element indices + type (derivative_t), intent(in) :: deriv ! horizontal derivative data struct + logical, intent(in) :: compute_diagnostics ! TODO: enable energy diagnostics + real (rl), intent(in) :: eta_ave_w ! TODO: enable qsplit + type(solver_args) :: a ! resultant arg data structure + + a%np1 = np1; + a%nm1 = nm1; + a%n0 = n0; + a%qn0 = qn0; + a%dt = dt + a%hvcoord = hvcoord; + a%hybrid = hybrid; + a%nets = nets; + a%nete = nete; + a%deriv = deriv; + a%compute_diagnostics = compute_diagnostics + a%eta_ave_w = eta_ave_w + + end function + + !_____________________________________________________________________ + real(rl) elemental function s2eta(s_coord, eta_t, eta_b) + + ! map reference-element coordinate to vertical coordinate s + + real(dd), intent(in) :: s_coord ! coorindate in ref element space + real(rl), intent(in) :: eta_t,eta_b ! top and bottom coord of element + real(rl) :: deta_ds ! metric term + deta_ds = (eta_b - eta_t)/2.0_dd + s2eta = (s_coord + 1.0_dd)*deta_ds + eta_t + + end function + + !_____________________________________________________________________ + real(dd) elemental function eta2s(eta_coord, eta_t,eta_b) + + ! map eta coordinate to reference-element coordinate + + real(rl), intent(in) :: eta_coord + real(rl), intent(in) :: eta_t,eta_b ! top and bottom coord of element + real(rl) :: ds_deta ! metric term + ds_deta = 2.0_dd/(eta_b-eta_t) + eta2s = (eta_coord - eta_t)*ds_deta - 1.0_dd + + end function + + !_____________________________________________________________________ + function mass_matrix(gll,n) result(M) + + ! get diagonal mass matrix + + type (quadrature_t), intent(in) :: gll ! gll nodes and weights + integer, intent(in) :: n ! number of points in 1d element + real(rl) :: M(n,n) ! resultant mass matrix + integer :: i + M = 0 + forall(i=1:n) M(i,i) = gll%weights(i) + + end function + + !_____________________________________________________________________ + function barycentric_weights(gll,n) result(weights) + + ! compute barycentric weights from node points (kopriva2009 p.76) + + type (quadrature_t), intent(in) :: gll ! gll nodes and weights + integer, intent(in) :: n ! number of points in 1d element + real(rl) :: weights(n) ! result + integer :: j,k + + weights = 1.0d0 + do j=2,n + do k=1,j-1 + weights(k) = weights(k)*(gll%points(k)-gll%points(j)) + weights(j) = weights(j)*(gll%points(j)-gll%points(k)) + enddo + enddo + weights = 1.0d0/weights + + end function + + !_____________________________________________________________________ + function first_derivative_matrix(gll,n) result(D) + + ! build 1st derivative matrix + + type (quadrature_t), intent(in) :: gll ! gll nodes and weights + integer, intent(in) :: n ! number of points in 1d element + real(rl) :: w(n) ! barycentric weights + real(rl) :: D(n,n) ! resultant derivative matrix + integer :: i,j ! loop indices + + w = barycentric_weights(gll,n) + D = 0 + do i = 1,n + do j = 1,n + if(j .ne. i) then + D(i,j) = w(j)/w(i) * 1.0d0/(gll%points(i)-gll%points(j)) + D(i,i) = D(i,i) - D(i,j) + endif + enddo + enddo + end function + + !_____________________________________________________________________ + function column_sum(f) result(f_sum) + + ! get quadrature of f over the vertical column + + real (rl), intent(in) :: f(np,np,nlev) + real (rl) :: f_sum(np,np) + integer :: kt,k,l + + f_sum = 0.0d0 + + do l=1,nev + kt = ev(l)%kt + do k=1,npv; f_sum = f_sum + deta_ds * gll%weights(k) * f(:,:,(kt-1)+k); enddo + enddo + + end function + + !_____________________________________________________________________ + function compress(f) result(fout) + + ! remove duplicate values at vertical edge nodes + + real(rl), intent(in) :: f(nlev) + real(rl) :: fout(n_unique) + integer :: l,i,j, ni + ni = npv-1 ! number of interiod nodes per element + + i=1; j=1 + do l=1,nev-1 + fout(i:i+ni-1) = f(j:j+ni-1) + i=i+ni; j=j+npv + enddo + fout(i:i+npv-1)=f(j:j+npv-1) + + end function + + + !_____________________________________________________________________ + function decompress(f) result(fout) + + ! restore duplicate values at vertical edge nodes + + real(rl), intent(in) :: f(n_unique) + real(rl) :: fout(nlev) + integer :: l,i,j + i=1; j=1 + do l=1,nev + fout(i:i+npv-1)=f(j:j+npv-1) + i=i+npv; j=j+npv-1 + enddo + + end function + !_____________________________________________________________________ + subroutine precompute_LU() + + ! pre-compute LU decomposition to make integration faster + + integer :: info ! status flag: 0=success + + ! get LU decomposition of vertical integration matrix ddn + LU_1 = ddn_1 + LU_n = ddn_n + + call DGETRF(npv,npv,LU_1,npv,ipiv_1,info) + if(info .ne. 0) then ! halt if routine failed + print *,"LU_1=",LU_1 + print *,"least squares DGETRF info = ",info; stop + endif + + call DGETRF(npv,npv,LU_n,npv,ipiv_n,info) + if(info .ne. 0) then ! halt if routine failed + print *,"LU_n=",LU_n + print *,"least squares DGETRF info = ",info; stop + endif + + end subroutine + +#if 0 + !_____________________________________________________________________ + function least_squares_integral(B,D,LU,ipiv) result(x) + + ! solve D^T D x = D^T B where D is diff matrix, x is anti-deriv of 1d curve B + + real(rl), intent(in) :: B(npv) ! 1d field to integrate + real(rl), intent(in) :: D(npv,npv) ! derivative matrix + real(rl), intent(in):: LU(npv,npv) ! LU factorization + integer, intent(in):: ipiv(npv) ! pivot indices + + real(rl) :: x (npv) ! resulting 1d integral + integer :: info ! status flag: 0=success + logical :: LU_initialized = .false. + real(rl) :: Dtr(npv,npv) !transpose of A + + ! apply LU factorization to solve for definite integral function + Dtr = transpose(D) + x = matmul(Dtr,B) + + call DGETRS('N',npv,1,LU,npv,ipiv,x,npv,info) ! solve for x + if(info .ne. 0) then ! halt if routine failed + print *,"integrate DGETRS info = ",info; stop + endif + + end function +#endif + + !_____________________________________________________________________ + function solve_LU(B,LU,ipiv,NRHS) result(x) + + ! LU x = B were LU is decomposed version of matrix A + + real(rl), intent(in):: B(npv,NRHS) ! 1d field to integrate + real(rl), intent(in):: LU(npv,npv) ! LU factorization + integer, intent(in):: ipiv(npv) ! pivot indices + integer, intent(in):: NRHS ! pivot indices + + real(rl) :: x(npv,NRHS) ! resulting 1d integral + integer :: info ! status flag: 0=success + + x = B + call DGETRS('N',npv,NRHS,LU,npv,ipiv,x,npv,info) ! solve for x + if(info .ne. 0) then ! halt if routine failed + print *,"integrate DGETRS info = ",info; stop + endif + + end function + + + !_____________________________________________________________________ + function least_squares_integral(B,A) result(x) + + ! to solve A x = B using least squares + ! where A is diff matrix, x is anti-deriv of 1d curve B + ! solve [A^T A] x = [A^T B] + + real(rl), intent(in) :: B(npv) ! 1d field to integrate + real(rl), intent(in) :: A(npv,npv) ! derivative matrix + + integer, parameter :: M=npv, N=npv, NRHS=1, LDA=npv, LDB=npv + integer, parameter :: NB=32, LWORK = M + M*NB + + + real(rl) :: x(M), WORK(LWORK) + integer :: INFO ! status flag: 0=success + + call DGELS('N',M,N,NRHS,A,LDA,B,LDB,WORK,LWORK,INFO) + + if(info .ne. 0) then ! halt if routine failed + print *,"least_squares_integral: DGELS info = ",info; stop + endif + + end function + + !_____________________________________________________________________ + function solve(A,B,M) result (x) + + ! solve A x = B for x, using DGESV + + real(rl), intent(in) :: B(M) ! 1d field to integrate + real(rl), intent(in) :: A(M,M) ! derivative matrix + integer, intent(in) :: M ! nrows = ncols + + integer :: IPIV(M), INFO + real(rl) :: x(M), B_(M), A_(M,M) + B_=B; A_=A + + !call DGESV(N, NRHS, A_, LDA, IPIV, B_, LDB, INFO) + call DGESV(M, 1 , A_, M , IPIV, B_, M , INFO) + + if(info .ne. 0) then ! halt if routine failed + print *,"solve: DGESV info = ",info; + endif + + x = B_ + + end function + + !_____________________________________________________________________ + function eta_integral_from_1(f, bc_1) result(x) + + ! Get anti-derivative wrt vertical eta coordinates + + real (rl), intent(in) :: f(np,np,nlev) ! field to differentiate + real (rl), intent(in) :: bc_1(np,np) ! boundary condition at n + + integer, parameter :: NRHS=np*np + real (rl) :: x(np,np,nlev),bc(np,np) + real (rl) :: B(npv,NRHS),R(npv,NRHS) + integer :: kt,kb,l,i,j + + bc = bc_1 + + do l=1,nev,1 + kt = ev(l)%kt; kb = ev(l)%kb + B = reshape( f(:,:,kt:kb), (/npv,NRHS/), order=(/2,1/) ) + B(1,:) = reshape(bc,(/NRHS/)) + R = solve_LU(B,LU_1,ipiv_1,NRHS) + x(:,:,kt:kb)= reshape( R, (/np,np,npv/), order=(/3,1,2/)) + bc = x(:,:,kb) + enddo + + end function + + !_____________________________________________________________________ + function eta_integral_from_n(f, bc_n) result(x) + + ! Get anti-derivative wrt vertical eta coordinates + + real (rl), intent(in) :: f(np,np,nlev) ! field to differentiate + real (rl), intent(in) :: bc_n(np,np) ! boundary condition at n + + integer, parameter :: NRHS=np*np + real (rl) :: x(np,np,nlev),bc(np,np) + real (rl) :: B(npv,NRHS),R(npv,NRHS) + integer :: kt,kb,l,i,j + + bc = bc_n + + do l=nev,1,-1 + kt = ev(l)%kt; kb = ev(l)%kb + B = reshape( f(:,:,kt:kb), (/npv,NRHS/), order=(/2,1/) ) + B(npv,:) = reshape(bc,(/NRHS/)) + R = solve_LU(B,LU_n,ipiv_n,NRHS) + x(:,:,kt:kb)= reshape( R, (/np,np,npv/), order=(/3,1,2/)) + bc = x(:,:,kt) + enddo + + end function + + !_____________________________________________________________________ + function eta_derivative_1d(f) result(deriv) + + ! get vertical derivative spanning entire column + + real (rl), intent(in) :: f(nlev) ! field to differentiate + real (rl) :: deriv(nlev) + integer :: kt,kb,l,i,j + + do l=1,nev + kt = ev(l)%kt; kb = ev(l)%kb; + deriv(kt:kb) = matmul(ddn,f(kt:kb)) + enddo + call vertical_dss_1d(deriv) + + end function + + !_____________________________________________________________________ + function eta_derivative(f) result(deriv) + + ! get vertical derivative spanning entire column + + real (rl), intent(in) :: f(np,np,nlev) ! field to differentiate + + integer, parameter :: NRHS=np*np + real (rl) :: deriv(np,np,nlev), B(npv,NRHS), R(npv,NRHS) + integer :: i,j,kt,kb,l + + do l=1,nev + kt = ev(l)%kt; kb = ev(l)%kb; + B = reshape( f(:,:,kt:kb), (/npv,NRHS/), order=(/2,1/) ) + R = matmul(ddn, B ) + deriv(:,:,kt:kb) = reshape( R, (/np,np,npv/), order=(/3,1,2/) ) + enddo + call vertical_dss(deriv) + + end function + + !_____________________________________________________________________ + function eta_derivative_pow(f,x) result(deriv) + + ! apply eta deriv matrix x times to entire column + + real (rl), intent(in) :: f(np,np,nlev) ! field to differentiate + integer, intent(in) :: x ! exponent of the deriv + + real (rl) :: deriv(np,np,nlev) + integer :: i,j,m,kt,kb,l + + deriv = f + + do m=1,x + deriv = eta_derivative(deriv) + enddo + + end function + + !_____________________________________________________________________ + function evenly_spaced_eta_coords(ni) result(s) + integer :: ni ! number of points + real(rl) :: s(ni) + integer :: i + + forall(i=2:ni-1) s(i) = eta_t + (eta_b-eta_t)/(ni-1)*(i-1) + s(1) = eta_t + s(ni) = eta_b + + end function + + !_____________________________________________________________________ + function advection(f,u,v,eta_dot,a, elem) result (f_adv) + + ! compute advection terms adv(f) = v*grad(f) + eta_dot df/deta + + real(rl), intent(in) :: f(np,np,nlev) ! scalar field to advect + real(rl), intent(in) :: u(np,np,nlev) ! horizontal velocity 1 + real(rl), intent(in) :: v(np,np,nlev) ! horizontal velocity 2 + real(rl), intent(in) :: eta_dot(np,np,nlev) ! vertical velocity component + type(solver_args), intent(in):: a ! solver args + type (element_t), intent(in), target :: elem ! element to operate upon + + real(rl) :: f_adv (np,np,nlev) ! resultant advection terms + real(rl) :: df_deta (np,np,nlev) ! vertical deriv of f + real(rl) :: grad_f (np,np,2,nlev) ! horizontal derivs of f + integer :: k + + ! get horizontal derivatives of f + do k=1,nlev; grad_f(:,:,:,k) = gradient_sphere(f(:,:,k),a%deriv, elem%Dinv); enddo + + ! get vertical derivative of f + df_deta = eta_derivative(f) + + ! compute and return advective terms + f_adv = u*grad_f(:,:,1,:) + v*grad_f(:,:,2,:) + eta_dot * df_deta +!call vertical_dss(f_adv) + + end function + + !_____________________________________________________________________ + function advection2(f,df_deta,u,v,eta_dot,a, elem) result (f_adv) + + ! compute advection terms adv(f) = v*grad(f) + eta_dot df/deta + + real(rl), intent(in) :: f(np,np,nlev) ! scalar field to advect + real(rl), intent(in) :: df_deta(np,np,nlev) ! scalar field to advect + real(rl), intent(in) :: u(np,np,nlev) ! horizontal velocity 1 + real(rl), intent(in) :: v(np,np,nlev) ! horizontal velocity 2 + real(rl), intent(in) :: eta_dot(np,np,nlev) ! vertical velocity component + type(solver_args), intent(in):: a ! solver args + type (element_t), intent(in), target :: elem ! element to operate upon + + real(rl) :: f_adv (np,np,nlev) ! resultant advection terms + real(rl) :: grad_f (np,np,2,nlev) ! horizontal derivs of f + integer :: k + + ! get horizontal derivatives of f + do k=1,nlev; grad_f(:,:,:,k) = gradient_sphere(f(:,:,k),a%deriv, elem%Dinv); enddo + + ! compute and return advective terms + f_adv = u*grad_f(:,:,1,:) + v*grad_f(:,:,2,:) + eta_dot * df_deta +!call vertical_dss(f_adv) + + end function + + !_____________________________________________________________________ + function self_advection(u,v,eta_dot,a, elem) result (v_adv) + + ! compute self advection of horiztonal velocity + ! v*grad(v) = 1/2 grad(v^2) + (curl u) cross u + eta_dot dv/deta + + real(rl), intent(in) :: u(np,np,nlev) ! horizontal velocity 1 + real(rl), intent(in) :: v(np,np,nlev) ! horizontal velocity 2 + real(rl), intent(in) :: eta_dot(np,np,nlev) ! vertical velocity component + type(solver_args), intent(in):: a ! solver args + type (element_t), intent(in), target :: elem ! element to operate upon + + real(rl) :: v_adv(np,np,2,nlev) ! advection terms + real(rl) :: vsqr (np,np,nlev) ! velocity^2 + real(rl) :: vort (np,np,nlev) ! vorticity + + real(rl) :: grad_vsqr(np,np,2,nlev) ! gradient of velocity^2 + real(rl) :: vort_cross_v(np,np,2,nlev) ! vorticity cross v + real(rl) :: v2d (np,np,2,nlev) ! (u,v) velocity vector + real(rl) :: du_deta(np,np,nlev) ! vertical deriv of u + real(rl) :: dv_deta(np,np,nlev) ! vertical deriv of v + integer :: k + + ! get vertical derivative of u and v + du_deta = eta_derivative(u) + dv_deta = eta_derivative(v) + + ! get grad v^2 term + vsqr = u*u + v*v + do k=1,nlev; grad_vsqr(:,:,:,k) = gradient_sphere(vsqr(:,:,k),a%deriv, elem%Dinv); enddo + + ! get curl u terms + v2d(:,:,1,:) = u; v2d(:,:,2,:)=v + do k=1,nlev; vort(:,:,k)= vorticity_sphere(v2d(:,:,:,k),a%deriv,elem); enddo + vort_cross_v(:,:,1,:) = -v*vort + vort_cross_v(:,:,2,:) = +u*vort + + ! compute and return advective terms + v_adv(:,:,1,:) = 0.5d0*grad_vsqr(:,:,1,:) + vort_cross_v(:,:,1,:) + eta_dot*du_deta + v_adv(:,:,2,:) = 0.5d0*grad_vsqr(:,:,2,:) + vort_cross_v(:,:,2,:) + eta_dot*dv_deta +!call vertical_dss(v_adv(:,:,1,:)) +!call vertical_dss(v_adv(:,:,2,:)) + + end function + + !_____________________________________________________________________ + function self_advection2(u,v,du_dn,dv_dn,eta_dot,a,elem) result (v_adv) + + ! compute self advection of horiztonal velocity + ! v*grad(v) = 1/2 grad(v^2) + (curl u) cross u + eta_dot dv/deta + + real(rl), intent(in) :: u(np,np,nlev) ! u vel + real(rl), intent(in) :: v(np,np,nlev) ! v vel + real(rl), intent(in) :: du_dn(np,np,nlev) ! u velocity gradient + real(rl), intent(in) :: dv_dn(np,np,nlev) ! v velcoity gradient + real(rl), intent(in) :: eta_dot(np,np,nlev) ! vertical velocity component + type(solver_args), intent(in):: a ! solver args + type (element_t), intent(in), target :: elem ! element to operate upon + + real(rl) :: v_adv(np,np,2,nlev) ! advection terms + real(rl) :: vsqr (np,np,nlev) ! velocity^2 + real(rl) :: vort (np,np,nlev) ! vorticity + + real(rl) :: grad_vsqr(np,np,2,nlev) ! gradient of velocity^2 + real(rl) :: vort_cross_v(np,np,2,nlev) ! vorticity cross v + real(rl) :: v2d (np,np,2,nlev) ! (u,v) velocity vector + integer :: k + + ! get grad v^2 term + vsqr = u*u + v*v + do k=1,nlev; grad_vsqr(:,:,:,k) = gradient_sphere(vsqr(:,:,k),a%deriv, elem%Dinv); enddo + + ! get curl u terms + v2d(:,:,1,:) = u; v2d(:,:,2,:)=v + do k=1,nlev; vort(:,:,k)= vorticity_sphere(v2d(:,:,:,k),a%deriv,elem); enddo + vort_cross_v(:,:,1,:) = -v*vort + vort_cross_v(:,:,2,:) = +u*vort + + ! compute and return advective terms + v_adv(:,:,1,:) = 0.5d0*grad_vsqr(:,:,1,:) + vort_cross_v(:,:,1,:) + eta_dot*du_dn + v_adv(:,:,2,:) = 0.5d0*grad_vsqr(:,:,2,:) + vort_cross_v(:,:,2,:) + eta_dot*dv_dn +!call vertical_dss(v_adv(:,:,1,:)) +!call vertical_dss(v_adv(:,:,2,:)) + + end function + + !_____________________________________________________________________ + subroutine vertical_dss(f) + + ! Apply direct-stiffness-summation (DSS) at vertical element edges + + real(rl), intent(inout) :: f(np,np,nlev) + integer :: kt,kb, iv + real(rl):: f_star(np,np) + + do iv=1,nev-1 + + kb = ev(iv )%kb; ! bottom of element iv + kt = ev(iv+1)%kt; ! top of element iv+1 + + f_star = 0.5d0*(f(:,:,kt) + f(:,:,kb)) + f(:,:,kt) = f_star; + f(:,:,kb) = f_star + + enddo + + end subroutine + + !_____________________________________________________________________ + subroutine vertical_dss_1d(f) + + ! Apply direct-stiffness-summation (DSS) at vertical element edges + + real(rl), intent(inout) :: f(nlev) + integer :: kt,kb, iv + real(rl):: f_star + + do iv=1,nev-1 + + kb = ev(iv )%kb; ! bottom of element iv + kt = ev(iv+1)%kt; ! top of element iv+1 + + f_star = 0.5d0*(f(kt) + f(kb)) + f(kt) = f_star; + f(kb) = f_star + + enddo + + end subroutine + + !_____________________________________________________________________ + function lagrange_polynomial(s,j,gll) result(L) + + ! get jth Lagrange interpolating-polynomial at points s + ! L_j(s) = PROD(i/=j) (s-x_i)/(x_j-x_i)° + + real(rl), intent(in) :: s(:) ! s points in range [-1,1] + integer, intent(in) :: j ! index of polynomial + type (quadrature_t), intent(in) :: gll ! gll nodes and weights + + real(rl) :: L(size(s)) + real(rl) :: D + integer :: i,npts + npts = size(gll%points) + + L = 1.0d0 + D = 1.0d0 + + do i=1,j-1 + L = L * (s - gll%points(i)) + D = D * (gll%points(j) - gll%points(i)) + enddo + + do i=j+1,npts + L = L * (s - gll%points(i)) + D = D * (gll%points(j) - gll%points(i)) + enddo + + L = L/D + + end function + + !_____________________________________________________________________________ + subroutine init_vertical_interp(levels, ni) + + ! Get vertically interpolated levels + + real(rl), intent(in) :: levels(ni) ! vector of vertical interp coordinates + integer, intent(in) :: ni ! number of interpolated levels + + ! allocate and store vertical interpolation matrix + + if(.not. allocated(M_interp)) then + + allocate(L_interp(ni)) + allocate(M_interp(ni,nlev)) + + L_interp = levels + M_interp = vertical_interp_matrix(levels,ni) + endif + + end subroutine + + !_____________________________________________________________________________ + function v_interpolate(f,ni) result(f_i) + + real(rl), intent(in) :: f(nlev) + integer, intent(in) :: ni + real(rl) :: f_i(ni) + + f_i = matmul(M_interp,f) + + end function + + !_____________________________________________________________________ + function vertical_interp_matrix(s, ni) result(M) + + ! get matrix mapping nlev values to vertical interpolation coordinates s_i + ! note: + real(rl), intent(in) :: s(:) ! vector of vertical interp coordinates + integer :: ni ! number of points in array s_i + + real(rl):: M(ni,nlev ) ! result, interpolation matrix + + integer :: i,j,k ! loop index + integer :: v_ind(ni) ! vertical element indices + real(rl):: ref(ni) ! ref coord locations + real(rl):: ds + real(rl):: val(1) + + ! initialize matrix to 0 + M = 0 + v_ind =-1 + ref =-2 + + ! assign interpolation coefficients for each point s(i) + do i=1,ni + + ! locate element and get position in reference element + do j=1,nev + ! check to see if point is inside vertical element j + if( s(i)<=ev(j)%b .and. s(i)>=ev(j)%t ) then + + ! store element index + v_ind(i) = j + + ! find reference coordinate location + ref(i) = eta2s(s(i), ev(j)%t, ev(j)%b) + + ! assign matrix coefficients for row(i) + do k=1,npv + val =lagrange_polynomial(ref(i:i),k,gll) + M(i, ev(j)%kt-1 + k) = val(1) + enddo + + exit ! exit loop, to make sure we assign only one element + + endif + enddo + + ! output for debugging + !print *,"s(i)=",s(i)," v_ind(i)=",v_ind(i)," ref(i)=",ref(i) + + enddo + + ! do j=1,nev + ! print *, "ev(j)%b=",ev(j)%b ," ev(j)%t=", ev(j)%t + ! enddo + + if(any(v_ind==-1)) stop 'vertical_interp_matrix: point outside vertical mesh' + + end function + + !_____________________________________________________________________ + subroutine neumann_bcs_bottom_elem(f,dfdn) + + ! Neumann bc derived without explicit surface terms + + real(rl), intent(inout) :: f(np,np,npv) + real(rl), intent(in) :: dfdn(np,np) + + real(rl) :: r_k + integer :: i,j + logical :: mask(npv) + + mask(:) = .true. + mask(npv)= .false. + + ! compute unknown value at index k + do j=1,np + do i=1,np + + ! get rhs of Neumann boundary condition + r_k = dfdn(i,j) - sum( ddn(npv,:)*f(i,j,:), MASK=mask) + + ! solve for unknown value + f(i,j,npv) = r_k/ddn(npv,npv) + + enddo + enddo + end subroutine + + !_____________________________________________________________________ + subroutine neumann_bcs_top_elem(f,dfdn) + + ! Neumann bc derived without explicit surface terms + + real(rl), intent(inout) :: f(np,np,npv) + real(rl), intent(in) :: dfdn(np,np) + + real(rl) :: r_k + integer :: i,j + logical :: mask(npv) + + mask(:)= .true. + mask(1)= .false. + + ! compute unknown value at index k + do j=1,np + do i=1,np + + ! get rhs of Neumann boundary condition + r_k = dfdn(i,j) - sum( ddn(1,:)*f(i,j,:), MASK=mask) + + ! solve for unknown value + f(i,j,1) = r_k/ddn(1,1) + + enddo + enddo + end subroutine + + !_____________________________________________________________________ + subroutine neumann_bcs_bottom(f,dfdn) + + ! Neumann bc derived without explicit surface terms + + real(rl), intent(inout) :: f(np,np,nlev) + real(rl), intent(in) :: dfdn(np,np) + call neumann_bcs_bottom_elem(f(:,:,1+(nlev-npv):nlev),dfdn) +call vertical_dss(f) + + end subroutine + + !_____________________________________________________________________ + subroutine neumann_bcs_top(f,dfdn) + + ! Neumann bc derived without explicit surface terms + + real(rl), intent(inout) :: f(np,np,nlev) + real(rl), intent(in) :: dfdn(np,np) + call neumann_bcs_top_elem(f(:,:,1:npv),dfdn) +call vertical_dss(f) + + end subroutine + !_____________________________________________________________________ + subroutine neumann_bcs_top_bottom(f,dfdn_t,dfdn_b) + + ! assumes nelemv > 1 + + real(rl), intent(inout) :: f(np,np,nlev) + real(rl), intent(in) :: dfdn_t(np,np) + real(rl), intent(in) :: dfdn_b(np,np) + + call neumann_bcs_top (f,dfdn_t) + call neumann_bcs_bottom(f,dfdn_b) + + end subroutine + + !_____________________________________________________________________ + subroutine set_hvcoeffs_from_etai(hvcoord,hybrid) + + ! Set hybrid coefficients consistent with analytical etai levels + + type(hybrid_t), intent(in) :: hybrid + type(hvcoord_t), intent(inout) :: hvcoord + real(rl) :: hyai,hybi, eta, tmp + integer :: k + real(rl), parameter :: c = 3.0 ! 3.5 is best + + if(hybrid%par%masterproc) print *,"set_hvcoeffs_from_etai" + + ! set hybrid coefficients as suggested in dcmip test-case document + do k=1,nlevp + eta = hvcoord%etai(k) + + hybi = 0.0 + tmp = ((eta-vtop)/(1.0-vtop)) + if(tmp>0.0) hybi = tmp**c + + hyai = eta - hybi + hvcoord%hyai(k) = hyai + hvcoord%hybi(k) = hybi + if(hybrid%par%masterproc) print *,k,': etai = ',eta,' Ai = ',hyai,' Bi = ',hybi + enddo + + end subroutine + + !_____________________________________________________________________ + subroutine append_scalar_to_file(filename,scalar,erase) + + character*(*), intent(in) :: filename + real(rl), intent(in) :: scalar + logical, intent(in) :: erase + + if(erase) then; open(unit=12, file=adjustl(filename),action='write',status='replace') + else ; open(unit=12, file=adjustl(filename),action='write',status='old',position='append') + endif + + write(12,*) scalar + close(unit=12) + + end subroutine + + !_____________________________________________________________________ + subroutine write_scalar_field_1d(filename,field) + character*(*), intent(in) :: filename + real(rl), intent(in) :: field(:) + + open(unit=12, file=adjustl(filename),action='write',status='replace') + write(12,*) field + + end subroutine + + !_____________________________________________________________________ + subroutine write_scalar_field_2d(filename,field) + character*(*), intent(in) :: filename + real(rl), intent(in) :: field(:,:) + integer i + + open(unit=12, file=adjustl(filename),action='write',status='replace') + do i=1,size(field,1) + write(12,*) field(i,:) + enddo + + end subroutine + + !_____________________________________________________________________ + subroutine write_scalar_field_3d(filename,field) + character*(*), intent(in) :: filename + real(rl), intent(in) :: field(:,:,:) + integer i,j + + open(unit=12, file=adjustl(filename),action='write',status='replace') + do i=1,size(field,1) + do j=1,size(field,2) + write(12,*) field (i,j,:) + enddo + enddo + + end subroutine + + !_____________________________________________________________________ + subroutine make_vertical_mesh(hybrid, hvcoord) + + type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct + type (hvcoord_t),intent(inout) :: hvcoord ! hybrid vertical coord data struct + + integer :: ie, k, iv, k1, k2 + real(rl) :: eta1, eta2 + real(rl) :: D(npv,npv) ! resultant derivative matrix + + if (hybrid%masterthread) print *,"make vertical mesh:" + + ! ensure nlev is an integer multple of npv + !nev = nlev/npv + if( MOD(nlev,npv) /= 0) then + print *,"error: nlev=",nlev,"must be a multple of npv=",npv + stop + endif + if (hybrid%masterthread) print *,"nev=",nev," npv=",npv + + ! set extent of vertical domain + eta_b = 1.0d0 ! set bottom coord + eta_t = hvcoord%etam(1) ! set top coord from test or file + + ! if analytic vcoords get vtop from namelist + if(vanalytic==1) eta_t = vtop + if(hybrid%masterthread) print *, "eta_b=",eta_b, "eta_t=",eta_t + + ! store gll nodes and weights + gll = gausslobatto(npv) + + ! init linear map from s to eta coordinates + elem_height = (eta_b - eta_t)/nev + ds_deta = 2.0_dd/elem_height + deta_ds = 1.0_dd/ds_deta + if (hybrid%masterthread) print *, "elem_height = ",elem_height + + ! set matrix operators + D = ds_deta*first_derivative_matrix(gll,npv) + ddn = D ! get 1st deriv matrix + ddn2 = matmul(ddn,ddn) ! get 2nd deriv matrix + ddn3 = matmul(ddn,ddn2) ! get 3rd deriv matrix + ddn4 = matmul(ddn,ddn3) ! get 4th deriv matrix + M = mass_matrix(gll,npv) + + ddn_1 = ddn; ddn_1(1 ,:)=0.0_rl; ddn_1(1 ,1 )=1.0_rl + ddn_n = ddn; ddn_n(npv,:)=0.0_rl; ddn_n(npv,npv)=1.0_rl + + ! store LU decomposition needed for least-squares integration + call precompute_LU() + + ! allocate vertical element array + allocate ( ev(nev) ) + + ! construct vertical mesh + do iv=1,nev + + eta1 = eta_t + elem_height*(iv-1) ! get eta at element top + eta2 = eta_b - elem_height*(nev-iv) ! get eta at element bottom + k1 = 1 + npv*(iv-1) ! get index at element top + k2 = k1 + npv-1 ! get index at element bottom + + ! store coordinates and indices in a data structure + ev(iv)%t = eta1; ev(iv)%b = eta2; + ev(iv)%kt = k1; ev(iv)%kb = k2; + + ! set eta levels at gll node locations + eta(k1:k2) = s2eta(gll%points, eta1, eta2) + if (hybrid%masterthread) print *,"eta1 = ",eta1," eta2 = ",eta2 + + enddo + + ! store vertical coords in hvcoord struct + hvcoord%etai(2:nlev+1)= eta + hvcoord%etai(1) = eta(1)/2.0d0 + call set_hvcoeffs_from_etai(hvcoord,hybrid) + + hvcoord%etam = eta + hvcoord%hyam = hvcoord%hyai(2:nlev+1) + hvcoord%hybm = hvcoord%hybi(2:nlev+1) + call set_layer_locations(hvcoord, .true., hybrid%masterthread) + + ! compute vertical derivatives of hybrid coefficients + ddn_hyam = eta_derivative_1d(hvcoord%hyam) + ddn_hybm = eta_derivative_1d(hvcoord%hybm) + ddn_hyai(2:nlevp) = ddn_hyam; ddn_hyai(1)=0 + ddn_hybi(2:nlevp) = ddn_hybm; ddn_hybi(1)=0 + + call test_vertical_operators(hybrid) + + end subroutine + + !_____________________________________________________________________ + subroutine test_vertical_operators(hybrid) + + integer, parameter :: ni = 200 ! number of interpolation pts + logical, parameter :: write_files = .false. + + type (hybrid_t), intent(in) :: hybrid + + real(rl), dimension(np,np,nlev) :: f,f2,f3,f4, f_deriv,f_integral, f_integral2 + + real(rl), dimension(np,np,nlev) :: err_deriv, err_integral, err_integral2 + real(rl), dimension(np,np,nlev) :: comp, comp2, err_comp, err_comp2 + real(rl), dimension(np,np,nlev) :: c,s,d1,d2,d3,d4 + real(rl), dimension(np,np) :: zeros + + real(rl), dimension(nlev)::f1 ! 1d function + real(rl), dimension(ni )::f1_i ! 1d interpolation function + + real(rl), dimension(ni) :: s_interp + real(rl), dimension(ni,nlev) :: M_interp + + real(rl) :: max_err_deriv, max_err_integral,max_err_integral2, max_err_comp, max_err_comp2 + real(rl) :: e1,e2,e3,e4 + real(rl) :: frq + integer :: i,j,k,l + + if (hybrid%masterthread) print *,"test_vertical_operators" + zeros = 0.0d0 + + ! let f = e^eta * something + do j=1,np; do i=1,np; do k=1,nlev; + f(i,j,k) = exp(eta(k))*cos(1.0d0*i/np)*cos(1.0d0*j/np) + ! f(i,j,k) = cos(5.0d0*eta(k)) + enddo; enddo; enddo; + + ! compute derivative of f wrt eta + f_deriv = eta_derivative(f) + err_deriv = f_deriv - f + max_err_deriv = maxval(abs(err_deriv)) + + ! compute integral of f wrt eta from top + f_integral = eta_integral_from_1(f, f(:,:,1)) + err_integral = f_integral - f + + max_err_integral= maxval(abs(err_integral)) + + ! get interpolated vertical values + s_interp = evenly_spaced_eta_coords(ni) + M_interp = vertical_interp_matrix(s_interp, ni) + do k=1,nlev; f1(k)=cos(7.0*pi*eta(k)); enddo + f1_i = matmul(M_interp,f1) + + + ! compute integral of f wrt eta from bottom + f_integral2 = eta_integral_from_n(f, f(:,:,nlev)) + err_integral2 = f_integral2 - f + max_err_integral2= maxval(abs(err_integral2)) + + ! compose deriv and inverse matrices + comp = eta_integral_from_1( f_deriv, f(:,:,1)) ! integral of deriv + comp2 = eta_derivative( f_integral ) ! deriv of integral + err_comp = comp - f + err_comp2 = comp2 - f + + max_err_comp = maxval(abs(err_comp)) + max_err_comp2 = maxval(abs(err_comp2)) + + !if (hybrid%masterthread) then + ! print *, "f(1,1,:) =", f(1,1,:) + ! print *, "f_integral(1,1,:) =", f_integral(1,1,:) + ! print *, "f_deriv(1,1,:) =", f_deriv(1,1,:) + ! endif + + if(write_files) then + call write_scalar_field_1d("eta.txt", eta(:)) + call write_scalar_field_1d("f.txt", f(1,1,:)) + call write_scalar_field_1d("f_deriv.txt", f_deriv(1,1,:)) + call write_scalar_field_1d("f_integral.txt", f_integral(2,2,:)) + call write_scalar_field_1d("f_integral2.txt", f_integral2(1,1,:)) + call write_scalar_field_1d("comp.txt", comp(1,1,:)) + call write_scalar_field_1d("comp2.txt", comp2(1,1,:)) + call write_scalar_field_1d("f1.txt", f1) + call write_scalar_field_1d("f1_i.txt", f1_i) + call write_scalar_field_1d("s_i.txt", s_interp) + endif + + if (hybrid%masterthread) then + print *,"max error, vertical deriv of exp(eta)=",max_err_deriv + print *,"max error, vertical integral of exp(eta)=",max_err_integral + print *,"max error, vertical integral2 of exp(eta)=",max_err_integral2 + print *,"max error, integral of derivative =",max_err_comp + print *,"max error, derivative of integral =",max_err_comp2 + endif + + ! test higher vertical derivatives + frq = 1.0_rl + forall(k=1:nlev) s(:,:,k) = sin(frq*eta(k)) + forall(k=1:nlev) c(:,:,k) = cos(frq*eta(k)) + + ! compute 1st, 2nd, 3rd, and 4th deriv of sin(eta) wrt eta + d1 = eta_derivative_pow(s,1) + d2 = eta_derivative_pow(s,2) + d3 = eta_derivative_pow(s,3) + d4 = eta_derivative_pow(s,4) + + ! compute max error in higher vertical derivs + e1 = maxval(abs( d1 - frq**1 * c)) + e2 = maxval(abs( d2 + frq**2 * s)) + e3 = maxval(abs( d3 + frq**3 * c)) + e4 = maxval(abs( d4 - frq**4 * s)) + + if (hybrid%masterthread) then + print *,"max error, D cos'(eta)=",e1 + print *,"max error, D^2 cos'(eta)=",e2 + print *,"max error, D^3 cos'(eta)=",e3 + print *,"max error, D^4 cos'(eta)=",e4 + endif + + ! test solution of Neumann boundary conditions + f2 = f; f3 = f; f4 = f + + ! compute neumann bcs at top and bottom of the column + !call neumann_bcs_top_bottom(f2,f_deriv(:,:,1),f_deriv(:,:,nlev)) + + !if (hybrid%masterthread) then + ! print *,"Check neumann_bcs_top_bottom" + ! print *,"f2_t - f_t =",f2(:,:,1)-f(:,:,1) + ! print *,"f2_b - f_b =",f2(:,:,nlev)-f(:,:,nlev) + !endif + + ! apply neuman bc at top of column + call neumann_bcs_top(f3,f_deriv(:,:,1)) + + ! apply neumann bc at bottom of column + call neumann_bcs_bottom(f4,f_deriv(:,:,nlev)) + + if (hybrid%masterthread) then + print *,"Check neumann_bcs at index 1 and n" + print *,"max error f3 - f =",maxval(abs(f3(:,:,1)-f(:,:,1))) + print *,"max error f4 - f =",maxval(abs(f4(:,:,nlev)-f(:,:,nlev) )) + endif + + end subroutine + + end module + diff --git a/components/homme/src/share/vertremap_mod_base.F90 b/components/homme/src/share/vertremap_base.F90 similarity index 100% rename from components/homme/src/share/vertremap_mod_base.F90 rename to components/homme/src/share/vertremap_base.F90 diff --git a/components/homme/src/share/viscosity_mod_base.F90 b/components/homme/src/share/viscosity_base.F90 similarity index 86% rename from components/homme/src/share/viscosity_mod_base.F90 rename to components/homme/src/share/viscosity_base.F90 index c5409ad5c646..28f1c63126cc 100644 --- a/components/homme/src/share/viscosity_mod_base.F90 +++ b/components/homme/src/share/viscosity_base.F90 @@ -2,7 +2,7 @@ #include "config.h" #endif -module viscosity_mod_base +module viscosity_base ! ! This module should be renamed "global_deriv_mod.F90" ! @@ -774,6 +774,156 @@ subroutine neighbor_minmax_finish(hybrid,edgeMinMax,nets,nete,min_neigh,max_neig end subroutine neighbor_minmax_finish +subroutine smooth_phis(phis,elem,hybrid,deriv,nets,nete,minf,numcycle) + use control_mod, only : smooth_phis_nudt, hypervis_scaling + implicit none + + integer :: nets,nete + real (kind=real_kind), dimension(np,np,nets:nete), intent(inout) :: phis + type (hybrid_t) , intent(in) :: hybrid + type (element_t) , intent(inout), target :: elem(:) + type (derivative_t) , intent(in) :: deriv + real (kind=real_kind), intent(in) :: minf + integer, intent(in) :: numcycle + + ! local + type (EdgeBuffer_t) :: edgebuf + real (kind=real_kind), dimension(np,np,nets:nete) :: pstens + real (kind=real_kind), dimension(nets:nete) :: pmin,pmax + real (kind=real_kind) :: mx,mn + integer :: nt,ie,ic,i,j,order,order_max, iuse + logical :: use_var_coef + + + ! create edge buffer for 1 field + call initEdgeBuffer(hybrid%par,edgebuf,elem,1) + + + ! compute local element neighbor min/max + do ie=nets,nete + pstens(:,:,ie)=minval(phis(:,:,ie)) + call edgeVpack(edgebuf,pstens(:,:,ie),1,0,ie) + enddo + + call t_startf('smooth_phis_bexchV1') + call bndry_exchangeV(hybrid,edgebuf) + call t_stopf('smooth_phis_bexchV1') + + do ie=nets,nete + call edgeVunpackMin(edgebuf, pstens(:,:,ie), 1, 0, ie) + pmin(ie)=minval(pstens(:,:,ie)) + enddo + do ie=nets,nete + pstens(:,:,ie)=maxval(phis(:,:,ie)) + call edgeVpack(edgebuf,pstens(:,:,ie),1,0,ie) + enddo + + call t_startf('smooth_phis_bexchV2') + call bndry_exchangeV(hybrid,edgebuf) + call t_stopf('smooth_phis_bexchV2') + + do ie=nets,nete + call edgeVunpackMax(edgebuf, pstens(:,:,ie), 1, 0, ie) + pmax(ie)=maxval(pstens(:,:,ie)) + enddo + + ! order = 1 grad^2 laplacian + ! order = 2 grad^4 (need to add a negative sign) + ! order = 3 grad^6 + ! order = 4 grad^8 (need to add a negative sign) + order_max = 1 + + + use_var_coef=.true. + if (hypervis_scaling/=0) then + ! for tensorHV option, we turn off the tensor except for *last* laplace operator + use_var_coef=.false. + if (hypervis_scaling>=3) then + ! with a 3.2 or 4 scaling, assume hyperviscosity + order_max = 2 + endif + endif + + + do ic=1,numcycle + pstens=phis + + do order=1,order_max-1 + + do ie=nets,nete + pstens(:,:,ie)=laplace_sphere_wk(pstens(:,:,ie),deriv,elem(ie),var_coef=use_var_coef) + call edgeVpack(edgebuf,pstens(:,:,ie),1,0,ie) + enddo + + call t_startf('smooth_phis_bexchV3') + call bndry_exchangeV(hybrid,edgebuf) + call t_stopf('smooth_phis_bexchV3') + + do ie=nets,nete + call edgeVunpack(edgebuf, pstens(:,:,ie), 1, 0, ie) + pstens(:,:,ie)=pstens(:,:,ie)*elem(ie)%rspheremp(:,:) + enddo + enddo + do ie=nets,nete + pstens(:,:,ie)=laplace_sphere_wk(pstens(:,:,ie),deriv,elem(ie),var_coef=.true.) + enddo + if (mod(order_max,2)==0) pstens=-pstens + + do ie=nets,nete + ! ps(t+1) = ps(t) + Minv * DSS * M * RHS + ! ps(t+1) = Minv * DSS * M [ ps(t) + RHS ] + ! but output of biharminc_wk is of the form M*RHS. rewrite as: + ! ps(t+1) = Minv * DSS * M [ ps(t) + M*RHS/M ] + ! so we can apply limiter to ps(t) + (M*RHS)/M +#if 1 + mn=pmin(ie) + mx=pmax(ie) + iuse = numcycle+1 ! always apply min/max limiter +#endif + phis(:,:,ie)=phis(:,:,ie) + & + smooth_phis_nudt*pstens(:,:,ie)/elem(ie)%spheremp(:,:) + + + ! remove new extrema. could use conservative reconstruction from advection + ! but no reason to conserve mean PHI. + if (ic < iuse) then + do i=1,np + do j=1,np + if (phis(i,j,ie)>mx) phis(i,j,ie)=mx + if (phis(i,j,ie)=1) then ! HS with tracer (Galewsky type forcing, with flux of 2.3e-5 kg/m^2/s @@ -99,18 +78,18 @@ subroutine hs_forcing(elemin,hvcoord,tl,dt) do j=1,np do i=1,np FQ = rdp * g * 2.3E-5 * COS(elemin%spherep(i,j)%lat)**2 - elemin%derived%FQ(i,j,nlev,q,nm1) =elemin%derived%FQ(i,j,nlev,q,nm1)+FQ + elemin%derived%FQ(i,j,nlev,q) =elemin%derived%FQ(i,j,nlev,q)+FQ enddo enddo do j=1,np do i=1,np do k=1,nlev - pmid = hvcoord%hyam(k)*hvcoord%ps0 + hvcoord%hybm(k)*(elemin%state%ps_v(i,j,nfrc)) + pmid = hvcoord%hyam(k)*hvcoord%ps0 + hvcoord%hybm(k)*(elemin%state%ps_v(i,j,nm1)) r0=elemin%state%Q(i,j,k,q) r1=r0 - call Prim_Condense(r1,elemin%state%T(i,j,k,nfrc),pmid) - elemin%derived%FQ(i,j,k,q,nm1) = elemin%derived%FQ(i,j,k,q,nm1) + & + call Prim_Condense(r1,temperature(i,j,k),pmid) + elemin%derived%FQ(i,j,k,q) = elemin%derived%FQ(i,j,k,q) + & (r1-r0)/(dtf_q) enddo enddo @@ -135,35 +114,19 @@ function hs_v_forcing(hvcoord,ps,v,npts,nlevels) result(hs_v_frc) integer i,j,k real (kind=real_kind) :: k_v real (kind=real_kind) :: rps(npts,npts) -#ifdef _USE_VECTOR - real (kind=real_kind) :: p,etam,rec_one_minus_sigma_b -#else real (kind=real_kind) :: p,etam -#endif do j=1,npts -#ifdef _USE_VECTOR - call vrec(rps(1,j), ps(1,j), npts) -#else do i=1,npts rps(i,j) = 1.0D0/ps(i,j) end do -#endif end do -#ifdef _USE_VECTOR - rec_one_minus_sigma_b = 1.0D0/(1.0_real_kind - sigma_b) -#endif - do k=1,nlevels do j=1,npts do i=1,npts p = hvcoord%hyam(k)*hvcoord%ps0 + hvcoord%hybm(k)*ps(i,j) etam = hvcoord%hyam(k) + hvcoord%hybm(k) -#ifdef _USE_VECTOR - k_v = k_f*MAX(0.0_real_kind,(etam - sigma_b )*rec_one_minus_sigma_b) -#else k_v = k_f*MAX(0.0_real_kind,(etam - sigma_b )/(1.0_real_kind - sigma_b)) -#endif hs_v_frc(i,j,1,k) = -k_v*v(i,j,1,k) hs_v_frc(i,j,2,k) = -k_v*v(i,j,2,k) end do @@ -198,58 +161,23 @@ function hs_T_forcing(hvcoord,ps,T,sphere,npts,nlevels) result(hs_T_frc) real (kind=real_kind) :: rec_one_minus_sigma_b -#ifdef _USE_VECTOR - real (kind=real_kind) :: xarray(npts), yarray(npts) -#endif - integer i,j,k logps0 = LOG(hvcoord%ps0 ) do j=1,npts -#ifdef _USE_VECTOR - call vrec(rps(1,j), ps(1,j), npts) - do i=1,npts - snlat = SIN(sphere(i,j)%lat) - snlatsq(i,j) = snlat*snlat - cslatsq(i,j) = 1.0D0 - snlatsq(i,j) - end do -#else do i=1,npts snlat = SIN(sphere(i,j)%lat) snlatsq(i,j) = snlat*snlat cslatsq(i,j) = 1.0D0 - snlatsq(i,j) rps(i,j) = 1.0D0/ps(i,j) end do -#endif end do rec_one_minus_sigma_b = 1.0D0/(1.0D0 - sigma_b) do k=1,nlevels do j=1,npts - -#ifdef _USE_VECTOR - do i=1,npts - xarray(i) = hvcoord%hyam(k)*hvcoord%ps0 + hvcoord%hybm(k)*ps(i,j) - end do - call vlog(xarray, xarray, npts) - do i=1,npts - yarray(i) = kappa*(xarray(i) - logps0) - end do - call vexp(yarray, yarray, npts) - - do i=1,npts - logprat = xarray(i) - logps0 - pratk = yarray(i) - etam = hvcoord%hyam(k) + hvcoord%hybm(k) - - k_t(i,j)= k_a + (k_s-k_a)*cslatsq(i,j)*cslatsq(i,j)* & - MAX(0.0D0,(etam - sigma_b)*rec_one_minus_sigma_b) - Teq = MAX(200.0D0,(315.0D0 - dT_y*snlatsq(i,j) - dtheta_z*logprat*cslatsq(i,j))*pratk) - hs_T_frc(i,j,k)= -k_t(i,j)*(T(i,j,k)-Teq) - end do -#else do i=1,npts p = hvcoord%hyam(k)*hvcoord%ps0 + hvcoord%hybm(k)*ps(i,j) logprat = LOG(p)-logps0 @@ -272,7 +200,6 @@ function hs_T_forcing(hvcoord,ps,T,sphere,npts,nlevels) result(hs_T_frc) #endif hs_T_frc(i,j,k)= -k_t(i,j)*(T(i,j,k)-Teq) end do -#endif /* _USE_VECTOR */ end do end do @@ -289,11 +216,12 @@ subroutine hs0_init_state(elem, hybrid, hvcoord,nets,nete,Tinit) ! Local variables - integer ie,i,j,k,q + integer ie,i,j,k,q,tl integer :: nm1 integer :: n0 integer :: np1 real (kind=real_kind) :: lat_mtn,lon_mtn,r_mtn,h_mtn,rsq,lat,lon + real (kind=real_kind) :: temperature(np,np,nlev) if (hybrid%masterthread) write(iulog,*) 'initializing Held-Suarez primitive equations test' @@ -324,10 +252,6 @@ subroutine hs0_init_state(elem, hybrid, hvcoord,nets,nete,Tinit) enddo #endif - elem(ie)%state%T(:,:,:,n0) =Tinit - elem(ie)%state%T(:,:,:,nm1)=elem(ie)%state%T(:,:,:,n0) - elem(ie)%state%T(:,:,:,np1)=elem(ie)%state%T(:,:,:,n0) - elem(ie)%state%v(:,:,:,:,n0) =0.0D0 elem(ie)%state%v(:,:,:,:,nm1)=elem(ie)%state%v(:,:,:,:,n0) elem(ie)%state%v(:,:,:,:,np1)=elem(ie)%state%v(:,:,:,:,n0) @@ -336,14 +260,18 @@ subroutine hs0_init_state(elem, hybrid, hvcoord,nets,nete,Tinit) q=1 elem(ie)%state%Q(:,:,:,q) =0 ! moist HS tracer IC=0 do q=2,qsize - elem(ie)%state%Q(:,:,:,q) =elem(ie)%state%T(:,:,:,n0)/400 + elem(ie)%state%Q(:,:,:,q) =temperature(:,:,:)/400 enddo endif + + temperature(:,:,:)=Tinit + do tl=1,timelevels + call set_thermostate(elem(ie),temperature,hvcoord,n0,1) + enddo end do end subroutine hs0_init_state -#endif end module held_suarez_mod diff --git a/components/homme/test_execs/CMakeLists.txt b/components/homme/test_execs/CMakeLists.txt index bb6120176ed3..830406841934 100644 --- a/components/homme/test_execs/CMakeLists.txt +++ b/components/homme/test_execs/CMakeLists.txt @@ -164,7 +164,6 @@ ENDIF() IF(${BUILD_HOMME_PREQX}) ADD_SUBDIRECTORY(baroC) ADD_SUBDIRECTORY(baroCam) - ADD_SUBDIRECTORY(preqx-nlev20-interp) ADD_SUBDIRECTORY(preqx-nlev20-native) ADD_SUBDIRECTORY(preqx-nlev30-interp) @@ -176,6 +175,13 @@ IF(${BUILD_HOMME_PREQX}) ENDIF() ENDIF() +# Add the test exec subdirs for the prim executable +IF(${BUILD_HOMME_PESE}) + ADD_SUBDIRECTORY(pese-nlev20) + ADD_SUBDIRECTORY(pese-nlev30) + ADD_SUBDIRECTORY(pese-nlev60) +ENDIF() + # Add the test exec subdirs for the prim executable IF(${BUILD_HOMME_PRIM}) ADD_SUBDIRECTORY(primA) diff --git a/components/homme/test_execs/pese-nlev20/CMakeLists.txt b/components/homme/test_execs/pese-nlev20/CMakeLists.txt new file mode 100644 index 000000000000..4165d5c7d791 --- /dev/null +++ b/components/homme/test_execs/pese-nlev20/CMakeLists.txt @@ -0,0 +1,6 @@ +#_______________________________________________________________________ +# create executable needed by DCMIP2012_test3.0 (interpolated output) + +pese_setup() +# name target NP NC PLEV USE_PIO WITH_ENERGY QSIZE_D +createTestExec(pese-nlev20 pese 4 4 20 FALSE FALSE 4 ) diff --git a/components/homme/test_execs/pese-nlev30/CMakeLists.txt b/components/homme/test_execs/pese-nlev30/CMakeLists.txt new file mode 100644 index 000000000000..fe9ee356c21c --- /dev/null +++ b/components/homme/test_execs/pese-nlev30/CMakeLists.txt @@ -0,0 +1,7 @@ +#_______________________________________________________________________ +# create executable needed by DCMIP2012_test2.0 (interpolated output) + +pese_setup() +# name target NP NC PLEV USE_PIO WITH_ENERGY QSIZE_D +createTestExec(pese-nlev30 pese 4 4 30 FALSE FALSE 4 ) + diff --git a/components/homme/test_execs/pese-nlev60/CMakeLists.txt b/components/homme/test_execs/pese-nlev60/CMakeLists.txt new file mode 100644 index 000000000000..9463fc8905c9 --- /dev/null +++ b/components/homme/test_execs/pese-nlev60/CMakeLists.txt @@ -0,0 +1,7 @@ +#_______________________________________________________________________ +# create executable needed by DCMIP2012_test2.x (interpolated output) + +pese_setup() +# name target NP NC PLEV USE_PIO WITH_ENERGY QSIZE_D +createTestExec(pese-nlev60 pese 4 4 60 FALSE FALSE 4 ) +