From af8b165592e0fb368d65a7ffba8eea262ed07b42 Mon Sep 17 00:00:00 2001 From: bruno villasenor Date: Sun, 17 Mar 2024 17:17:28 -0700 Subject: [PATCH 1/6] added host files for Lockhart (AMD internal system) --- builds/make.host.lockhart | 25 +++++++++++++++++++++++++ builds/setup.lockhart.cce.sh | 14 ++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 builds/make.host.lockhart create mode 100644 builds/setup.lockhart.cce.sh diff --git a/builds/make.host.lockhart b/builds/make.host.lockhart new file mode 100644 index 000000000..34c3f2049 --- /dev/null +++ b/builds/make.host.lockhart @@ -0,0 +1,25 @@ +#-- make.host for AMD Lockhart +#-- Compiler and flags for different build type +CXX = CC +GPUCXX ?= hipcc + +CXXFLAGS_DEBUG = -g -O0 -std=c++17 +CXXFLAGS_OPTIMIZE = -g -Ofast -std=c++17 -Wno-unused-result + +GPUFLAGS_OPTIMIZE = -std=c++17 --offload-arch=gfx90a -Wall -Wno-unused-result -munsafe-fp-atomics +GPUFLAGS_DEBUG = -g -O0 -std=c++17 --offload-arch=gfx90a -Wall -Wno-unused-result +HIPCONFIG = -I$(ROCM_PATH)/include $(shell hipconfig -C) # workaround for Rocm 5.2 warnings +#HIPCONFIG = $(shell hipconfig -C) + +OMP_NUM_THREADS = 7 +#-- How to launch job +JOB_LAUNCH = srun -u -n 1 -c 8 + +#-- Library +MPI_ROOT = ${CRAY_MPICH_DIR} +#HDF5_ROOT = ${HDF5_ROOT} +#FFTW_ROOT = $(shell dirname $(FFTW_DIR)) +GOOGLETEST_ROOT := $(if $(GOOGLETEST_ROOT),$(GOOGLETEST_ROOT),$(OLCF_GOOGLETEST_ROOT)) + +#-- Use GPU-aware MPI +MPI_GPU = -DMPI_GPU \ No newline at end of file diff --git a/builds/setup.lockhart.cce.sh b/builds/setup.lockhart.cce.sh new file mode 100644 index 000000000..a010ca409 --- /dev/null +++ b/builds/setup.lockhart.cce.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +#-- This script needs to be source-d in the terminal, e.g. +# source ./setup.lockhart.cce.sh + +module load cray-python +module load rocm +module load craype-accel-amd-gfx90a +module load cray-hdf5 cray-fftw + +#-- GPU-aware MPI +export MPICH_GPU_SUPPORT_ENABLED=1 + +export CHOLLA_ENVSET=1 From 9f1272ae7d9d5f9f4338ba2daa8656e9280a13e4 Mon Sep 17 00:00:00 2001 From: bruno villasenor Date: Sun, 17 Mar 2024 17:18:27 -0700 Subject: [PATCH 2/6] added script to load a distributed snapshot --- python_scripts/io_tools.py | 234 ++++++++++++++++++ .../load_cholla_snapshot_distributed.py | 58 +++++ 2 files changed, 292 insertions(+) create mode 100644 python_scripts/io_tools.py create mode 100644 python_scripts/load_cholla_snapshot_distributed.py diff --git a/python_scripts/io_tools.py b/python_scripts/io_tools.py new file mode 100644 index 000000000..d3a2cd82a --- /dev/null +++ b/python_scripts/io_tools.py @@ -0,0 +1,234 @@ +import sys, os +import h5py as h5 +import numpy as np + + +# Open Cholla file. Works with new and original output formatting +def open_cholla_file( data_type, input_dir, snapshot_id, box_id ): + if data_type == 'hydro': base_name = '' + elif data_type == 'particles': base_name = '_particles' + elif data_type == 'gravity': base_name = '_gravity' + else: + print(f'ERROR: not valid data_type: {data_type} ') + new_input_dir = input_dir + f'{snapshot_id}/' + file_name = f'{snapshot_id}{base_name}.h5.{box_id}' + + # Try loading with the new formatting + try: + input_file = h5.File( new_input_dir + file_name, 'r') + except: + found_file = False + else: + found_file = True + if found_file: return input_file + + # Try loading with the original formatting + try: + input_file = h5.File( input_dir + file_name, 'r') + except: + found_file = False + else: + found_file = True + if found_file: return input_file + else: + print( f'File {file_name} not found in input directory: {new_input_dir}') + print( f'File {file_name} not found in input directory: {input_dir}') + print( f'ERROR: Unable to load the snapshot file') + sys.exit() + + +def get_domain_block( proc_grid, box_bounds, box_size, grid_size ): + np_x, np_y, np_z = proc_grid + Lx, Ly, Lz = box_size + nx_g, ny_g, nz_g = grid_size + dx, dy, dz = Lx/np_x, Ly/np_y, Lz/np_z + nx_l, ny_l, nz_l = nx_g//np_x, ny_g//np_y, nz_g//np_z, + + bound_x, bound_y, bound_z = box_bounds + + domain = {} + domain['global'] = {} + domain['global']['dx'] = dx + domain['global']['dy'] = dy + domain['global']['dz'] = dz + for k in range(np_z): + for j in range(np_y): + for i in range(np_x): + pId = i + j*np_x + k*np_x*np_y + domain[pId] = { 'box':{}, 'grid':{} } + xMin, xMax = bound_x+i*dx, bound_x+(i+1)*dx + yMin, yMax = bound_y+j*dy, bound_y+(j+1)*dy + zMin, zMax = bound_z+k*dz, bound_z+(k+1)*dz + domain[pId]['box']['x'] = [xMin, xMax] + domain[pId]['box']['y'] = [yMin, yMax] + domain[pId]['box']['z'] = [zMin, zMax] + domain[pId]['box']['dx'] = dx + domain[pId]['box']['dy'] = dy + domain[pId]['box']['dz'] = dz + domain[pId]['box']['center_x'] = ( xMin + xMax )/2. + domain[pId]['box']['center_y'] = ( yMin + yMax )/2. + domain[pId]['box']['center_z'] = ( zMin + zMax )/2. + gxMin, gxMax = i*nx_l, (i+1)*nx_l + gyMin, gyMax = j*ny_l, (j+1)*ny_l + gzMin, gzMax = k*nz_l, (k+1)*nz_l + domain[pId]['grid']['x'] = [gxMin, gxMax] + domain[pId]['grid']['y'] = [gyMin, gyMax] + domain[pId]['grid']['z'] = [gzMin, gzMax] + return domain + +def select_procid( proc_id, subgrid, domain, ids, ax ): + domain_l, domain_r = domain + subgrid_l, subgrid_r = subgrid + if domain_l <= subgrid_l and domain_r > subgrid_l: + ids.append(proc_id) + if domain_l >= subgrid_l and domain_r <= subgrid_r: + ids.append(proc_id) + if domain_l < subgrid_r and domain_r >= subgrid_r: + ids.append(proc_id) + + + + +def select_ids_to_load( subgrid, domain, proc_grid ): + subgrid_x, subgrid_y, subgrid_z = subgrid + nprocs = proc_grid[0] * proc_grid[1] * proc_grid[2] + ids_x, ids_y, ids_z = [], [], [] + for proc_id in range(nprocs): + domain_local = domain[proc_id] + domain_x = domain_local['grid']['x'] + domain_y = domain_local['grid']['y'] + domain_z = domain_local['grid']['z'] + select_procid( proc_id, subgrid_x, domain_x, ids_x, 'x' ) + select_procid( proc_id, subgrid_y, domain_y, ids_y, 'y' ) + select_procid( proc_id, subgrid_z, domain_z, ids_z, 'z' ) + set_x = set(ids_x) + set_y = set(ids_y) + set_z = set(ids_z) + set_ids = (set_x.intersection(set_y)).intersection(set_z ) + return list(set_ids) + + +def load_cholla_snapshot_distributed( data_type, fields_to_load, snapshot_id, input_directory, precision=np.float64, subgrid=None, show_progress=True, print_available_fields=True ): + + + + if input_directory[-1] != '/': input_directory += '/' + print( f'Input directory: {input_directory}') + + box_id = 0 + data = open_cholla_file( data_type, input_directory, snapshot_id, box_id ) + + domain_bounds = data.attrs['bounds'] + domain_length = data.attrs['domain'] + grid_size = data.attrs['dims'] + proc_grid= data.attrs['nprocs'] + available_fields = data.keys() + print( f'Domain Bounds: {domain_bounds} \nDomain Length: {domain_length} \nGrid size: {grid_size}\nProcs grid: {proc_grid}') + if print_available_fields: print( f'Available Fields: {available_fields}') + data.close() + + # Get the domain_decomposition + domain = get_domain_block( proc_grid, domain_bounds, domain_length, grid_size ) + + # Find the box ids to load given the subgrid + if not subgrid: + subgrid = [ [0, grid_size[0]], [0, grid_size[1]], [0, grid_size[2]] ] + else: + for i,sg in enumerate(subgrid): + if sg[0] < 0 : + print(f'ERROR: subgrid along {i} axis: {sg} is less than zero: Setting to {[0, sg[1]]} ') + sg[0] = 0 + if sg[1] == -1: sg[1] = grid_size[i] # If -1 is pass as the upper bound, use the grid size for that axis + if sg[1] < sg[0] : + sg[1] = sg[0] + 1 + print(f'ERROR: subgrid along {i} axis: right index is less than left index. Setting to slice of width 1 {sg} ') + if sg[1] > grid_size[i]: + print(f'ERROR: subgrid along {i} axis: {sg} is larger than the grid size: {grid_size[i]}. Setting to {[sg[0], grid_size[i] ]} ') + print( f'Loading subgrid: {subgrid}') + + + ids_to_load = select_ids_to_load( subgrid, domain, proc_grid ) + + # print(("Loading Snapshot: {0}".format(nSnap))) + #Find the boundaries of the volume to load + domains = { 'x':{'l':[], 'r':[]}, 'y':{'l':[], 'r':[]}, 'z':{'l':[], 'r':[]}, } + for id in ids_to_load: + for ax in list(domains.keys()): + d_l, d_r = domain[id]['grid'][ax] + domains[ax]['l'].append(d_l) + domains[ax]['r'].append(d_r) + boundaries = {} + for ax in list(domains.keys()): + boundaries[ax] = [ min(domains[ax]['l']), max(domains[ax]['r']) ] + + # Get the size of the volume to load + nx = int(boundaries['x'][1] - boundaries['x'][0]) + ny = int(boundaries['y'][1] - boundaries['y'][0]) + nz = int(boundaries['z'][1] - boundaries['z'][0]) + + print( f'Loading {data_type} data:') + + dims_all = [ nx, ny, nz ] + data_out = {} + for field in fields_to_load: + data_particles = False + if field in ['pos_x', 'pos_y', 'pos_z', 'vel_x', 'vel_y', 'vel_z', 'particle_IDs']: data_particles = True + if not data_particles: data_all = np.zeros( dims_all, dtype=precision ) + else: data_all = [] + added_header = False + n_to_load = len(ids_to_load) + + for i, box_id in enumerate(ids_to_load): + input_file = open_cholla_file( data_type, input_directory, snapshot_id, box_id ) + head = input_file.attrs + if not added_header : + for h_key in list(head.keys()): + if h_key in ['dims', 'dims_local', 'offset', 'bounds', 'domain', 'dx', ]: continue + data_out[h_key] = head[h_key][0] + added_header = True + + if show_progress: + terminalString = '\r Loading File: {0}/{1} {2} {3}'.format(i+1, n_to_load, data_type, field) + sys.stdout. write(terminalString) + sys.stdout.flush() + + if not data_particles: + procStart_x, procStart_y, procStart_z = head['offset'] + procEnd_x, procEnd_y, procEnd_z = head['offset'] + head['dims_local'] + # Subtract the offsets + procStart_x -= boundaries['x'][0] + procEnd_x -= boundaries['x'][0] + procStart_y -= boundaries['y'][0] + procEnd_y -= boundaries['y'][0] + procStart_z -= boundaries['z'][0] + procEnd_z -= boundaries['z'][0] + procStart_x, procEnd_x = int(procStart_x), int(procEnd_x) + procStart_y, procEnd_y = int(procStart_y), int(procEnd_y) + procStart_z, procEnd_z = int(procStart_z), int(procEnd_z) + data_local = input_file[field][...] + data_all[ procStart_x:procEnd_x, procStart_y:procEnd_y, procStart_z:procEnd_z] = data_local + else: + data_local = input_file[field][...] + data_all.append( data_local ) + + # Trim off the excess data outside of the subgrid: + if not data_particles: + trim_x_l = subgrid[0][0] - boundaries['x'][0] + trim_x_r = boundaries['x'][1] - subgrid[0][1] + trim_y_l = subgrid[1][0] - boundaries['y'][0] + trim_y_r = boundaries['y'][1] - subgrid[1][1] + trim_z_l = subgrid[2][0] - boundaries['z'][0] + trim_z_r = boundaries['z'][1] - subgrid[2][1] + trim_x_l, trim_x_r = int(trim_x_l), int(trim_x_r) + trim_y_l, trim_y_r = int(trim_y_l), int(trim_y_r) + trim_z_l, trim_z_r = int(trim_z_l), int(trim_z_r) + data_output = data_all[trim_x_l:nx-trim_x_r, trim_y_l:ny-trim_y_r, trim_z_l:nz-trim_z_r, ] + data_out[field] = data_output + + else: + data_all = np.concatenate( data_all ) + data_out[field] = data_all + if field == 'particle_IDs': data_out[field] = data_out[field].astype( np.int64 ) + + if show_progress: print("") + return data_out \ No newline at end of file diff --git a/python_scripts/load_cholla_snapshot_distributed.py b/python_scripts/load_cholla_snapshot_distributed.py new file mode 100644 index 000000000..509f7ed43 --- /dev/null +++ b/python_scripts/load_cholla_snapshot_distributed.py @@ -0,0 +1,58 @@ +import sys +import h5py as h5 +import numpy as np +from io_tools import load_cholla_snapshot_distributed + +# Script to load Cholla snapshot files without having to merge them into a single file before. +# This is specially useful when dealing with large grids and duplicating the memory footprint +# of a snapshot becomes inconvenient due to memory and time limitations. + +# The input directory where the snapshots are located +data_directory = '/lustre/user/bvillase/benchmarcks/cholla/cosmology/256_50Mpc_adiabatic_nmpi8/' +input_directory = data_directory + 'branch_cosmo/snapshot_files/' + +# The snapshot id +snapshot_id = 1 + +# The type of data to load. ['hydro', 'particles', 'gravity'] +data_type = 'hydro' + +# The fields to load +fields_to_load = ['density', 'momentum_x', 'Energy'] + +# Sometimes it is useful to load a subdomain, for example when only a slice of the domain is needed. +# In this case, only the minimum subset of the snapshot files is loaded +# If subgrid = None The entire domain will be loaded + +subgrid = None # The entire domain will be loaded + +# If subgrid is != None then the subdomain specified by the grid limits will be loaded. +# This is specially useful when dealing with very large snapshots that don't fully fit in memory, +# and only a slice or sub-volume needs to be loaded +# Examples: +# Load a sub-volume of 64 x 64 x 64 cells: +# subgrid = [[0,64], [64,128], [64,128]] + +# Load slice of dimensions of one cell width along the x axis full size over the y and z axis: +subgrid = [[0,1], [0,-1], [0,-1]] + + +# Load the data in a specified precision. for example when plotting an image, the data doesn't nit to be +# float64 precision and by lowering to float32 a larger volume can fit in memory +precision = np.float64 +# precision = np.float32 + +# Load the data +data_hydro = load_cholla_snapshot_distributed(data_type, fields_to_load, snapshot_id, input_directory, + precision=precision, subgrid=subgrid ) + +# Use the loaded fields for whatever you want +gas_density = data_hydro['density'] +gas_total_energy = data_hydro['Energy'] + + +# Load the particles density +fields_to_load = ['density'] +data_particles = load_cholla_snapshot_distributed('particles', fields_to_load, snapshot_id, input_directory, + precision=precision, subgrid=subgrid ) +particles_density = data_particles['density'] \ No newline at end of file From e1122d9cabd8314f64031b6b7341d31d56692c4c Mon Sep 17 00:00:00 2001 From: bruno villasenor Date: Sun, 17 Mar 2024 17:18:53 -0700 Subject: [PATCH 3/6] added script to compare two snapshots --- python_scripts/compare_snapshots.py | 70 +++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 python_scripts/compare_snapshots.py diff --git a/python_scripts/compare_snapshots.py b/python_scripts/compare_snapshots.py new file mode 100644 index 000000000..edb2c92e5 --- /dev/null +++ b/python_scripts/compare_snapshots.py @@ -0,0 +1,70 @@ +import sys +import h5py as h5 +import numpy as np +import argparse +from io_tools import load_cholla_snapshot_distributed + +CLI=argparse.ArgumentParser() +CLI.add_argument( "--type", action='store', dest='type', default='hydro') +CLI.add_argument( "--snap_id", type=int, action='store', dest='snap_id', default=0 ) +CLI.add_argument( "--snap_id_0", type=int, action='store', dest='snap_id_0', default=None ) +CLI.add_argument( "--snap_id_1", type=int, action='store', dest='snap_id_1', default=None ) +CLI.add_argument( "--dir_0", action='store', dest='dir_0', default='./') +CLI.add_argument( "--dir_1", action='store', dest='dir_1', default='./') +CLI.add_argument( "--fields", nargs="*", type=str, default=[] ) +CLI.add_argument( "--tolerance", type=float, action='store', dest='tolerance', default=1e-8) + +# parse the command line +args = CLI.parse_args() +data_type = args.type +fields = args.fields +snap_id = args.snap_id +snap_id_0 = args.snap_id_0 +snap_id_1 = args.snap_id_1 +dir_0 = args.dir_0 +dir_1 = args.dir_1 +tolerance = args.tolerance + +print(f'Data type: {data_type}') +print(f'Snapshot id: {snap_id}') +print(f'Fields: {fields}') +print(f'Input directory 0: {dir_0}') +print(f'Input directory 1: {dir_1}') +print(f'Tolerance: {tolerance}') + +# Load first snapshot +data_snap_0 = load_cholla_snapshot_distributed( data_type, fields, snap_id, dir_0 ) + +# Load second snapshot +data_snap_1 = load_cholla_snapshot_distributed( data_type, fields, snap_id, dir_1 ) + +# Compare fields +print( 'Comparing fields...') +validation_passed = True +for field in fields: + data_0 = data_snap_0[field] + data_1 = data_snap_1[field] + diff = data_1 - data_0 + indices = np.where( data_0 != 0 ) + diff[indices] /= data_0[indices] + diff = np.abs( diff ) + diff_min = diff.min() + diff_max = diff.max() + diff_avg = diff.mean() + if diff_max < tolerance: pass_text = 'PASSED' + else: + pass_text = 'FAILED' + validation_passed = False + print( f'{field:<10} diff. min: {diff_min:.2e} max: {diff_max:.2e} avg: {diff_avg:.2e} {pass_text}' ) + +if validation_passed: + print('VALIDATION PASSED') + sys.exit(0) +else: + print('VALIDATION FAILED') + sys.exit(1) + + + + + From a70b46809d228faeff29d199d3c77895c1b0f02e Mon Sep 17 00:00:00 2001 From: bruno villasenor Date: Sun, 17 Mar 2024 17:24:18 -0700 Subject: [PATCH 4/6] added Cosmology DM-Only 64^3 test --- .../dark_matter_only/64_N1/clean_test.sh | 6 ++ .../64_N1/output_a_values.txt | 2 + .../dark_matter_only/64_N1/parameter_file.txt | 43 ++++++++ .../dark_matter_only/64_N1/run_test.sh | 98 +++++++++++++++++++ 4 files changed, 149 insertions(+) create mode 100644 tests/cosmology/dark_matter_only/64_N1/clean_test.sh create mode 100644 tests/cosmology/dark_matter_only/64_N1/output_a_values.txt create mode 100644 tests/cosmology/dark_matter_only/64_N1/parameter_file.txt create mode 100644 tests/cosmology/dark_matter_only/64_N1/run_test.sh diff --git a/tests/cosmology/dark_matter_only/64_N1/clean_test.sh b/tests/cosmology/dark_matter_only/64_N1/clean_test.sh new file mode 100644 index 000000000..6026921fe --- /dev/null +++ b/tests/cosmology/dark_matter_only/64_N1/clean_test.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +echo "Cleaning Cosmology Dark Matter Only 64^3 test" + +rm run_output.log simulation_output.txt cholla.cosmology.* +rm -r ics_64_50Mpc_dmo reference_64_50Mpc_dmo snapshot_files \ No newline at end of file diff --git a/tests/cosmology/dark_matter_only/64_N1/output_a_values.txt b/tests/cosmology/dark_matter_only/64_N1/output_a_values.txt new file mode 100644 index 000000000..05f41f24d --- /dev/null +++ b/tests/cosmology/dark_matter_only/64_N1/output_a_values.txt @@ -0,0 +1,2 @@ +#Single output at z=0 (a=1.0) +1.0 \ No newline at end of file diff --git a/tests/cosmology/dark_matter_only/64_N1/parameter_file.txt b/tests/cosmology/dark_matter_only/64_N1/parameter_file.txt new file mode 100644 index 000000000..9a5430462 --- /dev/null +++ b/tests/cosmology/dark_matter_only/64_N1/parameter_file.txt @@ -0,0 +1,43 @@ +# +# Parameter File for a 3D Cosmological Simulation. +# + +###################################### +# number of grid cells in the x dimension +nx=64 +# number of grid cells in the y dimension +ny=64 +# number of grid cells in the z dimension +nz=64 +# output time +tout=1000 +# how often to output +outstep=1000 +# value of gamma +gamma=1.66666667 +# name of initial conditions +init=Read_Grid +nfile=0 +#Cosmological Parameters +H0=67.66 +Omega_M=0.3111 +Omega_L=0.6889 +Omega_b=0.0497 +scale_outputs_file=TEST_DIR/output_a_values.txt +# domain properties +xmin=0.0 +ymin=0.0 +zmin=0.0 +xlen=50000.0 +ylen=50000.0 +zlen=50000.0 +# type of boundary conditions +xl_bcnd=1 +xu_bcnd=1 +yl_bcnd=1 +yu_bcnd=1 +zl_bcnd=1 +zu_bcnd=1 +# path to output directory +indir=TEST_DIR/ics_64_50Mpc_dmo/N1_z100/ +outdir=TEST_DIR/snapshot_files/ diff --git a/tests/cosmology/dark_matter_only/64_N1/run_test.sh b/tests/cosmology/dark_matter_only/64_N1/run_test.sh new file mode 100644 index 000000000..cfbfacb52 --- /dev/null +++ b/tests/cosmology/dark_matter_only/64_N1/run_test.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +echo "Running Cosmology Dark Matter Only 64^3 test" + +cholla_root=$(eval "git rev-parse --show-toplevel") +cd $cholla_root +test_dir=${cholla_root}/tests/cosmology/dark_matter_only/64_N1 +echo "Cholla directory: $cholla_root" +echo "Test directory: $test_dir" + +if [[ -z "${CHOLLA_ENVSET}" ]]; then + echo "ERROR: The Cholla environment has not been set for your system." + echo "This is usually made by running: source builds/setup. " + echo "Make sure this is done before building Cholla " + exit 1 +fi + +# Get the MACHINE name +MACHINE=$(sh $cholla_root/builds/machine.sh) +echo "Machine: $MACHINE" + +# Build Cholla cosmology dark-matter only +BUILD_CMD="DFLAGS=-DONLY_PARTICLES make TYPE=cosmology -j" +echo "Building Cholla Cosmology DM-Only: $BUILD_CMD" +eval $BUILD_CMD + +cd $test_dir +CHOLLA_EXEC=${cholla_root}/bin/cholla.cosmology.${MACHINE} +if [[ -f $CHOLLA_EXEC ]]; then + echo "Found Cholla binary: $CHOLLA_EXEC" +else + echo "ERROR Cholla binary not found. $CHOLLA_EXEC" + exit 1 +fi + +# Download the initial conditions +ics_dir=$test_dir/ics_64_50Mpc_dmo/N1_z100 +ics_file=$ics_dir/0_particles.h5.0 +if [[ -f $ics_file ]]; then + echo "Found initial conditions file: $ics_file" +else + mkdir -p $ics_dir + echo "Downloading intial conditions file..." + wget https://www.dropbox.com/scl/fi/25ka9c9kc0csxybwy4gww/0_particles.h5.0?rlkey=cws8u6wlcfo0e53vrxd6k2ch6 -O $ics_file + if [[ -f $ics_file ]]; then + echo "Found initial conditions file: $ics_file" + else + echo "ERROR: Initial conditions file wans't dowloaded succesfully." + fi +fi + +# Create the output directory +if [[ ! -d snapshot_files ]]; then + mkdir snapshot_files +fi + +# Replave the TEST_DIR location in the parameter file +REPLACE_CMD="sed -i -e 's@TEST_DIR@'"${test_dir}"'@g' parameter_file.txt" +echo "Replicng TEST_DIR: $REPLACE_CMD" +eval $REPLACE_CMD + +# Run the simulation +cp $CHOLLA_EXEC . +RUN_CMD="./cholla.cosmology.${MACHINE} parameter_file.txt > simulation_output.txt" +echo "Running test: $RUN_CMD" +eval $RUN_CMD + +# # Download the reference snapshot +reference_dir=$test_dir/reference_64_50Mpc_dmo/N1_z0 +reference_file=$reference_dir/1_particles.h5.0 +if [[ -f $reference_file ]]; then + echo "Found reference snapshot: $reference_file" +else + mkdir -p $reference_dir + echo "Downloading reference snapshot file..." + wget https://www.dropbox.com/scl/fi/t7go9m3s1jwjnm9p2wkf4/1_particles.h5.0?rlkey=5of9te4qghtm7ge7l1ahldovg -O $reference_file + if [[ -f $reference_file ]]; then + echo "Found reference snapshot file: $reference_file" + else + echo "ERROR: Reference snapshot file wans't dowloaded succesfully." + fi +fi + +# Compare output against the reference snapshot +VALIDATION_CMD="python $cholla_root/python_scripts/compare_snapshots.py --type particles --snap_id 1 --fields density --dir_0 $reference_dir --dir_1 $test_dir/snapshot_files --tolerance 1e-8" +echo "Validating: $VALIDATION_CMD" +eval $VALIDATION_CMD +# Collace the output of the python script and evaluate if test passed +validation_result=$? +if [[ $validation_result == 0 ]]; then + echo "TEST PASSED" + exit 0 +else + echo "TEST FAILED" + exit 1 +fi + + From 4b091a9c42a435d574d57275eeeee74ec57abfd4 Mon Sep 17 00:00:00 2001 From: bruno villasenor Date: Sun, 17 Mar 2024 17:42:37 -0700 Subject: [PATCH 5/6] Update load_cholla_snapshot_distributed.py --- python_scripts/load_cholla_snapshot_distributed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_scripts/load_cholla_snapshot_distributed.py b/python_scripts/load_cholla_snapshot_distributed.py index 509f7ed43..6698bed1d 100644 --- a/python_scripts/load_cholla_snapshot_distributed.py +++ b/python_scripts/load_cholla_snapshot_distributed.py @@ -34,7 +34,7 @@ # subgrid = [[0,64], [64,128], [64,128]] # Load slice of dimensions of one cell width along the x axis full size over the y and z axis: -subgrid = [[0,1], [0,-1], [0,-1]] +# subgrid = [[0,1], [0,-1], [0,-1]] # Load the data in a specified precision. for example when plotting an image, the data doesn't nit to be From c193632670269987a3c85074445b156a92e9b470 Mon Sep 17 00:00:00 2001 From: bruno villasenor Date: Sun, 17 Mar 2024 18:00:19 -0700 Subject: [PATCH 6/6] make clean before build and exit test if initial conditions or reference snapshot are not downloaded correctly --- python_scripts/compare_snapshots.py | 2 +- python_scripts/load_cholla_snapshot_distributed.py | 2 +- tests/cosmology/dark_matter_only/64_N1/run_test.sh | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/python_scripts/compare_snapshots.py b/python_scripts/compare_snapshots.py index edb2c92e5..09165741b 100644 --- a/python_scripts/compare_snapshots.py +++ b/python_scripts/compare_snapshots.py @@ -55,7 +55,7 @@ else: pass_text = 'FAILED' validation_passed = False - print( f'{field:<10} diff. min: {diff_min:.2e} max: {diff_max:.2e} avg: {diff_avg:.2e} {pass_text}' ) + print( f'{field:<10} difference. min: {diff_min:.2e} max: {diff_max:.2e} avg: {diff_avg:.2e} {pass_text}' ) if validation_passed: print('VALIDATION PASSED') diff --git a/python_scripts/load_cholla_snapshot_distributed.py b/python_scripts/load_cholla_snapshot_distributed.py index 6698bed1d..5d02ddadb 100644 --- a/python_scripts/load_cholla_snapshot_distributed.py +++ b/python_scripts/load_cholla_snapshot_distributed.py @@ -55,4 +55,4 @@ fields_to_load = ['density'] data_particles = load_cholla_snapshot_distributed('particles', fields_to_load, snapshot_id, input_directory, precision=precision, subgrid=subgrid ) -particles_density = data_particles['density'] \ No newline at end of file +particles_density = data_particles['density'] diff --git a/tests/cosmology/dark_matter_only/64_N1/run_test.sh b/tests/cosmology/dark_matter_only/64_N1/run_test.sh index cfbfacb52..f12a62cab 100644 --- a/tests/cosmology/dark_matter_only/64_N1/run_test.sh +++ b/tests/cosmology/dark_matter_only/64_N1/run_test.sh @@ -19,7 +19,8 @@ fi MACHINE=$(sh $cholla_root/builds/machine.sh) echo "Machine: $MACHINE" -# Build Cholla cosmology dark-matter only +# Build Cholla cosmology dark-matter only +make clean BUILD_CMD="DFLAGS=-DONLY_PARTICLES make TYPE=cosmology -j" echo "Building Cholla Cosmology DM-Only: $BUILD_CMD" eval $BUILD_CMD @@ -46,6 +47,7 @@ else echo "Found initial conditions file: $ics_file" else echo "ERROR: Initial conditions file wans't dowloaded succesfully." + exit 1 fi fi @@ -65,7 +67,7 @@ RUN_CMD="./cholla.cosmology.${MACHINE} parameter_file.txt > simulation_output.tx echo "Running test: $RUN_CMD" eval $RUN_CMD -# # Download the reference snapshot +# Download the reference snapshot reference_dir=$test_dir/reference_64_50Mpc_dmo/N1_z0 reference_file=$reference_dir/1_particles.h5.0 if [[ -f $reference_file ]]; then @@ -78,6 +80,7 @@ else echo "Found reference snapshot file: $reference_file" else echo "ERROR: Reference snapshot file wans't dowloaded succesfully." + exit 1 fi fi @@ -85,7 +88,7 @@ fi VALIDATION_CMD="python $cholla_root/python_scripts/compare_snapshots.py --type particles --snap_id 1 --fields density --dir_0 $reference_dir --dir_1 $test_dir/snapshot_files --tolerance 1e-8" echo "Validating: $VALIDATION_CMD" eval $VALIDATION_CMD -# Collace the output of the python script and evaluate if test passed +# Collect the output of the python script and evaluate if test passed validation_result=$? if [[ $validation_result == 0 ]]; then echo "TEST PASSED"