diff --git a/environment_py3_win.yml b/environment_py3_win.yml index 9dd518c978..779ec70bd0 100644 --- a/environment_py3_win.yml +++ b/environment_py3_win.yml @@ -4,7 +4,7 @@ channels: dependencies: - python=3.6 - cachetools>=1.0.0 - - cgen + - cgen>=2020.1 - coverage - ffmpeg>=3.2.3,<3.2.6 - flake8>=2.1.0 diff --git a/parcels/__init__.py b/parcels/__init__.py index 4bb2616122..96e6403b04 100644 --- a/parcels/__init__.py +++ b/parcels/__init__.py @@ -4,6 +4,7 @@ from parcels.fieldset import * # noqa from parcels.particle import * # noqa from parcels.particleset import * # noqa +from parcels.particleset_benchmark import * # noqa from parcels.field import * # noqa from parcels.kernel import * # noqa import parcels.rng as random # noqa diff --git a/parcels/codegenerator.py b/parcels/codegenerator.py index 91f5740014..46341ad5cf 100644 --- a/parcels/codegenerator.py +++ b/parcels/codegenerator.py @@ -1,11 +1,11 @@ import ast import collections import math +import numpy as np import random from copy import copy import cgen as c -import numpy as np from parcels.field import Field from parcels.field import NestedField @@ -163,7 +163,7 @@ def __getattr__(self, attr): class ErrorCodeNode(IntrinsicNode): - symbol_map = {'Success': 'SUCCESS', 'Evaluate': 'EVALUATE', 'Repeat': 'REPEAT', 'Delete': 'DELETE', + symbol_map = {'Success': 'SUCCESS', 'Evaluate': 'EVALUATE', 'Repeat': 'REPEAT', 'Delete': 'DELETE', 'StopExecution': 'STOP_EXECUTION', 'Error': 'ERROR', 'ErrorInterpolation': 'ERROR_INTERPOLATION', 'ErrorOutOfBounds': 'ERROR_OUT_OF_BOUNDS', 'ErrorThroughSurface': 'ERROR_THROUGH_SURFACE'} diff --git a/parcels/examples/example_dask_chunk_OCMs.py b/parcels/examples/example_dask_chunk_OCMs.py new file mode 100644 index 0000000000..87c02cd064 --- /dev/null +++ b/parcels/examples/example_dask_chunk_OCMs.py @@ -0,0 +1,494 @@ +import math +from datetime import timedelta as delta +from glob import glob +from os import path + +import numpy as np +import pytest +import dask + +from parcels import AdvectionRK4 +from parcels import Field +from parcels import FieldSet +from parcels import JITParticle +from parcels import ParticleFile +from parcels import ParticleSet +from parcels import ScipyParticle +from parcels import Variable +from parcels import VectorField, NestedField, SummedField +from parcels.tools.loggers import logger + +ptype = {'scipy': ScipyParticle, 'jit': JITParticle} + + +def fieldset_from_nemo_3D(chunk_mode): + data_path = path.join(path.dirname(__file__), 'NemoNorthSeaORCA025-N006_data/') + ufiles = sorted(glob(data_path + 'ORCA*U.nc')) + vfiles = sorted(glob(data_path + 'ORCA*V.nc')) + wfiles = sorted(glob(data_path + 'ORCA*W.nc')) + mesh_mask = data_path + 'coordinates.nc' + + filenames = {'U': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': ufiles}, + 'V': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': vfiles}, + 'W': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': wfiles}} + variables = {'U': 'uo', + 'V': 'vo', + 'W': 'wo'} + dimensions = {'U': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}, + 'V': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}, + 'W': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}} + chs = False + if chunk_mode == 'auto': + chs = 'auto' + elif chunk_mode == 'specific': + chs = {'U': {'depthu': 75, 'depthv': 75, 'depthw': 75, 'y': 16, 'x': 16}, + 'V': {'depthu': 75, 'depthv': 75, 'depthw': 75, 'y': 16, 'x': 16}, + 'W': {'depthu': 75, 'depthv': 75, 'depthw': 75, 'y': 16, 'x': 16}} + + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, field_chunksize=chs) + return fieldset + + +def fieldset_from_globcurrent(chunk_mode): + filenames = path.join(path.dirname(__file__), 'GlobCurrent_example_data', + '200201*-GLOBCURRENT-L4-CUReul_hs-ALT_SUM-v02.0-fv01.0.nc') + variables = {'U': 'eastward_eulerian_current_velocity', 'V': 'northward_eulerian_current_velocity'} + dimensions = {'lat': 'lat', 'lon': 'lon', 'time': 'time'} + chs = False + if chunk_mode == 'auto': + chs = 'auto' + elif chunk_mode == 'specific': + chs = {'U': {'lat': 16, 'lon': 16}, + 'V': {'lat': 16, 'lon': 16}} + + fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, field_chunksize=chs) + return fieldset + + +def fieldset_from_pop_1arcs(chunk_mode): + filenames = path.join(path.join(path.dirname(__file__), 'POPSouthernOcean_data'), 't.x1_SAMOC_flux.1690*.nc') + variables = {'U': 'UVEL', 'V': 'VVEL', 'W': 'WVEL'} + timestamps = np.expand_dims(np.array([np.datetime64('2000-%.2d-01' % m) for m in range(1, 7)]), axis=1) + dimensions = {'lon': 'ULON', 'lat': 'ULAT', 'depth': 'w_dep'} + chs = False + if chunk_mode == 'auto': + chs = 'auto' + elif chunk_mode == 'specific': + chs = {'i': 8, 'j': 8, 'w_dep': 3} + + fieldset = FieldSet.from_pop(filenames, variables, dimensions, field_chunksize=chs, timestamps=timestamps) + return fieldset + + +def fieldset_from_swash(chunk_mode): + filenames = path.join(path.join(path.dirname(__file__), 'SWASH_data'), 'field_*.nc') + variables = {'U': 'cross-shore velocity', + 'V': 'along-shore velocity', + 'W': 'vertical velocity', + 'depth': 'time varying depth', + 'depth_u': 'time varying depth_u'} + dimensions = {'U': {'lon': 'x', 'lat': 'y', 'depth': 'not_yet_set', 'time': 't'}, + 'V': {'lon': 'x', 'lat': 'y', 'depth': 'not_yet_set', 'time': 't'}, + 'W': {'lon': 'x', 'lat': 'y', 'depth': 'not_yet_set', 'time': 't'}, + 'depth': {'lon': 'x', 'lat': 'y', 'depth': 'not_yet_set', 'time': 't'}, + 'depth_u': {'lon': 'x', 'lat': 'y', 'depth': 'not_yet_set', 'time': 't'}} + chs = False + if chunk_mode == 'auto': + chs = 'auto' + elif chunk_mode == 'specific': + chs = (1, 7, 4, 4) + fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, mesh='flat', allow_time_extrapolation=True, field_chunksize=chs) + fieldset.U.set_depth_from_field(fieldset.depth_u) + fieldset.V.set_depth_from_field(fieldset.depth_u) + fieldset.W.set_depth_from_field(fieldset.depth) + return fieldset + + +def compute_nemo_particle_advection(field_set, mode, lonp, latp): + + def periodicBC(particle, fieldSet, time): + if particle.lon > 15.0: + particle.lon -= 15.0 + if particle.lon < 0: + particle.lon += 15.0 + if particle.lat > 60.0: + particle.lat -= 11.0 + if particle.lat < 49.0: + particle.lat += 11.0 + + pset = ParticleSet.from_list(field_set, ptype[mode], lon=lonp, lat=latp) + pfile = ParticleFile("nemo_particles_chunk", pset, outputdt=delta(days=1)) + kernels = pset.Kernel(AdvectionRK4) + periodicBC + pset.execute(kernels, runtime=delta(days=4), dt=delta(hours=6), output_file=pfile) + return pset + + +def compute_globcurrent_particle_advection(field_set, mode, lonp, latp): + pset = ParticleSet(field_set, pclass=ptype[mode], lon=lonp, lat=latp) + pfile = ParticleFile("globcurrent_particles_chunk", pset, outputdt=delta(hours=2)) + pset.execute(AdvectionRK4, runtime=delta(days=1), dt=delta(minutes=5), output_file=pfile) + return pset + + +def compute_pop_particle_advection(field_set, mode, lonp, latp): + pset = ParticleSet.from_list(field_set, ptype[mode], lon=lonp, lat=latp) + pfile = ParticleFile("globcurrent_particles_chunk", pset, outputdt=delta(days=15)) + pset.execute(AdvectionRK4, runtime=delta(days=90), dt=delta(days=2), output_file=pfile) + return pset + + +def compute_swash_particle_advection(field_set, mode, lonp, latp, depthp): + pset = ParticleSet.from_list(field_set, ptype[mode], lon=lonp, lat=latp, depth=depthp) + pfile = ParticleFile("swash_particles_chunk", pset, outputdt=delta(seconds=0.05)) + pset.execute(AdvectionRK4, runtime=delta(seconds=0.2), dt=delta(seconds=0.005), output_file=pfile) + return pset + + +@pytest.mark.parametrize('mode', ['jit']) +@pytest.mark.parametrize('chunk_mode', [False, 'auto', 'specific']) +def test_nemo_3D(mode, chunk_mode): + if chunk_mode == 'auto': + dask.config.set({'array.chunk-size': '2MiB'}) + else: + dask.config.set({'array.chunk-size': '128MiB'}) + field_set = fieldset_from_nemo_3D(chunk_mode) + npart = 20 + lonp = 2.5 * np.ones(npart) + latp = [i for i in 52.0+(-1e-3+np.random.rand(npart)*2.0*1e-3)] + compute_nemo_particle_advection(field_set, mode, lonp, latp) + # Nemo sample file dimensions: depthu=75, y=201, x=151 + assert (len(field_set.U.grid.load_chunk) == len(field_set.V.grid.load_chunk)) + assert (len(field_set.U.grid.load_chunk) == len(field_set.W.grid.load_chunk)) + if chunk_mode is False: + assert (len(field_set.U.grid.load_chunk) == 1) + elif chunk_mode == 'auto': + assert (len(field_set.U.grid.load_chunk) != 1) + elif chunk_mode == 'specific': + assert (len(field_set.U.grid.load_chunk) == (1 * int(math.ceil(201.0/16.0)) * int(math.ceil(151.0/16.0)))) + + +@pytest.mark.parametrize('mode', ['jit']) +@pytest.mark.parametrize('chunk_mode', [False, 'auto', 'specific']) +def test_pop(mode, chunk_mode): + if chunk_mode == 'auto': + dask.config.set({'array.chunk-size': '1MiB'}) + else: + dask.config.set({'array.chunk-size': '128MiB'}) + field_set = fieldset_from_pop_1arcs(chunk_mode) + npart = 20 + lonp = 70.0 * np.ones(npart) + latp = [i for i in -45.0+(-0.25+np.random.rand(npart)*2.0*0.25)] + compute_pop_particle_advection(field_set, mode, lonp, latp) + # POP sample file dimensions: k=20, j=60, i=60 + assert (len(field_set.U.grid.load_chunk) == len(field_set.V.grid.load_chunk)) + assert (len(field_set.U.grid.load_chunk) == len(field_set.W.grid.load_chunk)) + if chunk_mode is False: + assert (len(field_set.U.grid.load_chunk) == 1) + elif chunk_mode == 'auto': + assert (len(field_set.U.grid.load_chunk) != 1) + elif chunk_mode == 'specific': + assert (len(field_set.U.grid.load_chunk) == (int(math.ceil(21.0/3.0)) * int(math.ceil(60.0/8.0)) * int(math.ceil(60.0/8.0)))) + + +@pytest.mark.parametrize('mode', ['jit']) +@pytest.mark.parametrize('chunk_mode', [False, 'auto', 'specific']) +def test_swash(mode, chunk_mode): + if chunk_mode == 'auto': + dask.config.set({'array.chunk-size': '32KiB'}) + else: + dask.config.set({'array.chunk-size': '128MiB'}) + field_set = fieldset_from_swash(chunk_mode) + npart = 20 + lonp = [i for i in 9.5 + (-0.2 + np.random.rand(npart) * 2.0 * 0.2)] + latp = [i for i in np.arange(start=12.3, stop=13.1, step=0.04)[0:20]] + depthp = [-0.1, ] * npart + compute_swash_particle_advection(field_set, mode, lonp, latp, depthp) + # SWASH sample file dimensions: t=1, z=7, z_u=6, y=21, x=51 + assert (len(field_set.U.grid.load_chunk) == len(field_set.V.grid.load_chunk)) + if chunk_mode != 'auto': + assert (len(field_set.U.grid.load_chunk) == len(field_set.W.grid.load_chunk)) + if chunk_mode is False: + assert (len(field_set.U.grid.load_chunk) == 1) + elif chunk_mode == 'auto': + assert (len(field_set.U.grid.load_chunk) != 1) + elif chunk_mode == 'specific': + assert (len(field_set.U.grid.load_chunk) == (1 * int(math.ceil(6.0 / 7.0)) * int(math.ceil(21.0 / 4.0)) * int(math.ceil(51.0 / 4.0)))) + assert (len(field_set.U.grid.load_chunk) == (1 * int(math.ceil(7.0 / 7.0)) * int(math.ceil(21.0 / 4.0)) * int(math.ceil(51.0 / 4.0)))) + + +@pytest.mark.parametrize('mode', ['jit']) +@pytest.mark.parametrize('chunk_mode', [False, 'auto', 'specific']) +def test_globcurrent_2D(mode, chunk_mode): + if chunk_mode == 'auto': + dask.config.set({'array.chunk-size': '32KiB'}) + else: + dask.config.set({'array.chunk-size': '128MiB'}) + field_set = fieldset_from_globcurrent(chunk_mode) + lonp = [25] + latp = [-35] + pset = compute_globcurrent_particle_advection(field_set, mode, lonp, latp) + # GlobCurrent sample file dimensions: time=UNLIMITED, lat=41, lon=81 + assert (len(field_set.U.grid.load_chunk) == len(field_set.V.grid.load_chunk)) + if chunk_mode is False: + assert (len(field_set.U.grid.load_chunk) == 1) + elif chunk_mode == 'auto': + assert (len(field_set.U.grid.load_chunk) != 1) + elif chunk_mode == 'specific': + assert (len(field_set.U.grid.load_chunk) == (1 * int(math.ceil(41.0/16.0)) * int(math.ceil(81.0/16.0)))) + assert(abs(pset[0].lon - 23.8) < 1) + assert(abs(pset[0].lat - -35.3) < 1) + + +@pytest.mark.parametrize('mode', ['jit']) +def test_diff_entry_dimensions_chunks(mode): + data_path = path.join(path.dirname(__file__), 'NemoNorthSeaORCA025-N006_data/') + ufiles = sorted(glob(data_path + 'ORCA*U.nc')) + vfiles = sorted(glob(data_path + 'ORCA*V.nc')) + mesh_mask = data_path + 'coordinates.nc' + + filenames = {'U': {'lon': mesh_mask, 'lat': mesh_mask, 'data': ufiles}, + 'V': {'lon': mesh_mask, 'lat': mesh_mask, 'data': vfiles}} + variables = {'U': 'uo', + 'V': 'vo'} + dimensions = {'U': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, + 'V': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}} + chs = {'U': {'depthu': 75, 'depthv': 75, 'y': 16, 'x': 16}, + 'V': {'depthu': 75, 'depthv': 75, 'y': 16, 'x': 16}} + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, field_chunksize=chs) + npart = 20 + lonp = 5.2 * np.ones(npart) + latp = [i for i in 52.0+(-1e-3+np.random.rand(npart)*2.0*1e-3)] + compute_nemo_particle_advection(fieldset, mode, lonp, latp) + # Nemo sample file dimensions: depthu=75, y=201, x=151 + assert (len(fieldset.U.grid.load_chunk) == len(fieldset.V.grid.load_chunk)) + + +# ==== TO BE EXTERNALIZED OR CHECKED WHEN #782 IS FIXED ==== # +@pytest.mark.parametrize('mode', ['scipy', 'jit']) +def test_3d_2dfield_sampling(mode): + logger.warning("Test is to be re-enabled after #782 is fixed to test tertiary effects.") + return True + data_path = path.join(path.dirname(__file__), 'NemoNorthSeaORCA025-N006_data/') + ufiles = sorted(glob(data_path + 'ORCA*U.nc')) + vfiles = sorted(glob(data_path + 'ORCA*V.nc')) + mesh_mask = data_path + 'coordinates.nc' + + filenames = {'U': {'lon': mesh_mask, 'lat': mesh_mask, 'data': ufiles}, + 'V': {'lon': mesh_mask, 'lat': mesh_mask, 'data': vfiles}, + # 'nav_lon': {'lon': mesh_mask, 'lat': mesh_mask, 'data': ufiles[0]}} + 'nav_lon': {'lon': mesh_mask, 'lat': mesh_mask, 'data': [ufiles[0], ]}} + variables = {'U': 'uo', + 'V': 'vo', + 'nav_lon': 'nav_lon'} + dimensions = {'U': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, + 'V': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, + 'nav_lon': {'lon': 'glamf', 'lat': 'gphif'}} + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, field_chunksize=False) + fieldset.nav_lon.data = np.ones(fieldset.nav_lon.data.shape, dtype=np.float32) + fieldset.add_field(Field('rectilinear_2D', np.ones((2, 2)), + lon=np.array([-10, 20]), lat=np.array([40, 80]), field_chunksize=False)) + + class MyParticle(ptype[mode]): + sample_var_curvilinear = Variable('sample_var_curvilinear') + sample_var_rectilinear = Variable('sample_var_rectilinear') + pset = ParticleSet(fieldset, pclass=MyParticle, lon=2.5, lat=52) + + def Sample2D(particle, fieldset, time): + particle.sample_var_curvilinear += fieldset.nav_lon[time, particle.depth, particle.lat, particle.lon] + particle.sample_var_rectilinear += fieldset.rectilinear_2D[time, particle.depth, particle.lat, particle.lon] + + runtime, dt = 86400*4, 6*3600 + pset.execute(pset.Kernel(AdvectionRK4) + Sample2D, runtime=runtime, dt=dt) + print(pset.xi) + + for f in fieldset.get_fields(): + if type(f) in [VectorField, NestedField, SummedField]: # or not f.grid.defer_load: + continue + g = f.grid + npart = 1 + npart = [npart * k for k in f.nchunks[1:]] + print("Field '{}': grid type: {}; grid chunksize: {}; grid mesh: {}; field N partitions: {}; field nchunks: {}; grid chunk_info: {}; grid load_chunk: {}; grid layout: {}".format(f.name, g.gtype, g.master_chunksize, g.mesh, npart, f.nchunks, g.chunk_info, g.load_chunk, (g.tdim, g.zdim, g.ydim, g.xdim))) + for i in range(0, len(fieldset.gridset.grids)): + g = fieldset.gridset.grids[i] + print( + "Grid {}: grid type: {}; grid chunksize: {}; grid mesh: {}; grid chunk_info: {}; grid load_chunk: {}; grid layout: {}".format( + i, g.gtype, g.master_chunksize, g.mesh, g.chunk_info, g.load_chunk, (g.tdim, g.zdim, g.ydim, g.xdim))) + + assert pset.sample_var_rectilinear == runtime/dt + assert pset.sample_var_curvilinear == runtime/dt + + +@pytest.mark.parametrize('mode', ['jit']) +def test_diff_entry_chunksize_error_nemo_simple(mode): + data_path = path.join(path.dirname(__file__), 'NemoNorthSeaORCA025-N006_data/') + ufiles = sorted(glob(data_path + 'ORCA*U.nc')) + vfiles = sorted(glob(data_path + 'ORCA*V.nc')) + wfiles = sorted(glob(data_path + 'ORCA*W.nc')) + mesh_mask = data_path + 'coordinates.nc' + + filenames = {'U': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': ufiles}, + 'V': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': vfiles}, + 'W': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': wfiles}} + variables = {'U': 'uo', + 'V': 'vo', + 'W': 'wo'} + dimensions = {'U': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}, + 'V': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}, + 'W': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}} + chs = {'U': {'depthu': 75, 'y': 16, 'x': 16}, + 'V': {'depthv': 20, 'y': 4, 'x': 16}, + 'W': {'depthw': 15, 'y': 16, 'x': 4}} + try: + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, field_chunksize=chs) + except ValueError: + return True + npart = 20 + lonp = 5.2 * np.ones(npart) + latp = [i for i in 52.0+(-1e-3+np.random.rand(npart)*2.0*1e-3)] + compute_nemo_particle_advection(fieldset, mode, lonp, latp) + return False + + +@pytest.mark.parametrize('mode', ['jit']) +def test_diff_entry_chunksize_error_nemo_complex_conform_depth(mode): + # ==== this test is expected to fall-back to a pre-defined minimal chunk as ==== # + # ==== the requested chunks don't match, or throw a value error. ==== # + data_path = path.join(path.dirname(__file__), 'NemoNorthSeaORCA025-N006_data/') + ufiles = sorted(glob(data_path + 'ORCA*U.nc')) + vfiles = sorted(glob(data_path + 'ORCA*V.nc')) + wfiles = sorted(glob(data_path + 'ORCA*W.nc')) + mesh_mask = data_path + 'coordinates.nc' + + filenames = {'U': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': ufiles}, + 'V': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': vfiles}, + 'W': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': wfiles}} + variables = {'U': 'uo', + 'V': 'vo', + 'W': 'wo'} + dimensions = {'U': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}, + 'V': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}, + 'W': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}} + chs = {'U': {'depthu': 75, 'depthv': 75, 'depthw': 75, 'y': 16, 'x': 16}, + 'V': {'depthu': 75, 'depthv': 75, 'depthw': 75, 'y': 4, 'x': 16}, + 'W': {'depthu': 75, 'depthv': 75, 'depthw': 75, 'y': 16, 'x': 4}} + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, field_chunksize=chs) + npart = 20 + lonp = 5.2 * np.ones(npart) + latp = [i for i in 52.0+(-1e-3+np.random.rand(npart)*2.0*1e-3)] + compute_nemo_particle_advection(fieldset, mode, lonp, latp) + # Nemo sample file dimensions: depthu=75, y=201, x=151 + npart_U = 1 + npart_U = [npart_U * k for k in fieldset.U.nchunks[1:]] + npart_V = 1 + npart_V = [npart_V * k for k in fieldset.V.nchunks[1:]] + npart_W = 1 + npart_W = [npart_W * k for k in fieldset.V.nchunks[1:]] + chn = {'U': {'lat': int(math.ceil(201.0/chs['U']['y'])), + 'lon': int(math.ceil(151.0/chs['U']['x'])), + 'depth': int(math.ceil(75.0/chs['U']['depthu']))}, + 'V': {'lat': int(math.ceil(201.0/chs['V']['y'])), + 'lon': int(math.ceil(151.0/chs['V']['x'])), + 'depth': int(math.ceil(75.0/chs['V']['depthv']))}, + 'W': {'lat': int(math.ceil(201.0/chs['W']['y'])), + 'lon': int(math.ceil(151.0/chs['W']['x'])), + 'depth': int(math.ceil(75.0/chs['W']['depthw']))}} + npart_U_request = 1 + npart_U_request = [npart_U_request * chn['U'][k] for k in chn['U']] + npart_V_request = 1 + npart_V_request = [npart_V_request * chn['V'][k] for k in chn['V']] + npart_W_request = 1 + npart_W_request = [npart_W_request * chn['W'][k] for k in chn['W']] + assert (len(fieldset.U.grid.load_chunk) == len(fieldset.V.grid.load_chunk)) + assert (len(fieldset.U.grid.load_chunk) == len(fieldset.W.grid.load_chunk)) + assert (npart_U == npart_V) + assert (npart_U == npart_W) + assert (npart_U != npart_U_request) + assert (npart_V != npart_V_request) + assert (npart_W != npart_W_request) + + +@pytest.mark.parametrize('mode', ['jit']) +def test_diff_entry_chunksize_error_nemo_complex_nonconform_depth(mode): + # ==== this test is expected to fall-back to a pre-defined minimal chunk as the ==== # + # ==== requested chunks don't match, or throw a value error ==== # + data_path = path.join(path.dirname(__file__), 'NemoNorthSeaORCA025-N006_data/') + ufiles = sorted(glob(data_path + 'ORCA*U.nc')) + vfiles = sorted(glob(data_path + 'ORCA*V.nc')) + wfiles = sorted(glob(data_path + 'ORCA*W.nc')) + mesh_mask = data_path + 'coordinates.nc' + + filenames = {'U': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': ufiles}, + 'V': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': vfiles}} + variables = {'U': 'uo', + 'V': 'vo'} + dimensions = {'U': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}, + 'V': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}} + chs = {'U': {'depthu': 75, 'depthv': 15, 'y': 16, 'x': 16}, + 'V': {'depthu': 75, 'depthv': 15, 'y': 4, 'x': 16}} + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, field_chunksize=chs) + npart = 20 + lonp = 5.2 * np.ones(npart) + latp = [i for i in 52.0+(-1e-3+np.random.rand(npart)*2.0*1e-3)] + try: + compute_nemo_particle_advection(fieldset, mode, lonp, latp) + except IndexError: # incorrect data access, in case grids were created + return True + except AssertionError: # U-V grids are not equal to one another, throwing assertion errors + return True + return False + + +@pytest.mark.parametrize('mode', ['jit']) +def test_erroneous_fieldset_init(mode): + data_path = path.join(path.dirname(__file__), 'NemoNorthSeaORCA025-N006_data/') + ufiles = sorted(glob(data_path + 'ORCA*U.nc')) + vfiles = sorted(glob(data_path + 'ORCA*V.nc')) + wfiles = sorted(glob(data_path + 'ORCA*W.nc')) + mesh_mask = data_path + 'coordinates.nc' + + filenames = {'U': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': ufiles}, + 'V': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': vfiles}, + 'W': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': wfiles}} + variables = {'U': 'uo', + 'V': 'vo', + 'W': 'wo'} + dimensions = {'U': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}, + 'V': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}, + 'W': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}} + chs = {'U': {'depthu': 75, 'y': 16, 'x': 16}, + 'V': {'depthv': 75, 'y': 16, 'x': 16}, + 'W': {'depthw': 75, 'y': 16, 'x': 16}} + + try: + FieldSet.from_nemo(filenames, variables, dimensions, field_chunksize=chs) + except ValueError: + return True + return False + + +@pytest.mark.parametrize('mode', ['jit']) +def test_diff_entry_chunksize_correction_globcurrent(mode): + filenames = path.join(path.dirname(__file__), 'GlobCurrent_example_data', + '200201*-GLOBCURRENT-L4-CUReul_hs-ALT_SUM-v02.0-fv01.0.nc') + variables = {'U': 'eastward_eulerian_current_velocity', 'V': 'northward_eulerian_current_velocity'} + dimensions = {'lat': 'lat', 'lon': 'lon', 'time': 'time'} + chs = {'U': {'lat': 16, 'lon': 16}, + 'V': {'lat': 16, 'lon': 4}} + fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, field_chunksize=chs) + lonp = [25] + latp = [-35] + compute_globcurrent_particle_advection(fieldset, mode, lonp, latp) + # GlobCurrent sample file dimensions: time=UNLIMITED, lat=41, lon=81 + npart_U = 1 + npart_U = [npart_U * k for k in fieldset.U.nchunks[1:]] + npart_V = 1 + npart_V = [npart_V * k for k in fieldset.V.nchunks[1:]] + npart_V_request = 1 + chn = {'U': {'lat': int(math.ceil(41.0/chs['U']['lat'])), + 'lon': int(math.ceil(81.0/chs['U']['lon']))}, + 'V': {'lat': int(math.ceil(41.0/chs['V']['lat'])), + 'lon': int(math.ceil(81.0/chs['V']['lon']))}} + npart_V_request = [npart_V_request * chn['V'][k] for k in chn['V']] + assert (npart_U == npart_V) + assert (npart_V != npart_V_request) + assert (len(fieldset.U.grid.load_chunk) == len(fieldset.V.grid.load_chunk)) diff --git a/parcels/examples/example_nemo_curvilinear.py b/parcels/examples/example_nemo_curvilinear.py index 40abaeb2c7..7131df1792 100644 --- a/parcels/examples/example_nemo_curvilinear.py +++ b/parcels/examples/example_nemo_curvilinear.py @@ -1,3 +1,4 @@ +import math from argparse import ArgumentParser from datetime import timedelta as delta from glob import glob @@ -5,6 +6,7 @@ import numpy as np import pytest +import dask from parcels import AdvectionRK4 from parcels import FieldSet @@ -12,6 +14,7 @@ from parcels import ParticleFile from parcels import ParticleSet from parcels import ScipyParticle +from parcels import ErrorCode ptype = {'scipy': ScipyParticle, 'jit': JITParticle} @@ -28,7 +31,7 @@ def run_nemo_curvilinear(mode, outfile): 'data': data_path + 'V_purely_zonal-ORCA025_grid_V.nc4'}} variables = {'U': 'U', 'V': 'V'} dimensions = {'lon': 'glamf', 'lat': 'gphif'} - field_chunksize = {'lon': 2, 'lat': 2} + field_chunksize = {'y': 2, 'x': 2} field_set = FieldSet.from_nemo(filenames, variables, dimensions, field_chunksize=field_chunksize) assert field_set.U.field_chunksize == field_chunksize @@ -101,6 +104,102 @@ def test_nemo_3D_samegrid(): assert fieldset.U.dataFiles is not fieldset.W.dataFiles +def fieldset_nemo_setup(): + data_path = path.join(path.dirname(__file__), 'NemoNorthSeaORCA025-N006_data/') + ufiles = sorted(glob(data_path + 'ORCA*U.nc')) + vfiles = sorted(glob(data_path + 'ORCA*V.nc')) + wfiles = sorted(glob(data_path + 'ORCA*W.nc')) + mesh_mask = data_path + 'coordinates.nc' + + filenames = {'U': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': ufiles}, + 'V': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': vfiles}, + 'W': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': wfiles}} + variables = {'U': 'uo', + 'V': 'vo', + 'W': 'wo'} + dimensions = {'U': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}, + 'V': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}, + 'W': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}} + + return filenames, variables, dimensions + + +def compute_particle_advection(field_set, mode, lonp, latp): + + def periodicBC(particle, fieldSet, time): + if particle.lon > 15.0: + particle.lon -= 15.0 + if particle.lon < 0: + particle.lon += 15.0 + if particle.lat > 60.0: + particle.lat -= 11.0 + if particle.lat < 49.0: + particle.lat += 11.0 + + def OutOfBounds_reinitialisation(particle, fieldset, time): + particle.lat = 2.5 + particle.lon = 52.0 + (-1e-3 + np.random.rand() * 2.0 * 1e-3) + + pset = ParticleSet.from_list(field_set, ptype[mode], lon=lonp, lat=latp) + pfile = ParticleFile("nemo_particles", pset, outputdt=delta(days=1)) + kernels = pset.Kernel(AdvectionRK4) + periodicBC + pset.execute(kernels, runtime=delta(days=4), dt=delta(hours=6), + output_file=pfile, recovery={ErrorCode.ErrorOutOfBounds: OutOfBounds_reinitialisation}) + return pset + + +@pytest.mark.parametrize('mode', ['jit']) # Only testing jit as scipy is very slow +def test_nemo_curvilinear_auto_chunking(mode): + dask.config.set({'array.chunk-size': '2MiB'}) + filenames, variables, dimensions = fieldset_nemo_setup() + field_set = FieldSet.from_nemo(filenames, variables, dimensions, field_chunksize='auto') + assert field_set.U.dataFiles is not field_set.W.dataFiles + npart = 20 + lonp = 2.5 * np.ones(npart) + latp = [i for i in 52.0+(-1e-3+np.random.rand(npart)*2.0*1e-3)] + compute_particle_advection(field_set, mode, lonp, latp) + # Nemo sample file dimensions: depthu=75, y=201, x=151 + assert (len(field_set.U.grid.load_chunk) == len(field_set.V.grid.load_chunk)) + assert (len(field_set.U.grid.load_chunk) == len(field_set.W.grid.load_chunk)) + assert (len(field_set.U.grid.load_chunk) != 1) + + +@pytest.mark.parametrize('mode', ['jit']) # Only testing jit as scipy is very slow +def test_nemo_curvilinear_no_chunking(mode): + dask.config.set({'array.chunk-size': '128MiB'}) + filenames, variables, dimensions = fieldset_nemo_setup() + field_set = FieldSet.from_nemo(filenames, variables, dimensions, field_chunksize=False) + assert field_set.U.dataFiles is not field_set.W.dataFiles + npart = 20 + lonp = 2.5 * np.ones(npart) + latp = [i for i in 52.0+(-1e-3+np.random.rand(npart)*2.0*1e-3)] + compute_particle_advection(field_set, mode, lonp, latp) + # Nemo sample file dimensions: depthu=75, y=201, x=151 + assert (len(field_set.U.grid.load_chunk) == len(field_set.V.grid.load_chunk)) + assert (len(field_set.U.grid.load_chunk) == len(field_set.W.grid.load_chunk)) + assert (len(field_set.U.grid.load_chunk) == 1) + + +@pytest.mark.parametrize('mode', ['jit']) # Only testing jit as scipy is very slow +def test_nemo_curvilinear_specific_chunking(mode): + dask.config.set({'array.chunk-size': '128MiB'}) + filenames, variables, dimensions = fieldset_nemo_setup() + chs = {'U': {'depthu': 75, 'y': 16, 'x': 16}, + 'V': {'depthv': 75, 'y': 16, 'x': 16}, + 'W': {'depthw': 75, 'y': 16, 'x': 16}} + + field_set = FieldSet.from_nemo(filenames, variables, dimensions, field_chunksize=chs) + assert field_set.U.dataFiles is not field_set.W.dataFiles + npart = 20 + lonp = 2.5 * np.ones(npart) + latp = [i for i in 52.0+(-1e-3+np.random.rand(npart)*2.0*1e-3)] + compute_particle_advection(field_set, mode, lonp, latp) + # Nemo sample file dimensions: depthu=75, y=201, x=151 + assert (len(field_set.U.grid.load_chunk) == len(field_set.V.grid.load_chunk)) + assert (len(field_set.U.grid.load_chunk) == len(field_set.W.grid.load_chunk)) + assert (len(field_set.U.grid.load_chunk) == (1 * int(math.ceil(201.0/16.0)) * int(math.ceil(151.0/16.0)))) + + if __name__ == "__main__": p = ArgumentParser(description="""Chose the mode using mode option""") p.add_argument('--mode', choices=('scipy', 'jit'), nargs='?', default='jit', diff --git a/parcels/examples/example_recursive_errorhandling.py b/parcels/examples/example_recursive_errorhandling.py index 6f935f055c..ad6c0462a6 100644 --- a/parcels/examples/example_recursive_errorhandling.py +++ b/parcels/examples/example_recursive_errorhandling.py @@ -13,7 +13,7 @@ @pytest.mark.parametrize('mode', ['scipy', 'jit']) def test_recursive_errorhandling(mode, xdim=2, ydim=2): - """Example script to show how recursaive error handling can work. + """Example script to show how recursive error handling can work. In this example, a set of Particles is started at Longitude 0.5. These are run through a Kernel that throws an error if the diff --git a/parcels/examples/tutorial_timevaryingdepthdimensions.ipynb b/parcels/examples/tutorial_timevaryingdepthdimensions.ipynb new file mode 100644 index 0000000000..17f83e8324 --- /dev/null +++ b/parcels/examples/tutorial_timevaryingdepthdimensions.ipynb @@ -0,0 +1,167 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tutorial on how to use S-grids with time-evolving depth dimensions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Some hydrodynamic models (such as SWASH) have time-evolving depth dimensions, for example because they follow the waves on the free surface. Parcels can work with these types of models, but it is a bit involved to set up. That is why we explain here how to run Parcels on `FieldSets` with time-evoloving depth dimensions" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "from parcels import FieldSet, ParticleSet, JITParticle, AdvectionRK4, ParticleFile, plotTrajectoriesFile\n", + "import numpy as np\n", + "from datetime import timedelta as delta\n", + "from os import path" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here, we use sample data from the SWASH model. We first set the `filenames` and `variables`" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "filenames = path.join('SWASH_data', 'field_*.nc')\n", + "variables = {'U': 'cross-shore velocity',\n", + " 'V': 'along-shore velocity',\n", + " 'depth_u': 'time varying depth_u'}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, the first key step when reading time-evolving depth dimensions is that we specify `depth` as **`'not_yet_set'`** in the `dimensions` dictionary" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "dimensions = {'U': {'lon': 'x', 'lat': 'y', 'depth': 'not_yet_set', 'time': 't'},\n", + " 'V': {'lon': 'x', 'lat': 'y', 'depth': 'not_yet_set', 'time': 't'},\n", + " 'depth_u': {'lon': 'x', 'lat': 'y', 'depth': 'not_yet_set', 'time': 't'}}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then, _after_ we create the `FieldSet` object, we set the `depth` dimension of the relevant `Fields` to `fieldset.depth_u` and `fieldset.depth_w`, using the `Field.set_depth_from_field()` method" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Casting lon data to np.float32\n", + "WARNING: Casting lat data to np.float32\n", + "WARNING: Flipping lat data from North-South to South-North\n", + "WARNING: Casting depth data to np.float32\n" + ] + } + ], + "source": [ + "fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, mesh='flat', allow_time_extrapolation=True)\n", + "fieldset.U.set_depth_from_field(fieldset.depth_u)\n", + "fieldset.V.set_depth_from_field(fieldset.depth_u)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, we can create a ParticleSet, run those and plot them" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO: Compiled JITParticleAdvectionRK4 ==> /var/folders/r2/8593q8z93kd7t4j9kbb_f7p00000gr/T/parcels-504/6c662dabb803077cfcabb1b87d3c87bb_0.so\n", + "WARNING: Casting field data to np.float32\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEICAYAAABxiqLiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VNX5x/HPNyv7DhHZhagsyhYB9yha0VpxAQVtrVaktGhrVap2UX+2tPVXLW644FKpVRDxh6KlogJxqewICAQwgghVQGTRsAee3x/3RsdxQoaQMJnkeb9eeWVy7jnnPieTzDP33Dv3yMxwzjnnDlZKogNwzjmXnDyBOOecKxNPIM4558rEE4hzzrky8QTinHOuTDyBOOecKxNPIK7akfQbSU/EUe9pSX88HDGVsP9/S/pxovYfEUdcvy9X/cg/B+IqG0kfA1nAPmA7MAW43swKy9BXLvBPM2tZhrZPA+vM7HdlaGtAtpkVHGzb8iTpTqCDmf0wkXG4qsmPQFxl9QMzqwP0AE4AyvIinlbuUZWTyhxbpGSJ0yWGJxBXqZnZf4F/A10AJF0tKV/SV5JWSfppcV1JuZLWSbpF0npgXNj2SEmF4deRku6U9M+IdqdIek/SVklrJV0VKxZJ50taGNZ7T9LxJdR7O3y4KNznZTFi+7ukhpJelfS5pC3h45YR/eRJGhLx80/CsW+RNFVSm4htnSW9IWmzpA3htFM/4DfAZWEci8K6R0qaHNYtkHRtRD93Spoo6Z+SvgSuivH76hPx+1oUHuUVb7sqfF6+krRa0hUlP7su2XkCcZWapFbAecD7YdFG4HygHnA1MEpSj4gmRwCNgDbAlcC5wKdmVif8+jSq/9YESeZBoCnQDVgYI44ewFPAT4HGwGPAZEmZ0XXN7LTwYddwn8/HiG0owf/f38OfWwM7gYdK+D1cSJAMLg7jfIcgQSKpLvAm8BpwJNABmGZmrwF/Ap4P4+gadjcOWBfWHQD8SVLfiN31ByYCDYBno+JoAfwL+GM4lpuBFyU1lVQbeAA418zqAifF+l26qqNaJxBJAyUtlbRfUk4JdVpJmhG+81sq6Zcx6twsySQ1CX9uKGmSpMWS5kjqElH3l5KWhH3dEEeMp0laIKlI0oBDGW+SeUnSVuBd4C2CF0LM7F9m9pEF3gJeB06NaLcfuMPMdpvZzjj2cwXwppmNM7O9ZvaFmcV60bsWeMzMZpvZPjMbC+wG+hzEmL4VW7ivF81sh5l9BYwETi+h7U+BP5tZvpkVEfw+uoVHIecD683sXjPbZWZfmdnsWJ2ECfkU4Jaw7kLgCeBHEdVmmtlLZrY/xu/wh8AUM5sSbn8DmEeQ5IvH2EVSTTP7zMyWHsTvxyWZapNAwimEp6OKlxC8o3v7uy2+VgTcZGYdCV4shkvqFNFvK+Bs4JOINr8BFprZ8QTvgu8P63YheCHqBXQFzpeUXUronwBXAc+VUq+qudDMGphZGzP7efELmaRzJc0Kp1+2ErxwNYlo97mZ7TqI/bQCPoqjXhvgpnDaZmu471YE7+Lj9a3YJNWS9JikNeF00dtAA0mpJez//oh9bwYEtDiIMRDGuzlMWMXWhP0UW3uA9m2AgVG/h1OA5ma2HbgMGAZ8Julfko6NMy6XhKpNAoklfDe3opQ6n5nZgvDxV0A+3/5nGwX8Goi8nK0TMC1ssxxoKykL6AjMCt9xFhG8s74IQFJ7Sa9Jmi/pneJ/PDP72MwWE7yzq9bC6aIXgXuALDNrQHCFliKqRV9WWNplhmuB9nHsfi0wMkxqxV+1zGxcnOHHiuUm4Bigt5nVA4qnvsR3rQV+GrX/mmb2XiljiN7np0CjcNqrWGvgvwdoEx3HM1Fx1DazvwCY2VQzOxtoDiwHHj9AXy7JVesEcrAktQW6A7PDny8A/mtmi6KqLiI4skFSL4J3bS0JjnhOk9RYUi2Cd8+twjZjCC5V7Ukwr/xwhQ4mOWUAmcDnQJGkc4HvldJmA9BYUv0Stj8LnCXpUklp4XPTLUa9x4FhknorUFvS96NeiKP3e1QpsdUlOO+xVVIj4I4D1H0UuE1SZwBJ9SUNDLe9Chwh6QZJmZLqSuodEUdbSSkAZrYWeA/4s6QaCi4EuIaocx0H8E/gB5LOkZQa9pErqaWkLEkXhOdCdgOFBJdiuyqqyicQSbMlFc/zXqDgKpqFks45yH7qELz7vcHMvgwTwG+B22NU/wvQMNzv9QQngIvMLB+4G3iD4ITnIoIXwjoEJxxfCNs8RvAOzkUIjwB/AUwAtgCXA5NLabOc4KTxqnDK5cio7Z8QJPKbCKaFFhJML0b3M49g+vGhcN8FBFOLJbkTGBvu89IS6twH1AQ2AbMI/iZKGsckgr+d8eF01xKCCwSKfy9nAz8A1gMfAmeETV8Iv38haUH4eDDQluBoZBLBeZk3DjCWyDjWEpxk/w1BIl8LjCB4LUkh+D1+SvC7PB34eTz9uuRUbT5IqOBSw6vM7KoY2/KAm8MXiVht0wne5U01s7+FZccRTFPtCKu1JPjH6WVm6yPaClgNHG9mX0b1+yeCq2H+CawwsxKTRnj+5lUzmxjHcF0VoOBy4CfM7B+JjsW5WKr8EcihChPAk0B+cfIAMLMPzKyZmbU1s7YEiaCHma2X1EBSRlh1CPB2cfKQ1Cz83ppgmmtcuG118ZREOEXynXfBrvoIj3CPInjz4VylVK0TiKSLJK0DTgT+JWlqWH6kpClhtZMJLnE8M2L667wSuizWEVgqaTnBNEPkpb8vSloGvAIMN7MtYfkVwDUKPuy1lGCaAEknhDEOBB6T5JdFVnHhm4z1BBdZvJvgcJwrUbWZwnLOOVe+qvURiHPOubKr0jdKa9KkibVt27ZMbbdv307t2rXLN6BKxMeXvKry2MDHVxnMnz9/k5k1La1elU4gbdu2Zd68mBdWlSovL4/c3NzyDagS8fElr6o8NvDxVQaS1sRTz6ewnHPOlYknEOecc2XiCcQ551yZeAJxzjlXJp5AnHPOlUlcCURSP0krFCx/eWuM7ZmSng+3zw7vWlu87bawfEXkDQwlPSVpo6QlUX01UrA054fh94ZhuSQ9EPa1WN9ehc4559xhVmoCUbC4zWiCW3J0AgYrYkGl0DXAFjPrQLA+xt1h207AIKAz0A94WN8slvN0WBbtVoLlOLMJblZYnLDOBbLDr6HAI/EN8eDNX7OFVz/aw/w1W0qvXMnNX7OF0TMKqsRYnHOVSzyfA+kFFJjZKgBJ4wnu07Qsok5/gttXQ7CW8kPhTQj7A+PNbDfBzQILwv5mmtnbkUcqUX3lho/HAnnALWH5Pyy498qs8IaFzc3ss/iGGp/5a7Zw+eOz2F20n5c+mslF3Y+kef2a5bmLw+azbTuZ9P6n7NtvpKel8OyQ3pzQtlGiw3LOVRHxJJAWfHuJy3VA75LqmFmRpG1A47B8VlTbFhxYVnFSMLPPiu9eW0IcLYBvJRBJQwmOUMjKyiIvL6+U3X3bqx/tYU9RsPhf0X7jhfn/jbk8XDKIvMvZnqL9XPH4TE7ISqNbs1Ta1dh10L+bZFJYWFhlx1eVxwY+vmQSTwKJ9foZfQfGkurE0zZecfVlZmMIVvcjJyfHDvYTn3XbbeHVj2exZ+9+MtJTeHZIH3q2aViWeBNu/potXPHELPYW7SclRfQ5qjGL1m3jP5/uJk3ixA41OatjFn07NqNlw1qJDrdcJcOnfcuqKo8NfHzJJJ4Eso5vll2FbxZOilVnnaQ0oD7BimTxtI22oXhqSlJzYONBxHHIerZpyLND+jDuzbkMPuuEpE0e8M1YZq36gj5HNaZnm4YU7dvP/DVb+Pvr81m5dSd3TF7KHZOXcuwRdenbsRl9O2bRrWUDUlKS9bjLOXe4xJNA5gLZktoB/yU4KX55VJ3JwI+BmcAAYLqZmaTJwHOS/gYcSXACfE4p+yvu6y/h95cjyq8Lz8H0BraV9/mPYj3bNOSr9hlJnTyK9WzT8FvjSEtNofdRjdl5bAa5ubms3rSdafkbeDN/A4++tYrRMz6iSZ0MzjgmSCanZjehdmaVvmWac66MSn1lCM9pXAdMBVKBp8xsqaS7gHlmNplgxb5nwpPkmwmSDGG9CQQn3IsIFlDaByBpHMHJ8ibhgkl3mNmTBIljgqRrgE8IFlICmEKwdnUBwTKyV5fHL6C6a9ekNkNOPYohpx7Fth17yVu5kWn5G5m6dD0vzF9HRmoKJ7ZvzFkdm3FmxyxaNEjOCwqcc+UvrreWZjaF4AU8suz2iMe7+OaFPrrtSGBkjPLBJdT/Augbo9yA4fHE68qmfq10+ndrQf9uLdi7bz/zPt7CtPwNTFu+kd+/vJTfvxxMdRWfN+nqU13OVWs+N+FiSg+PPE5s35jfnd+Jjz4vDKe6NvLIWx/x0IwCmtTJ5Mxjm3491VUrw/+cnKtO/D/exaV90zq0b1qHoae1Z+uOPeSt+Jw38zfw7yXrmTBvHRlpKZzUvjF9O2ZxVsdmSfvZGedc/DyBuIPWoFYGF3ZvwYXdg6muuas382b+RqYt38DvX1rC71+CTs3rcVZ4VddxLer7VJdzVZAnEHdI0lNTOKlDE07q0ITfn9+Rjz4vDJJJ/gYemlHAA9MLaFo3k77HBsnklA5NqJmRWnrHzrlKzxOIKzeS6NCsLh2a1WXY6e3Zsn0PM1YEV3W9uvgzxs9dS2ZaCid3aBJ85uTYLI6oXyPRYTvnysgTiKswDWtncHGPllzcoyV7ivYzZ/Vmpi3fwLT8jUxfvpHfsoQuLerR99gszuqYRZcW9QhuoeacSwaeQNxhkZGWwinZTTgluwm3n9+Jgo3fTHU9OP1D7p/2IVn1Mjnz2OAk/MkdmlAj3ae6nKvMPIG4w04S2Vl1yc6qy89y27N5+x5mLA9Owr+y6FPGzfmEGukpnNKhCX07ZtH32GY0q+dTXc5VNp5AXMI1qp3BJT1bcknPluwu2hdMdeVv5M3wcycAx7esz5nHNuOsjll0PtKnupyrDDyBuEolMy2VU7Obcmp2U+74QSdWbijkzfwNTMvfwP3TPuS+Nz/kiHo1OLNjM87q2IyT2vtUl3OJ4gnEVVqSOOaIuhxzRF2Gn9GBTYW7g6mu/I289P5/eW72J9RMT+XkDk3Ce3U1o1ldn+py7nDxBOKSRpM6mQzMacXAnFbsLtrHrFWbg3t1hdNdAF1b1qdvxyzqb9/H/I83M2v15q9vZe+cK1+eQFxSykxL5fSjm3L60U35nwuM5eu/+vpeXaPeXIkZ8N5MBGSmpfDstcm7MJhzlZUnEJf0JNGxeT06Nq/HdWdm8/lXuxn2xAzmb9iHAbuK9vPUu6vp0bqBn3x3rhylJDoA58pb07qZnNs2nRrpKaQIJPjXB59x2WOzyP/sy0SH51yV4Ucgrkrq0DD16+V8e7drRMHGQu5+bTnff+AdrjyxLb86+2jq10xPdJjOJbW4jkAk9ZO0QlKBpFtjbM+U9Hy4fbakthHbbgvLV0g6p7Q+JZ0paYGkJZLGhmusI6m+pFckLZK0VJKvSOgOqGebhgw/owM5bRsxqFdrZtycyxW92/CPmR9z5j15TJi3lv37LdFhOpe0Sk0gklKB0cC5QCdgsKROUdWuAbaYWQdgFHB32LYTwfK2nYF+wMOSUkvqU1IKMBYYZGZdgDUE66JDsBrhMjPrSrAU7r2SMso8clftNKiVwR8u7MLk606hTeNa/HriYi559D0+WLct0aE5l5TiOQLpBRSY2Soz2wOMB/pH1elP8MIPMBHoq+BsZX9gvJntNrPVBOuZ9zpAn42B3Wa2MuzrDeCS8LEBdcN+6xCsvV500CN21V6XFvWZOOwk7h3YlbWbd3LB6Hf5zaQP2LJ9T6JDcy6pxHMOpAWwNuLndUDvkuqYWZGkbQTJoAUwK6pti/BxrD43AemScsxsHjAAaBXWeQiYDHwK1AUuM7P90cFKGgoMBcjKyiIvLy+OIX5XYWFhmdsmAx9f8Ad6V+9UXipIY/ycT3h5wScMyM7g9FZppFTiq7X8uUtuVWl88SSQWP9J0RPHJdUpqTzWkY+ZmUkaBIySlAm8zjdHGecAC4EzgfbAG5LeMbMvozoZA4wByMnJsdzc3JiDKk1eXh5lbZsMfHzfOO9sWL7+S+54eSljl21mwbaa/E//zvRoXTk/N+LPXXKrSuOLZwprHd8cBQC0JDgKiFknPOldn2CKqaS2JfZpZjPN7FQz6wW8DXwY1rka+D8LFACrgWPjiN+5Uh17RD3GD+3DA4O7s/GrXVz88HuMeGERmwp3Jzo05yqteBLIXCBbUrvwpPUggqmkSJP55mT3AGC6mVlYPii8SqsdkA3MOVCfkpqF3zOBW4BHw34/AfqG27KAY4BVBz9k52KTxAVdj2TaTbn89PSjmPT+fznjnjye/s9qivZ9Z7bUuWqv1ARiZkXAdcBUIB+YYGZLJd0l6YKw2pNAY0kFwI3ArWHbpcAEYBnwGjDczPaV1GfY1whJ+cBi4BUzmx6W/wE4SdIHwDTgFjPbdIjjd+476mSmcdu5HXnthtPo2rIBd76yjPMffJc5qzcnOjTnKpW4PkhoZlOAKVFlt0c83gUMLKHtSGBkPH2G5SOAETHKPwW+F0+8zpWHDs3q8Mw1vZi6dD1/eDWfSx+byYXdjuQ353X0Ba6cw29l4twBSaJfl+a8eePpXH9mB6Z8sJ4z7snj8bdXsdentVw15wnEuTjUzEjlpu8dw+u/Oo1e7Roxcko+597/Dv8p8FlUV315AnHuILRtUpu/X92LJ67MYXfRPq54YjbDn13Ap1t3Jjo05w47TyDOlcFZnbJ441enc+PZR/Nm/gb63vsWo2cUsLtoX6JDc+6w8QTiXBnVSE/lF32zefPG0znt6Cb8deoK+t33DnkrNiY6NOcOC08gzh2iVo1q8diPchj7k14IuOrvc7n2H/NYu3lHokNzrkJ5AnGunJx+dFNeu+E0bul3LP8p2MRZf3uL+95cya69Pq3lqiZPIM6Vo4y0FH6W255pN53O2Z2yuO/NDznrb2/x+tL1BDdncK7q8ATiXAVoXr8mD13eg+eu7U3N9FSGPjOfq5+ey+pN2xMdmnPlxhOIcxXopPZNmPLLU/nd9zsy7+MtnDPqbf46dTk79vhSNi75eQJxroKlp6Yw5NSjmH7z6Zx/fHNGz/iIvve+xb8Wf+bTWi6peQJx7jBpVrcGf7usGy8MO5EGtTIY/twCfvjkbAo2fpXo0JwrE08gzh1mJ7RtxCvXncxd/Tvzwbpt9LvvHf40JZ/C3T6t5ZKLJxDnEiAtNYUrT2zLjJtzuaRHS8a8vYoz78njpff/69NaLml4AnEugRrXyeTuAcfz0vCTOaJ+DW54fiGXPTaL/M++LL2xcwkWVwKR1E/SCkkFkm6NsT1T0vPh9tmS2kZsuy0sXyHpnNL6lHSmpAWSlkgaGy6RW7wtV9JCSUslvVXWQTtX2XRr1YBJPz+ZP198HB9u/IrzH3yXOycvZdvOvYkOzbkSlZpAJKUCo4FzgU7AYEmdoqpdA2wxsw7AKODusG0nguVqOwP9gIclpZbUp6QUYCwwyMy6AGsIl8qV1AB4GLjAzDpTwgJWziWr1BQxuFdrZtycy+BerRg782POvCePCfPWsn+/T2u5yieeI5BeQIGZrTKzPcB4oH9Unf4EL/wAE4G+khSWjzez3Wa2GigI+yupz8bAbjNbGfb1BnBJ+Phy4P/M7BMAM/M71rkqqUGtDP544XG8ct0ptGlci19PXMwlj77HB+u2JTo0574lngTSAlgb8fO6sCxmnXC9820EyaCktiWVbwLSJeWE5QOAVuHjo4GGkvIkzZd0ZRyxO5e0urSoz8RhJ3HPwK6s3byDC0a/y28nfUDhHj8acZVDPGuiK0ZZ9F9wSXVKKo+VuMzMTNIgYJSkTOB1oPjaxjSgJ9AXqAnMlDQr4mglCEQaCgwFyMrKIi8vL+agSlNYWFjmtsnAx5c8mgB39U5jUsF+xs35hEmpxtz1b3B6qzRSFOtfLLlVpeculqo0vngSyDq+OQoAaAl8WkKddeFJ7/rA5lLaxiw3s5nAqQCSvkdw5FG8j01mth3YLultoCvwrQRiZmOAMQA5OTmWm5sbxxC/Ky8vj7K2TQY+vuRz3tmwfP2X/HLsfxi7bA8LttXkf/p3pkfrhokOrVxVxecuUlUaXzxTWHOBbEntJGUQnBSfHFVnMuHJboJpp+kWXMw+GRgUXqXVDsgG5hyoT0nNwu+ZwC3Ao2G/LwOnSkqTVAvoDeSXZdDOJatjj6jHrb1qcP+gbmz8ahcXP/wev564iE2FuxMdmquGSj0CMbMiSdcBU4FU4CkzWyrpLmCemU0GngSekVRAcOQxKGy7VNIEYBnBVNRwM9sHEKvPcJcjJJ1PkNweMbPpYV/5kl4DFgP7gSfMbEn5/BqcSx6S6N+tBX07ZvHgtA958t3VvLZkPTd97xiu6N2atFT/eJc7POKZwsLMpgBTospuj3i8ixIuqzWzkcDIePoMy0cAI0ro66/AX+OJ2bmqrk5mGred15GBOa24c/JS7pi8lHFzPuGu/l3o1a5RosNz1YC/VXEuyXVoVodnrunFI1f04Mude7n0sZncMP59Nn65K9GhuSrOE4hzVYAkzj2uOW/edDrXndGBKR+s54x78nj87VXs3bc/0eG5KiquKSznXHKolZHGzeccw4CeLfmfV5Yycko+z89byw97t2b7nn30OaoxPdtUrau2XOJ4AnGuCmrbpDZPXXUC0/I38ptJi7nzlWUA1EhL4dlr+3gSceXCp7Ccq6IkcVanLK7o3ebrT/TuKtrPWyv8LkCufHgCca6KOyW7KZnpKaSEWeTlhZ/6CXZXLjyBOFfF9WzTkGeH9OGm7x3DHed34vPC3Qx4dCaffLEj0aG5JOcJxLlqoGebhgw/owNXn9KOZ4f0ZtvOvQx49D2Wr/eFq1zZeQJxrprp3rohLww7EQkufXQm89dsSXRILkl5AnGuGjo6qy4Th51Eo9oZ/PCJ2by18vNEh+SSkCcQ56qpVo1q8cKwk2jbpDZDxs7l1cXRN9l27sA8gThXjTWtm8n4oX3o1qoB1497n2dnr0l0SC6JeAJxrpqrXzOdf/ykN2cc04zfTlrC6BkFBKsxOHdgnkCcc9TMSOWxH/Wkf7cj+evUFfxpSr4nEVcqv5WJcw6A9NQURl3ajQY103n8ndVs3bGXP198nK8v4krkCcQ597WUFHHnBZ1pUCuD+6d9yLade3lgcHdqpKcmOjRXCcX11kJSP0krJBVIujXG9kxJz4fbZ0tqG7HttrB8haRzSutT0pmSFkhaImlsuMZ65L5OkLRP0oCyDNg5d2CS+NXZR3PHDzrx+rINXP33uRTuLkp0WK4SKjWBSEoFRgPnAp2AwZI6RVW7BthiZh2AUcDdYdtOBMvbdgb6AQ9LSi2pT0kpwFhgkJl1AdbwzVrrxbHcTbAUrnOuAl19cjtGXdaVOR9v5vLHZ/GFr7vuosRzBNILKDCzVWa2BxgP9I+q05/ghR9gItBXksLy8Wa228xWAwVhfyX12RjYbWYrw77eAC6J2M/1wIuA307UucPgou4tGfOjnqxY/xWXPjaTT7fuTHRIrhKJ5xxIC2BtxM/rgN4l1TGzIknbCJJBC2BWVNsW4eNYfW4C0iXlmNk8YADQCkBSC+Ai4EzghJKClTQUGAqQlZVFXl5eHEP8rsLCwjK3TQY+vuR1uMeWCtzYI4P7Fmzn/PtmMCKnBs3rVNyJ9ar83EHVGl88CUQxyqKv7yupTknlsf76zMxM0iBglKRM4HWgePL1PuAWM9sXHNzEZmZjgDEAOTk5lpubW2LdA8nLy6OsbZOBjy95JWJsucDJvbfx46fm8Nf39/GPn/SkS4v6FbKvqvzcQdUaXzxvI9YRHgWEWgLR9zz4uk540rs+sPkAbUvs08xmmtmpZtYLeBv4MKyTA4yX9DHBkcnDki6MI37nXDno0qI+Lww7kZrpqQwaM4uZH32R6JBcgsWTQOYC2ZLaScogOCk+OarOZL452T0AmG7Bp5AmA4PCq7TaAdnAnAP1KalZ+D0TuAV4FMDM2plZWzNrS3Ce5edm9lIZx+2cK4OjmtZh4s9O5Ij6Nfjx3+fwxrINiQ7JJVCpCcTMioDrCK58ygcmmNlSSXdJuiCs9iTQWFIBcCNwa9h2KTABWAa8Bgw3s30l9Rn2NUJSPrAYeMXMppfTWJ1z5aB5/ZpM+OmJdDyiLsP+OZ8X569LdEguQeL6IKGZTQGmRJXdHvF4FzCwhLYjgZHx9BmWjwBGlBLPVfHE7ZyrGI1qZ/DstX346TPzuOmFRWzduZdrTmmX6LDcYeb3KHDOlUmdzDSeuuoE+nU+gj+8uox7X1/h98+qZjyBOOfKLDMtlYcu785lOa14cHoBt7+8lP37PYlUF34vLOfcIUlLTeEvlxxHg1rpPPb2Krbu3Mu9A7uSkebvT6s6TyDOuUMmidvO60iDWhnc/dpyvtq1l0eu6EnNDL8JY1XmbxGcc+XmZ7nt+fPFx/H2ys/54ZOz2bZjb6JDchXIE4hzrlwN7tWahy7vweJ1W7lszEw2frkr0SG5CuIJxDlX7s47rjlPXXUCn2zewYBHZ/LJFzsSHZKrAJ5AnHMV4tTspjw7pDfbdu5lwKPvsXz9l4kOyZUzTyDOuQrTvXVDXhh2IhJc+uhM5q/ZkuiQXDnyBOKcq1BHZ9Vl4rCTaFQ7gx8+MZu3Vn6e6JBcOfEE4pyrcK0a1eKFYSfRtklthoydy6uLo2/o7ZKRJxDn3GHRtG4m44f2oVurBlw/7n2enb0m0SG5Q+QJxDl32NSvmc4/ftKb3KOb8ttJSxg9o8Dvn5XEPIE45w6rmhmpjLkyh/7djuSvU1fwpyn5nkSSlN/KxDl32KWnpjDq0m40qJnO4++sZuuOvfz54uNIS/X3tMnEE4hzLiFSUsSdF3SmQa0M7p/2Idt27uWBwd0THZY7CHGle0n9JK2QVCDp1hjbMyU9H26fLaltxLbbwvIVks4prU9JZ0paIGmJpLHhGutIukLS4vDrPUldD2XgzrnEk8Svzj6aO37QideXN3lNAAAZvklEQVSXbeAnT89lZ5FPZyWLUhOIpFRgNHAu0AkYLKlTVLVrgC1m1gEYBdwdtu1EsN55Z6Af8LCk1JL6lJQCjAUGmVkXYA3frLW+GjjdzI4H/gCMKfuwnXOVydUnt2PUZV2ZvXoz/ztnF5u370l0SC4O8RyB9AIKzGyVme0BxgP9o+r0J3jhB5gI9JWksHy8me02s9VAQdhfSX02Bnab2cqwrzeASwDM7D0zK/4Y6yyg5cEP1zlXWV3UvSWP/bAn6wr3M/DR9/h0685Eh+RKEc85kBbA2oif1wG9S6pjZkWSthEkgxYEL/aRbVuEj2P1uQlIl5RjZvOAAUCrGDFdA/w7VrCShgJDAbKyssjLyytleLEVFhaWuW0y8PElr6o8tjRgeGfjsfztnH/fDEbk1KB5nap1Yr0qPX/xJBDFKIuepCypTknlsf4izMxM0iBglKRM4HWg6Fs7ks4gSCCnxArWzMYQTm/l5ORYbm5urGqlysvLo6xtk4GPL3lV5bEBkJfHC7nd+fFTc7jn/X2M/UlPurSon+ioyk1Vev7iSe3r+PZRQEsg+j4EX9cJT3rXBzYfoG2JfZrZTDM71cx6AW8DHxZXknQ88ATQ38y+iCN251wS6tKiPi8MO5Ea6akMGjOLWav8370yiieBzAWyJbWTlEFwUnxyVJ3JfHOyewAw3YJPBk0GBoVXabUDsoE5B+pTUrPweyZwC/Bo+HNr4P+AH0WcI3HOVVFHNa3DxJ+dyBH1a3DlU3N4Y9mGRIfkopSaQMysCLgOmArkAxPMbKmkuyRdEFZ7EmgsqQC4Ebg1bLsUmAAsA14DhpvZvpL6DPsaISkfWAy8YmbTw/LbCc6rPCxpoaR5hzp451zl1rx+TSb89EQ6HlGXYf+cz4vz1yU6JBchrg8SmtkUYEpU2e0Rj3cBA0toOxIYGU+fYfkIYESM8iHAkHjidc5VHY1qZ/DstX346TPzuOmFRWzduZdrTmmX6LAcfi8s51wSqJOZxlNXnUC/zkfwh1eXce/rK/z+WZWAJxDnXFLITEvlocu7c1lOKx6cXsDtLy9l/35PIonk98JyziWNtNQU/nLJcTSolc5jb69i28693DOwKxlp/l44ETyBOOeSiiRuO68jDWplcPdry/ly114euaInNTNSEx1ateNp2zmXlH6W254/X3wcb638nB89OZttO/cmOqRqxxOIcy5pDe7VmtGX92DRuq1c9thMNn61K9EhVSueQJxzSe2845rz1FUn8MnmHQx8dCZrN+9IdEjVhicQ51zSOzW7Kf8c0putO/ZyySPvsWL9V4kOqVrwBOKcqxJ6tG7IC8NORIJLH5vJ/DVbSm/kDoknEOdclXF0Vl0mDjuJhrXS+eETs3l75eeJDqlK8wTinKtSWjWqxQvDTqJtk9pcM3Yury6Ovnm4Ky+eQJxzVU7TupmMH9qHbq0acP2493lu9ieJDqlK8gTinKuS6tdM5x8/6U3u0U35zaQPGD2jwO+fVc78k+jOuSqrZkYqY67M4eYXFvHXqStYsf5LjjmiLn2OakLPNg0THV7S8wTinKvS0lNTGHVpN/YU7Wfyos9g0WfUSC/g2SF9PIkcorimsCT1k7RCUoGkW2Nsz5T0fLh9tqS2EdtuC8tXSDqntD4lnSlpgaQlksaGS+SiwANh/cWSehzKwJ1z1UdKiujSoh4Kf969d78vk1sOSk0gklKB0cC5QCdgsKROUdWuAbaYWQdgFHB32LYTwXK1nYF+BKsJppbUp6QUYCwwyMy6AGv4ZqnccwmWxM0GhgKPlHnUzrlqp89RTchMD17yjGChKndo4jkC6QUUmNkqM9sDjAf6R9XpT/DCDzAR6CtJYfl4M9ttZquBgrC/kvpsDOyOWPP8DeCSiH38wwKzgAaSmpdhzM65aqhnm4Y8O6QPvzizA03qZDB6RoHfgPEQxZNAWgBrI35eF5bFrBOud76NIBmU1Lak8k1AuqScsHwA0Oog4nDOuRL1bNOQG793DGOuzGH9tl3c+uJivzLrEMRzEl0xyqJ/4yXVKak8VuIyMzNJg4BRkjKB14Gig4gDSUMJprjIysoiLy8vRrPSFRYWlrltMvDxJa+qPDY4fOO7ODuNCUvWc8czb3Jm6/QK31+xqvT8xZNA1vHNUQBASyD6o53FddaFJ73rA5tLaRuz3MxmAqcCSPoecPRBxIGZjQHGAOTk5Fhubm4cQ/yuvLw8yto2Gfj4kldVHhscvvGddpqxcexcxq/8gsvO6kXnI+tX+D6haj1/8UxhzQWyJbWTlEFwUnxyVJ3JfHOyewAw3YLjwsnAoPAqrXYEJ8DnHKhPSc3C75nALcCjEfu4Mrwaqw+wzcw+K9OonXPVXkqKuHdgVxrWSuf6595n++6i0hu5byk1gYTnNK4DpgL5wAQzWyrpLkkXhNWeBBpLKgBuBG4N2y4FJgDLgNeA4Wa2r6Q+w75GSMoHFgOvmNn0sHwKsIrgRPzjwM8PbejOuequcZ1M7rusOx9/sZ3fv7wk0eEknbg+SGhmUwhewCPLbo94vAsYWELbkcDIePoMy0cAI2KUGzA8nnidcy5eJ7ZvzPVnZnP/tA85qX0TBvRsmeiQkobfC8s5V+39om82fY5qxO9fWkLBxsJEh5M0PIE456q91BRx/6Du1MxI5brnFrBr775Eh5QUPIE45xyQVa8G917aleXrv+IPry5LdDhJwROIc86FzjimGT897Sienf0J/1rsF3mWxhOIc85FuPmcY+jWqgG3vriYT77YkehwKjVPIM45FyE9NYUHB3cHwfXjFrCnaH+iQ6q0PIE451yUVo1q8dcBx7No3Tb+OnV5osOptDyBOOdcDP26NOdHfdrw+Durmb58Q6LDqZQ8gTjnXAl++/2OdGxej5smLOKzbTsTHU6l4wnEOedKUCM9ldGXd2d30X5+OW4hRfv8fEgkTyDOOXcARzWtwx8v7MKcjzfzwLQPEx1OpeIJxDnnSnFxj5YM6NmSB2cU8F7BpkSHU2l4AnHOuTjc1b8zRzWpzS+fX8imwt2JDqdS8ATinHNxqJWRxkOX92Dbzr386vmF7N/vS+F6AnHOuTh1bF6PO37QiXc+3MRjb69KdDgJ5wnEOecOwuW9WvP945pzz+srmL9mc6LDSai4EoikfpJWSCqQdGuM7ZmSng+3z5bUNmLbbWH5CknnlNanpL6SFkhaKOldSR3C8taSZkh6X9JiSecdysCdc64sJPHnS47jyAY1+MW4hWzdsSfRISVMqQlEUiowGjgX6AQMltQpqto1wBYz6wCMAu4O23YiWO+8M9APeFhSail9PgJcYWbdgOeA34XlvyNY+rZ72OfDZRuyc84dmno10nlocA82frWLX09cTLBgavUTzxFIL6DAzFaZ2R5gPNA/qk5/YGz4eCLQV5LC8vFmttvMVhOsZ96rlD4NqBc+rg98Wkq5c84ddl1bNeCWfsfy+rINjH3v40SHkxDxrIneAlgb8fM6oHdJdcysSNI2oHFYPiuqbYvwcUl9DgGmSNoJfAn0CcvvBF6XdD1QGzgrVrCShgJDAbKyssjLy4tjiN9VWFhY5rbJwMeXvKry2CC5xtfejK5NU/njq8tg0yra1k8ttU0yja808SQQxSiLPl4rqU5J5bGOfIr7/BVwnpnNljQC+BtBUhkMPG1m90o6EXhGUhcz+9a9BcxsDDAGICcnx3Jzc2OPqhR5eXmUtW0y8PElr6o8Nki+8XXttYfz7n+Hp1em8OovTqFO5oFfVpNtfAcSzxTWOqBVxM8t+e700dd1JKURTDFtPkDbmOWSmgJdzWx2WP48cFL4+BpgAoCZzQRqAE3iiN855ypMo9oZPDC4O59s3sFvJ31Qrc6HxJNA5gLZktpJyiA4gT05qs5k4Mfh4wHAdAt+i5OBQeFVWu2AbGDOAfrcAtSXdHTY19lAfvj4E6AvgKSOBAnk84MdsHPOlbde7Rrxq7OO5uWFn/LCvHWJDuewKXUKKzyncR0wFUgFnjKzpZLuAuaZ2WTgSYIppQKCI49BYdulkiYAy4AiYLiZ7QOI1WdYfi3woqT9BAnlJ2EoNwGPS/oVwXTXVVadUr1zrlL7+RkdmLnqC26fvITurRuQnVU30SFVuHjOgWBmU4ApUWW3RzzeBQwsoe1IYGQ8fYblk4BJMcqXASfHE69zzh1uqSnivsu6ce797zD8uQW8PPwUamaUflI9mfkn0Z1zrpw0q1eDUZd1Y+WGQu56dWmiw6lwnkCcc64cnXZ0U36W255xc9YyeVHV/riaJxDnnCtnN559ND1aN+A3//cBH2/anuhwKownEOecK2fpqSk8MLg7qSni+nHvs7toX6JDqhCeQJxzrgK0bFiL/x1wPB/8dxt3/3tFosOpEJ5AnHOugpzT+QiuOqktT/1nNW8s25DocMqdJxDnnKtAt513LF1a1GPExEV8unVnosMpV55AnHOuAmWmpfLg4B7sLdrPL8a9z74qtBSuJxDnnKtg7ZrU5k8XH8e8NVuYVLA30eGUG08gzjl3GPTv1oLLclrxr1V7effDTYkOp1x4AnHOucPkzgs607yOuOH5hWz8aleiwzlknkCcc+4wqZmRyvCuNSjcvZcbn1/E/iQ/H+IJxDnnDqMWdVO48wedebdgE4+89VGiwzkknkCcc+4wu+yEVvyg65Hc+/oK5n68OdHhlJknEOecO8wk8aeLutCqUS1+Me59tmzfk+iQysQTiHPOJUDdGuk8NLgHmwp3M2LioqRcCjeuBCKpn6QVkgok3Rpje6ak58PtsyW1jdh2W1i+QtI5pfUpqa+kBZIWSnpXUoeIbZdKWiZpqaTnyjpo55yrDI5rWZ/bzu3Im/kbeeo/Hyc6nINWagKRlAqMBs4FOgGDJXWKqnYNsMXMOgCjgLvDtp0IlrftDPQDHpaUWkqfjwBXmFk34Dngd2Ff2cBtwMlm1hm4ocyjds65SuLqk9tyVscs/vLvfBav25rocA5KPEcgvYACM1tlZnuA8UD/qDr9gbHh44lAX0kKy8eb2W4zWw0UhP0dqE8D6oWP6wPFK7JcC4w2sy0AZrbx4IbqnHOVjyTuGXg8Tetkct1z7/PlruT5pHo8a6K3ANZG/LwO6F1SHTMrkrQNaByWz4pq2yJ8XFKfQ4ApknYCXwJ9wvKjAST9B0gF7jSz16KDlTQUGAqQlZVFXl5eHEP8rsLCwjK3TQY+vuRVlccG1Xd8Vx8Lf56zgyGPTuNnXTMJ3oNXbvEkkFijiD7bU1KdkspjHfkU9/kr4Dwzmy1pBPA3gqSSBmQDuUBL4B1JXczsW8d8ZjYGGAOQk5Njubm5MXZVury8PMraNhn4+JJXVR4bVN/x5QJ7GhTw16kruOik9gzu1fpwh3bQ4pnCWge0ivi5Jd9MK32njqQ0gqmnzQdoG7NcUlOgq5nNDsufB06K2MfLZrY3nA5bQZBQnHOuSvjZ6e05NbsJd05eyor1XyU6nFLFk0DmAtmS2knKIDgpPjmqzmTgx+HjAcB0C65JmwwMCq/Sakfwgj/nAH1uAepLOjrs62wgP3z8EnAGgKQmBFNaqw52wM45V1mlpIi/XdqNujXSGf7cAnbsKUp0SAdUagIxsyLgOmAqwYv5BDNbKukuSReE1Z4EGksqAG4Ebg3bLgUmAMuA14DhZravpD7D8muBFyUtAn4EjAj3MRX4QtIyYAYwwsy+OPRfgXPOVR5N62Zy32Xd+OjzQu6cvDTR4RxQPOdAMLMpwJSostsjHu8CBpbQdiQwMp4+w/JJwKQY5UaQnG6MJ2bnnEtWp2Q3YXhuBx6aUcBJ7ZtwYfcWpTdKAP8kunPOVUI3nJXNCW0b8ttJH7B60/ZEhxOTJxDnnKuE0lJTuH9Qd9LTUrjuuQXsLtqX6JC+wxOIc85VUkc2qMk9A7qy9NMv+fOU5YkO5zs8gTjnXCV2VqcsrjmlHU+/9zGvLVmf6HC+xROIc85Vcrf0O5bjW9bn1xMXsW7LjkSH8zVPIM45V8llpKXw4ODu7De4ftz77N23P9EhAZ5AnHMuKbRpXJu/XHIc73+ylXtfX5nocABPIM45lzTOP/5IBvdqzaNvfcRbKz9PdDieQJxzLpnc8YNOHJNVlxufX8iGL3clNBZPIM45l0RqpKfy0OXd2bFnHzeMX8i+/YlbCtcTiHPOJZnsrLr8T//OzFz1BaNnFCQsDk8gzjmXhAb2bMmF3Y7kvjdXMmtVYu4r6wnEOeeSkCT+eNFxtGlcm1+Of5/N2/cc9hg8gTjnXJKqk5nGQ5d3Z8v2vdz8wiL2H+bzIZ5AnHMuiXU+sj6//X5Hpi/fyJPvrj6s+44rgUjqJ2mFpAJJt8bYninp+XD7bEltI7bdFpavkHROaX1K6itpgaSFkt6V1CFqXwMkmaScsgzYOeeqmitPbMM5nbO4+7XlLFy79bDtt9QEIikVGA2cC3QCBkvqFFXtGmCLmXUARgF3h207ESxX2xnoBzwsKbWUPh8BrjCzbsBzwO8iYqkL/AIoXjPdOeeqPUn87yVdyapXg+vHLWDbzr2HZb/xHIH0AgrMbJWZ7QHGA/2j6vQHxoaPJwJ9JSksH29mu81sNVAQ9negPg2oFz6uD3wasZ8/AP8LJPbTM845V8nUr5XOA4O78+nWXQx7Zh6jZ3zI/DVbKnSf8SSQFsDaiJ/XhWUx64Trmm8DGh+g7YH6HAJMkbSOYE30vwBI6g60MrNX44jZOeeqnZ5tGnJ5r9bMXLWZe6au5IonZlVoEolnTXTFKIs+1V9SnZLKYyWu4j5/BZxnZrMljQD+JmkowdTYVaUGG9QdCpCVlUVeXl5pTWIqLCwsc9tk4ONLXlV5bODjO1Q7vggu5zVgz979jHtzLl+1z6iQfcWTQNYBrSJ+bsm3p5Ui66yTlEYw9bS5lLbfKZfUFOhqZsXnOJ4HXgPqAl2AvGBmjCOAyZIuMLN5kYGY2RhgDEBOTo7l5ubGMcTvysvLo6xtk4GPL3lV5bGBj+9Q1W23hX99PIu9RftJT0th8Fkn0LNNwwrZVzwJZC6QLakd8F+Ck+KXR9WZDPwYmAkMAKabmUmaDDwn6W/AkUA2MIfgyCRWn1uA+pKONrOVwNlAvpltA5oU70xSHnBzdPJwzrnqrmebhjw7pA+zVn1Bn6MaV1jygDgSiJkVSboOmAqkAk+Z2VJJdwHzzGwy8CTwjKQCgiOPQWHbpZImAMuAImC4me0DiNVnWH4t8KKk/QQJ5SflOmLnnKvierZpWKGJo1g8RyCY2RRgSlTZ7RGPdwEDS2g7EhgZT59h+SRgUinx5MYTt3POuYrjn0R3zjlXJp5AnHPOlYknEOecc2XiCcQ551yZeAJxzjlXJjJL3Hq6FU3S58CaMjZvAmwqx3AqGx9f8qrKYwMfX2XQxsyallapSieQQyFpnplV2VvG+/iSV1UeG/j4kolPYTnnnCsTTyDOOefKxBNIycYkOoAK5uNLXlV5bODjSxp+DsQ551yZ+BGIc865MvEE4pxzrkyqTQKR9EtJSyQtlXRDjO25krZJWhh+3R6xrZ+kFZIKJN0aUd5O0mxJH0p6XlLFLPtVigoa29OSVke06Xa4xhPtEMf3lKSNkpZEtWkk6Y3wuXtDUsXf+7oEFTS+OyX9N6LNeYdjLNHKOjZJrSTNkJQftv1lRJukf+5KGV+leO7iYmZV/otgNcMlQC2CW9i/CWRH1ckFXo3RNhX4CDgKyAAWAZ3CbROAQeHjR4GfVaGxPQ0MSObnLtx2GtADWBJV/r/AreHjW4G7q9j47iRYdC0pnzugOdAjfFwXWBnxt5n0z10p40v4cxfvV3U5AukIzDKzHWZWBLwFXBRn215AgZmtMrM9wHigvyQBZwITw3pjgQvLOe54lPvYKijOsjqU8WFmbxMschatP8FzBol77qDixlcZlHlsZvaZmS0IH38F5AMtws1J/9yVMr6kUV0SyBLgNEmNJdUCzuPba7IXO1HSIkn/ltQ5LGsBrI2osy4sawxsDf9wIssPt4oYW7GRkhZLGiUps0KiL92hjO9AsszsMwj+mYFm5RfyQamo8QFcFz5/TyVomqdcxiapLdAdmB0WVannLsb4IPHPXVyqRQIxs3zgbuAN4DWCqZqiqGoLCO7/0hV4EHgpLFesLg9QflhV0NgAbgOOBU4AGgG3lG/k8TnE8VV6FTi+R4D2QDfgM+De8oo5XuUxNkl1gBeBG8zsywoP+iBU4PgS/tzFq1okEAAze9LMepjZaQSH/B9Gbf/SzArDx1OAdElNCN6VR76raAl8SnAztAaS0qLKD7sKGFvxIbaZ2W7g7wTTXQlxCOM7kA2SmgOE3zdWQOhxqYjxmdkGM9tnZvuBx0nQ83coY5OUTvDi+qyZ/V9Esyrx3JU0vsry3MWj2iQQSc3C762Bi4FxUduPCM9rIKkXwe/mC2AukK3giqsMYBAw2cwMmAEMCLv4MfDy4RhLtPIeW1iv+B9UBHPM37rK53A6hPEdyGSC5wwS+NxBxYyv+PkLXUSCnr+yji0sexLIN7O/RXWb9M/dgcZXWZ67uByOM/WV4Qt4B1hGcJjZNywbBgwLH18HLA23zwJOimh7HsFVEh8Bv40oPwqYAxQALwCZVWhs04EPCP54/wnUSdLnbhzBNMBegiOua8LyxsA0gneM04BGVWx8z4TP32KCF9zmyTQ24BSC6dTFwMLw67yq8tyVMr5K8dzF8+W3MnHOOVcm1WYKyznnXPnyBOKcc65MPIE455wrE08gzjnnysQTiHPOuTLxBOKcc65MPIE455wrk/8HLX6sE2PlrE4AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "pset = ParticleSet(fieldset, JITParticle, lon=9.5, lat=12.5, depth=-0.1)\n", + "pfile = pset.ParticleFile(\"SwashParticles\", outputdt=delta(seconds=0.05))\n", + "pset.execute(AdvectionRK4, dt=delta(seconds=0.005), output_file=pfile)\n", + "\n", + "pfile.export() # export the trajectory data to a netcdf file\n", + "plotTrajectoriesFile('SwashParticles.nc');" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that, even though we use 2-dimensional `AdvectionRK4`, the particle still moves down, because the grid itself moves down" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/parcels/field.py b/parcels/field.py index 1e188dc74b..3394978338 100644 --- a/parcels/field.py +++ b/parcels/field.py @@ -30,6 +30,7 @@ from parcels.tools.error import FieldSamplingError from parcels.tools.error import TimeExtrapolationError from parcels.tools.loggers import logger +from netCDF4 import Dataset as ncDataset __all__ = ['Field', 'VectorField', 'SummedField', 'NestedField'] @@ -85,6 +86,8 @@ def __init__(self, name, data, lon=None, lat=None, depth=None, time=None, grid=N self.data = data time_origin = TimeConverter(0) if time_origin is None else time_origin if grid: + # if grid.defer_load and isinstance(data, np.ndarray): + # raise ValueError('Cannot combine Grid from defer_loaded Field with np.ndarray data. please specify lon, lat, depth and time dimensions separately') self.grid = grid else: self.grid = Grid.create_grid(lon, lat, depth, time, time_origin=time_origin, mesh=mesh) @@ -166,7 +169,11 @@ def __init__(self, name, data, lon=None, lat=None, depth=None, time=None, grid=N self.netcdf_engine = kwargs.pop('netcdf_engine', 'netcdf4') self.loaded_time_indices = [] self.creation_log = kwargs.pop('creation_log', '') - self.field_chunksize = kwargs.pop('field_chunksize', 'auto') + self.field_chunksize = kwargs.pop('field_chunksize', None) + self.grid.depth_field = kwargs.pop('depth_field', None) + + if self.grid.depth_field == 'not_yet_set': + assert self.grid.z4d, 'Providing the depth dimensions from another field data is only available for 4d S grids' # data_full_zdim is the vertical dimension of the complete field data, ignoring the indices. # (data_full_zdim = grid.zdim if no indices are used, for A- and C-grids and for some B-grids). It is used for the B-grid, @@ -285,7 +292,11 @@ def from_netcdf(cls, filenames, variable, dimensions, indices=None, grid=None, if 'depth' in dimensions: with NetcdfFileBuffer(depth_filename, dimensions, indices, netcdf_engine, interp_method=interp_method, field_chunksize=False, lock_file=False) as filebuffer: filebuffer.name = filebuffer.parse_name(variable[1]) - depth = filebuffer.read_depth + if dimensions['depth'] == 'not_yet_set': + depth = filebuffer.read_depth_dimensions + kwargs['depth_field'] = 'not_yet_set' + else: + depth = filebuffer.read_depth data_full_zdim = filebuffer.data_full_zdim else: indices['depth'] = [0] @@ -330,6 +341,40 @@ def from_netcdf(cls, filenames, variable, dimensions, indices=None, grid=None, grid = Grid.create_grid(lon, lat, depth, time, time_origin=time_origin, mesh=mesh) grid.timeslices = timeslices + if 'field_chunksize' in kwargs.keys() and grid.master_chunksize is None: + grid.master_chunksize = kwargs['field_chunksize'] + kwargs['dataFiles'] = dataFiles + elif grid is not None and ('dataFiles' not in kwargs or kwargs['dataFiles'] is None): + # ==== means: the field has a shared grid, but may have different data files, so we need to collect the + # ==== correct file time series again. + if timestamps is not None: + dataFiles = [] + for findex in range(len(data_filenames)): + for f in [data_filenames[findex], ] * len(timestamps[findex]): + dataFiles.append(f) + field_timeslices = np.array([stamp for file in timestamps for stamp in file]) + field_time = field_timeslices + else: + field_timeslices = [] + dataFiles = [] + for fname in data_filenames: + with NetcdfFileBuffer(fname, dimensions, indices, netcdf_engine, field_chunksize=False, lock_file=True) as filebuffer: + ftime = filebuffer.time + field_timeslices.append(ftime) + dataFiles.append([fname] * len(ftime)) + field_timeslices = np.array(field_timeslices) + field_time = np.concatenate(field_timeslices) + dataFiles = np.concatenate(np.array(dataFiles)) + if field_time.size == 1 and field_time[0] is None: + field_time[0] = 0 + time_origin = TimeConverter(field_time[0]) + field_time = time_origin.reltime(field_time) + if not np.all((field_time[1:]-field_time[:-1]) > 0): + id_not_ordered = np.where(field_time[1:] < field_time[:-1])[0][0] + raise AssertionError('Please make sure your netCDF files are ordered in time. First pair of non-ordered files: %s, %s' + % (dataFiles[id_not_ordered], dataFiles[id_not_ordered+1])) + if 'field_chunksize' in kwargs.keys() and grid.master_chunksize is None: + grid.master_chunksize = kwargs['field_chunksize'] kwargs['dataFiles'] = dataFiles if 'time' in indices: @@ -459,6 +504,9 @@ def set_scaling_factor(self, factor): if not self.grid.defer_load: self.data *= factor + def set_depth_from_field(self, field): + self.grid.depth_field = field + def __getitem__(self, key): return self.eval(*key) @@ -493,15 +541,26 @@ def cell_areas(self): def search_indices_vertical_z(self, z): grid = self.grid z = np.float32(z) - if z < grid.depth[0]: - raise FieldOutOfBoundSurfaceError(0, 0, z, field=self) - elif z > grid.depth[-1]: - raise FieldOutOfBoundError(0, 0, z, field=self) - depth_index = grid.depth <= z - if z >= grid.depth[-1]: - zi = len(grid.depth) - 2 + if grid.depth[-1] > grid.depth[0]: + if z < grid.depth[0]: + raise FieldOutOfBoundSurfaceError(0, 0, z, field=self) + elif z > grid.depth[-1]: + raise FieldOutOfBoundError(0, 0, z, field=self) + depth_indices = grid.depth <= z + if z >= grid.depth[-1]: + zi = len(grid.depth) - 2 + else: + zi = depth_indices.argmin() - 1 if z >= grid.depth[0] else 0 else: - zi = depth_index.argmin() - 1 if z >= grid.depth[0] else 0 + if z > grid.depth[0]: + raise FieldOutOfBoundSurfaceError(0, 0, z, field=self) + elif z < grid.depth[-1]: + raise FieldOutOfBoundError(0, 0, z, field=self) + depth_indices = grid.depth >= z + if z <= grid.depth[-1]: + zi = len(grid.depth) - 2 + else: + zi = depth_indices.argmin() - 1 if z <= grid.depth[0] else 0 zeta = (z-grid.depth[zi]) / (grid.depth[zi+1]-grid.depth[zi]) return (zi, zeta) @@ -532,15 +591,27 @@ def search_indices_vertical_s(self, x, y, z, xi, yi, xsi, eta, ti, time): xsi*eta * grid.depth[:, yi+1, xi+1] + \ (1-xsi)*eta * grid.depth[:, yi+1, xi] z = np.float32(z) - depth_index = depth_vector <= z - if z >= depth_vector[-1]: - zi = len(depth_vector) - 2 + + if depth_vector[-1] > depth_vector[0]: + depth_indices = depth_vector <= z + if z >= depth_vector[-1]: + zi = len(depth_vector) - 2 + else: + zi = depth_indices.argmin() - 1 if z >= depth_vector[0] else 0 + if z < depth_vector[zi]: + raise FieldOutOfBoundSurfaceError(0, 0, z, field=self) + elif z > depth_vector[zi+1]: + raise FieldOutOfBoundError(x, y, z, field=self) else: - zi = depth_index.argmin() - 1 if z >= depth_vector[0] else 0 - if z < depth_vector[zi]: - raise FieldOutOfBoundSurfaceError(0, 0, z, field=self) - elif z > depth_vector[zi+1]: - raise FieldOutOfBoundError(x, y, z, field=self) + depth_indices = depth_vector >= z + if z <= depth_vector[-1]: + zi = len(depth_vector) - 2 + else: + zi = depth_indices.argmin() - 1 if z <= depth_vector[0] else 0 + if z > depth_vector[zi]: + raise FieldOutOfBoundSurfaceError(0, 0, z, field=self) + elif z < depth_vector[zi+1]: + raise FieldOutOfBoundError(x, y, z, field=self) zeta = (z - depth_vector[zi]) / (depth_vector[zi+1]-depth_vector[zi]) return (zi, zeta) @@ -901,8 +972,6 @@ def get_block(self, bid): return np.unravel_index(bid, self.nchunks[1:]) def chunk_setup(self): - if self.chunk_set: - return if isinstance(self.data, da.core.Array): chunks = self.data.chunks self.nchunks = self.data.numblocks @@ -921,11 +990,9 @@ def chunk_setup(self): self.data_chunks = [None] * npartitions self.c_data_chunks = [None] * npartitions - self.grid.load_chunk = np.zeros(npartitions, dtype=c_int) - # self.grid.chunk_info format: number of dimensions (without tdim); number of chunks per dimensions; - # chunksizes (the 0th dim sizes for all chunk of dim[0], then so on for next dims + # chunksizes (the 0th dim sizes for all chunk of dim[0], then so on for next dims self.grid.chunk_info = [[len(self.nchunks)-1], list(self.nchunks[1:]), sum(list(list(ci) for ci in chunks[1:]), [])] self.grid.chunk_info = sum(self.grid.chunk_info, []) self.chunk_set = True @@ -1585,10 +1652,12 @@ def __getitem__(self, key): class NetcdfFileBuffer(object): - _name_maps = {'lon': ['lon', 'nav_lon', 'x', 'longitude', 'lo', 'ln'], - 'lat': ['lat', 'nav_lat', 'y', 'latitude', 'la', 'lt'], - 'depth': ['depth', 'depthu', 'depths', 'depthw', 'depthz', 'z', 'd'], + _name_maps = {'lon': ['lon', 'nav_lon', 'x', 'longitude', 'lo', 'ln', 'i'], + 'lat': ['lat', 'nav_lat', 'y', 'latitude', 'la', 'lt', 'j'], + 'depth': ['depth', 'depthu', 'depthv', 'depthw', 'depths', 'deptht', 'depthx', 'depthy', 'depthz', + 'z', 'z_u', 'z_v', 'z_w', 'd', 'k', 'w_dep', 'w_deps'], 'time': ['time', 'time_count', 'time_counter', 'timer_count', 't']} + _min_dim_chunksize = 16 """ Class that encapsulates and manages deferred access to file data. """ def __init__(self, filename, dimensions, indices, netcdf_engine, timestamp=None, @@ -1622,7 +1691,8 @@ def __enter__(self): if self.field_chunksize not in [False, None]: init_chunk_dict = self._get_initial_chunk_dictionary() try: - # Unfortunately we need to do if-else here, cause the lock-parameter is either False or a Lock-object (we would rather want to have it auto-managed). + # Unfortunately we need to do if-else here, cause the lock-parameter is either False or a Lock-object + # (which we would rather want to have being auto-managed). # If 'lock' is not specified, the Lock-object is auto-created and managed bz xarray internally. if self.lock_file: self.dataset = xr.open_dataset(str(self.filename), decode_cf=True, engine=self.netcdf_engine, chunks=init_chunk_dict) @@ -1657,8 +1727,8 @@ def close(self): self.chunk_mapping = None def _get_initial_chunk_dictionary(self): - # ==== check-opening requested dataset to access metadata ==== # - # ==== opening the file for getting the dimensions does not require a decode - so don't even try. Save some computation ==== # + # ==== check-opening requested dataset to access metadata ==== # + # ==== file-opening and dimension-reading does not require a decode or lock ==== # self.dataset = xr.open_dataset(str(self.filename), decode_cf=False, engine=self.netcdf_engine, chunks={}, lock=False) self.dataset['decoded'] = False # ==== self.dataset temporarily available ==== # @@ -1666,38 +1736,60 @@ def _get_initial_chunk_dictionary(self): if isinstance(self.field_chunksize, dict): init_chunk_dict = self.field_chunksize elif isinstance(self.field_chunksize, tuple): # and (len(self.dimensions) == len(self.field_chunksize)): + tmp_chs = [0, ] * len(self.field_chunksize) chunk_index = len(self.field_chunksize)-1 + loni, lonname, _ = self._is_dimension_in_dataset('lon') - if loni >= 0: + if loni >= 0 and chunk_index >= 0: init_chunk_dict[lonname] = self.field_chunksize[chunk_index] - chunk_index -= 1 + tmp_chs[chunk_index] = self.field_chunksize[chunk_index] else: logger.warning_once(self._netcdf_DimNotFound_warning_message('lon')) + chunk_index -= 1 + lati, latname, _ = self._is_dimension_in_dataset('lat') - if lati >= 0: + if lati >= 0 and chunk_index >= 0: init_chunk_dict[latname] = self.field_chunksize[chunk_index] - chunk_index -= 1 + tmp_chs[chunk_index] = self.field_chunksize[chunk_index] else: logger.warning_once(self._netcdf_DimNotFound_warning_message('lat')) + chunk_index -= 1 + depthi, depthname, _ = self._is_dimension_in_dataset('depth') - if depthi >= 0: - init_chunk_dict[depthname] = self.field_chunksize[chunk_index] - chunk_index -= 1 + if depthi >= 0 and chunk_index >= 0: + if self._is_dimension_available('depth'): + init_chunk_dict[depthname] = self.field_chunksize[chunk_index] + tmp_chs[chunk_index] = self.field_chunksize[chunk_index] else: logger.warning_once(self._netcdf_DimNotFound_warning_message('depth')) + chunk_index -= 1 + timei, timename, _ = self._is_dimension_in_dataset('time') - if timei >= 0: - init_chunk_dict[timename] = self.field_chunksize[chunk_index] - chunk_index -= 1 + if timei >= 0 and chunk_index >= 0: + if self._is_dimension_available('time'): + init_chunk_dict[timename] = self.field_chunksize[chunk_index] + tmp_chs[chunk_index] = self.field_chunksize[chunk_index] else: logger.warning_once(self._netcdf_DimNotFound_warning_message('time')) + chunk_index -= 1 + + # ==== re-arrange the tupe and correct for empty dimensions ==== # + for chunk_index in range(len(self.field_chunksize)-1, -1, -1): + if tmp_chs[chunk_index] < 1: + tmp_chs.pop(chunk_index) + self.field_chunksize = tuple(tmp_chs) + elif self.field_chunksize == 'auto': av_mem = psutil.virtual_memory().available chunk_cap = av_mem * (1/8) * (1/3) if 'array.chunk-size' in da_conf.config.keys(): chunk_cap = da_utils.parse_bytes(da_conf.config.get('array.chunk-size')) else: - logger.info_once("Unable to locate chunking hints from dask, thus estimating the max. chunk size heuristically. Please consider defining the 'chunk-size' for 'array' in your local dask configuration file (see http://oceanparcels.org/faq.html#field_chunking_config and https://docs.dask.org).") + predefined_cap = da_conf.get('array.chunk-size') + if predefined_cap is not None: + chunk_cap = da_utils.parse_bytes(predefined_cap) + else: + logger.info_once("Unable to locate chunking hints from dask, thus estimating the max. chunk size heuristically. Please consider defining the 'chunk-size' for 'array' in your local dask configuration file (see http://oceanparcels.org/faq.html#field_chunking_config and https://docs.dask.org).") loni, lonname, lonvalue = self._is_dimension_in_dataset('lon') lati, latname, latvalue = self._is_dimension_in_dataset('lat') if lati >= 0 and loni >= 0: @@ -1712,6 +1804,43 @@ def _get_initial_chunk_dictionary(self): init_chunk_dict[depthname] = max(1, depthvalue) # ==== closing check-opened requested dataset ==== # self.dataset.close() + # ==== check if the chunksize reading is successfull. if not, load the file ONCE really into memory and ==== # + # ==== deduce the chunking from the array dims. ==== # + try: + self.dataset = xr.open_dataset(str(self.filename), decode_cf=True, engine=self.netcdf_engine, chunks=init_chunk_dict, lock=False) + except: + # ==== fail - open it as a normal array and deduce the dimensions from the read field ==== # + init_chunk_dict = {} + self.dataset = ncDataset(str(self.filename)) + refdims = self.dataset.dimensions.keys() + max_field = "" + max_dim_names = () + max_overlay_dims = 0 + for vname in self.dataset.variables: + var = self.dataset.variables[vname] + overlay_dims = [] + for vdname in var.dimensions: + if vdname in refdims: + overlay_dims.append(vdname) + n_overlay_dims = len(overlay_dims) + if n_overlay_dims > max_overlay_dims: + max_field = vname + max_dim_names = tuple(overlay_dims) + max_overlay_dims = n_overlay_dims + self.name = max_field + for dname in max_dim_names: + # if dname in self._name_maps['time'] and self.dataset.dimensions[dname].size == 1: + # continue + if isinstance(self.field_chunksize, dict): + if dname in self.field_chunksize.keys(): + init_chunk_dict[dname] = min(self.field_chunksize[dname], self.dataset.dimensions[dname].size) + continue + init_chunk_dict[dname] = min(self._min_dim_chunksize, self.dataset.dimensions[dname].size) + # ==== because in this case it has shown that the requested field_chunksize setup cannot be used, ==== # + # ==== replace the requested field_chunksize with this auto-derived version. ==== # + self.field_chunksize = init_chunk_dict + finally: + self.dataset.close() self.dataset = None # ==== self.dataset not available ==== # return init_chunk_dict @@ -1719,7 +1848,7 @@ def _get_initial_chunk_dictionary(self): def _is_dimension_available(self, dimension_name): if self.dimensions is None or self.dataset is None: return False - return (dimension_name in self.dimensions and self.dimensions[dimension_name] in self.dataset.dims) + return dimension_name in self.dimensions def _is_dimension_in_dataset(self, dimension_name): k, dname, dvalue = (-1, '', 0) @@ -1753,12 +1882,15 @@ def _chunksize_to_chunkmap(self): if self.field_chunksize in [False, 'auto', None]: return self.chunk_mapping = {} - if (isinstance(self.field_chunksize, tuple)): + if isinstance(self.field_chunksize, tuple): for i in range(len(self.field_chunksize)): self.chunk_mapping[i] = self.field_chunksize[i] else: # ====== 'time' is strictly excluded from the reading dimensions as it is implicitly organized with the data ====== # + timei, timename, timevalue = self._is_dimension_in_chunksize_request('time') + dtimei, dtimename, dtimevalue = self._is_dimension_in_dataset('time') depthi, depthname, depthvalue = self._is_dimension_in_chunksize_request('depth') + ddepthi, ddepthname, ddepthvalue = self._is_dimension_in_dataset('depth') lati, latname, latvalue = self._is_dimension_in_chunksize_request('lat') loni, lonname, lonvalue = self._is_dimension_in_chunksize_request('lon') dim_index = 0 @@ -1769,7 +1901,12 @@ def _chunksize_to_chunkmap(self): self.chunk_mapping[dim_index] = lonvalue dim_index += 1 elif len(self.field_chunksize) >= 3: - if depthi >= 0: + if timei >= 0 and timevalue > 1 and dtimei >= 0 and dtimevalue > 1 and self._is_dimension_available('time'): + # self.chunk_mapping[dim_index] = self.field_chunksize[self.dimensions['time']] + self.chunk_mapping[dim_index] = 1 # still need to make sure that we only load 1 time step at a time + dim_index += 1 + + if depthi >= 0 and depthvalue > 1 and ddepthi >= 0 and ddepthvalue > 1 and self._is_dimension_available('depth'): # self.chunk_mapping[dim_index] = self.field_chunksize[self.dimensions['depth']] self.chunk_mapping[dim_index] = depthvalue dim_index += 1 @@ -1783,28 +1920,28 @@ def _chunkmap_to_chunksize(self): return self.field_chunksize = {} chunk_map = self.chunk_mapping - timei, _, timevalue = self._is_dimension_in_chunksize_request('time') - depthi, _, depthvalue = self._is_dimension_in_chunksize_request('depth') + timei, _, timevalue = self._is_dimension_in_dataset('time') + depthi, _, depthvalue = self._is_dimension_in_dataset('depth') if len(chunk_map) == 2: self.field_chunksize[self.dimensions['lat']] = chunk_map[0] self.field_chunksize[self.dimensions['lon']] = chunk_map[1] elif len(chunk_map) == 3: chunk_dim_index = 0 - if self._is_dimension_available('depth') or (depthi >= 0 and depthvalue > 1): + if depthi >= 0 and depthvalue > 1 and self._is_dimension_available('depth'): self.field_chunksize[self.dimensions['depth']] = chunk_map[chunk_dim_index] chunk_dim_index += 1 - elif self._is_dimension_available('time') or (timei >= 0 and timevalue > 1): + elif timei >= 0 and timevalue > 1 and self._is_dimension_available('time'): self.field_chunksize[self.dimensions['time']] = chunk_map[chunk_dim_index] chunk_dim_index += 1 self.field_chunksize[self.dimensions['lat']] = chunk_map[chunk_dim_index] chunk_dim_index += 1 self.field_chunksize[self.dimensions['lon']] = chunk_map[chunk_dim_index] elif len(chunk_map) >= 4: - self.field_chunksize[self.dimensions['time']] = chunk_map[-1] - self.field_chunksize[self.dimensions['depth']] = chunk_map[0] - self.field_chunksize[self.dimensions['lat']] = chunk_map[1] - self.field_chunksize[self.dimensions['lon']] = chunk_map[2] - dim_index = 3 + self.field_chunksize[self.dimensions['time']] = chunk_map[0] + self.field_chunksize[self.dimensions['depth']] = chunk_map[1] + self.field_chunksize[self.dimensions['lat']] = chunk_map[2] + self.field_chunksize[self.dimensions['lon']] = chunk_map[3] + dim_index = 4 for dim_name in self.dimensions: if dim_name not in ['time', 'depth', 'lat', 'lon']: self.field_chunksize[self.dimensions[dim_name]] = chunk_map[dim_index] @@ -1869,12 +2006,21 @@ def read_depth(self): elif len(depth.shape) == 3: return np.array(depth[self.indices['depth'], self.indices['lat'], self.indices['lon']]) elif len(depth.shape) == 4: - raise NotImplementedError('Time varying depth data cannot be read in netcdf files yet') + # raise NotImplementedError('Time varying depth data cannot be read in netcdf files yet') return np.array(depth[:, self.indices['depth'], self.indices['lat'], self.indices['lon']]) else: self.indices['depth'] = [0] return np.zeros(1) + @property + def read_depth_dimensions(self): + if 'depth' in self.dimensions: + data = self.dataset[self.name] + depthsize = data.shape[-3] + self.data_full_zdim = depthsize + self.indices['depth'] = self.indices['depth'] if 'depth' in self.indices else range(depthsize) + return np.empty((0, len(self.indices['depth']), len(self.indices['lat']), len(self.indices['lon']))) + @property def data(self): return self.data_access() @@ -1957,13 +2103,13 @@ def data_access(self): data = data.rechunk(self.field_chunksize) self.chunk_mapping = {} chunkIndex = 0 - timei, _, timevalue = self._is_dimension_in_dataset('time') - depthi, _, depthvalue = self._is_dimension_in_dataset('depth') - has_time = timei >= 0 and timevalue > 1 - has_depth = depthi >= 0 and depthvalue > 1 + # timei, _, timevalue = self._is_dimension_in_dataset('time') + # depthi, _, depthvalue = self._is_dimension_in_dataset('depth') + # has_time = timei >= 0 and timevalue > 1 + # has_depth = depthi >= 0 and depthvalue > 1 startblock = 0 - startblock += 1 if has_time and not self._is_dimension_available('time') else 0 - startblock += 1 if has_depth and not self._is_dimension_available('depth') else 0 + # startblock += 1 if has_time and not self._is_dimension_available('time') else 0 + # startblock += 1 if has_depth and not self._is_dimension_available('depth') else 0 for chunkDim in data.chunksize[startblock:]: self.chunk_mapping[chunkIndex] = chunkDim chunkIndex += 1 @@ -1976,12 +2122,14 @@ def data_access(self): data = data.rechunk(self.chunk_mapping) self.chunking_finalized = True else: - self.chunking_finalized = True da_data = da.from_array(data, chunks=self.field_chunksize) if self.field_chunksize == 'auto' and da_data.shape[-2:] == da_data.chunksize[-2:]: data = np.array(data) else: data = da_data + if not self.chunking_finalized and self.rechunk_callback_fields is not None: + self.rechunk_callback_fields() + self.chunking_finalized = True return data diff --git a/parcels/fieldset.py b/parcels/fieldset.py index 315085ff1d..c5e3865935 100644 --- a/parcels/fieldset.py +++ b/parcels/fieldset.py @@ -14,6 +14,8 @@ from parcels.tools.converters import TimeConverter, convert_xarray_time_units from parcels.tools.error import TimeExtrapolationError from parcels.tools.loggers import logger +import functools + try: from mpi4py import MPI except: @@ -137,7 +139,65 @@ def add_field(self, field, name=None): raise NotImplementedError('FieldLists have been replaced by SummedFields. Use the + operator instead of []') else: setattr(self, name, field) + + if (isinstance(field.data, DeferredArray) or isinstance(field.data, da.core.Array)) and len(self.get_fields()) > 0: + # ==== check for inhabiting the same grid, and homogenise the grid chunking ==== # + g_set = field.grid + grid_chunksize = field.field_chunksize + dFiles = field.dataFiles + is_processed_grid = False + is_same_grid = False + for fld in self.get_fields(): # avoid re-processing/overwriting existing and working fields + if fld.grid == g_set: + is_processed_grid |= True + break + if not is_processed_grid: + for fld in self.get_fields(): + procdims = fld.dimensions + procinds = fld.indices + procpaths = fld.dataFiles + nowpaths = field.dataFiles + if procdims == field.dimensions and procinds == field.indices: + is_same_grid = False + if field.grid.mesh == fld.grid.mesh: + is_same_grid = True + else: + is_same_grid = True + for dim in ['lon', 'lat', 'depth', 'time']: + if dim in field.dimensions.keys() and dim in fld.dimensions.keys(): + is_same_grid &= (field.dimensions[dim] == fld.dimensions[dim]) + fld_g_dims = [fld.grid.tdim, fld.grid.zdim, fld.ydim, fld.xdim] + field_g_dims = [field.grid.tdim, field.grid.zdim, field.grid.ydim, field.grid.xdim] + for i in range(0, len(fld_g_dims)): + is_same_grid &= (field_g_dims[i] == fld_g_dims[i]) + if is_same_grid: + g_set = fld.grid + # ==== check here that the dims of field_chunksize are the same ==== # + if g_set.master_chunksize is not None: + res = False + if (isinstance(field.field_chunksize, tuple) and isinstance(g_set.master_chunksize, tuple)) or (isinstance(field.field_chunksize, dict) and isinstance(g_set.master_chunksize, dict)): + res |= functools.reduce(lambda i, j: i and j, map(lambda m, k: m == k, field.field_chunksize, g_set.master_chunksize), True) + else: + res |= (field.field_chunksize == g_set.master_chunksize) + if res: + grid_chunksize = g_set.master_chunksize + if field.grid.master_chunksize is not None: + logger.warning_once("Trying to initialize a shared grid with different chunking sizes - action prohibited. Replacing requested field_chunksize with grid's master chunksize.") + else: + raise ValueError("Conflict between grids of the same fieldset chunksize and requested field chunksize as well as the chunked name dimensions - Please apply the same chunksize to all fields in a shared grid!") + if procpaths == nowpaths: + dFiles = fld.dataFiles + break + if is_same_grid: + if field.grid != g_set: + field.grid = g_set + if field.field_chunksize != grid_chunksize: + field.field_chunksize = grid_chunksize + if field.dataFiles != dFiles: + field.dataFiles = dFiles + self.gridset.add_grid(field) + field.fieldset = self def add_vector_field(self, vfield): @@ -162,7 +222,8 @@ def check_complete(self): g.check_zonal_periodic() if len(g.time) == 1: continue - assert isinstance(g.time_origin.time_origin, type(self.time_origin.time_origin)), 'time origins of different grids must be have the same type' + emsg = "time origins of different grids must be have the same type - this grid: {} (type: {}) vs. compare-grid: {} (type: {})".format(self.time_origin, type(self.time_origin.time_origin), g.time_origin, type(g.time_origin.time_origin)) + assert isinstance(g.time_origin.time_origin, type(self.time_origin.time_origin)), emsg g.time = g.time + self.time_origin.reltime(g.time_origin) if g.defer_load: g.time_full = g.time_full + self.time_origin.reltime(g.time_origin) @@ -192,6 +253,16 @@ def check_complete(self): counter += 1 ccode_fieldnames.append(fld.ccode_name) + for f in self.get_fields(): + if type(f) in [VectorField, NestedField, SummedField] or f.dataFiles is None: + continue + if f.grid.depth_field is not None: + if f.grid.depth_field == 'not_yet_set': + raise ValueError("If depth dimension is set at 'not_yet_set', it must be added later using Field.set_depth_from_field(field)") + if not f.grid.defer_load: + depth_data = f.grid.depth_field.data + f.grid.depth = depth_data if isinstance(depth_data, np.ndarray) else np.array(depth_data) + @classmethod def parse_wildcards(cls, paths, filenames, var): if not isinstance(paths, list): @@ -206,7 +277,8 @@ def parse_wildcards(cls, paths, filenames, var): @classmethod def from_netcdf(cls, filenames, variables, dimensions, indices=None, fieldtype=None, - mesh='spherical', timestamps=None, allow_time_extrapolation=None, time_periodic=False, deferred_load=True, **kwargs): + mesh='spherical', timestamps=None, allow_time_extrapolation=None, time_periodic=False, + deferred_load=True, field_chunksize='auto', **kwargs): """Initialises FieldSet object from NetCDF files :param filenames: Dictionary mapping variables to file(s). The @@ -274,31 +346,50 @@ def from_netcdf(cls, filenames, variables, dimensions, indices=None, fieldtype=N cls.checkvaliddimensionsdict(dims) inds = indices[var] if (indices and var in indices) else indices fieldtype = fieldtype[var] if (fieldtype and var in fieldtype) else fieldtype + chunksize = field_chunksize[var] if (field_chunksize and var in field_chunksize) else field_chunksize grid = None + grid_chunksize = chunksize + dFiles = None # check if grid has already been processed (i.e. if other fields have same filenames, dimensions and indices) for procvar, _ in fields.items(): procdims = dimensions[procvar] if procvar in dimensions else dimensions procinds = indices[procvar] if (indices and procvar in indices) else indices procpaths = filenames[procvar] if isinstance(filenames, dict) and procvar in filenames else filenames nowpaths = filenames[var] if isinstance(filenames, dict) and var in filenames else filenames - if procdims == dims and procinds == inds and procpaths == nowpaths: - sameGrid = False + if procdims == dims and procinds == inds: + if 'depth' in dims and dims['depth'] == 'not_yet_set': + break + processedGrid = False if ((not isinstance(filenames, dict)) or filenames[procvar] == filenames[var]): - sameGrid = True + processedGrid = True elif isinstance(filenames[procvar], dict): - sameGrid = True + processedGrid = True for dim in ['lon', 'lat', 'depth']: if dim in dimensions: - sameGrid *= filenames[procvar][dim] == filenames[var][dim] - if sameGrid: + processedGrid *= filenames[procvar][dim] == filenames[var][dim] + if processedGrid: grid = fields[procvar].grid - kwargs['dataFiles'] = fields[procvar].dataFiles - break + # ==== check here that the dims of field_chunksize are the same ==== # + if grid.master_chunksize is not None: + res = False + if (isinstance(chunksize, tuple) and isinstance(grid.master_chunksize, tuple)) or (isinstance(chunksize, dict) and isinstance(grid.master_chunksize, dict)): + res |= functools.reduce(lambda i, j: i and j, map(lambda m, k: m == k, chunksize, grid.master_chunksize), True) + else: + res |= (chunksize == grid.master_chunksize) + if res: + grid_chunksize = grid.master_chunksize + logger.warning_once("Trying to initialize a shared grid with different chunking sizes - action prohibited. Replacing requested field_chunksize with grid's master chunksize.") + else: + raise ValueError("Conflict between grids of the same fieldset chunksize and requested field chunksize as well as the chunked name dimensions - Please apply the same chunksize to all fields in a shared grid!") + if procpaths == nowpaths: + dFiles = fields[procvar].dataFiles + break fields[var] = Field.from_netcdf(paths, (var, name), dims, inds, grid=grid, mesh=mesh, timestamps=timestamps, allow_time_extrapolation=allow_time_extrapolation, time_periodic=time_periodic, deferred_load=deferred_load, - fieldtype=fieldtype, **kwargs) + fieldtype=fieldtype, field_chunksize=grid_chunksize, dataFiles=dFiles, **kwargs) + u = fields.pop('U', None) v = fields.pop('V', None) return cls(u, v, fields=fields) @@ -306,7 +397,7 @@ def from_netcdf(cls, filenames, variables, dimensions, indices=None, fieldtype=N @classmethod def from_nemo(cls, filenames, variables, dimensions, indices=None, mesh='spherical', allow_time_extrapolation=None, time_periodic=False, - tracer_interp_method='cgrid_tracer', **kwargs): + tracer_interp_method='cgrid_tracer', field_chunksize='auto', **kwargs): """Initialises FieldSet object from NetCDF files of Curvilinear NEMO fields. :param filenames: Dictionary mapping variables to file(s). The @@ -354,13 +445,15 @@ def from_nemo(cls, filenames, variables, dimensions, indices=None, mesh='spheric This flag overrides the allow_time_interpolation and sets it to False :param tracer_interp_method: Method for interpolation of tracer fields. It is recommended to use 'cgrid_tracer' (default) Note that in the case of from_nemo() and from_cgrid(), the velocity fields are default to 'cgrid_velocity' + :param field_chunksize: size of the chunks in dask loading """ if 'creation_log' not in kwargs.keys(): kwargs['creation_log'] = 'from_nemo' fieldset = cls.from_c_grid_dataset(filenames, variables, dimensions, mesh=mesh, indices=indices, time_periodic=time_periodic, - allow_time_extrapolation=allow_time_extrapolation, tracer_interp_method=tracer_interp_method, **kwargs) + allow_time_extrapolation=allow_time_extrapolation, tracer_interp_method=tracer_interp_method, + field_chunksize=field_chunksize, **kwargs) if hasattr(fieldset, 'W'): fieldset.W.set_scaling_factor(-1.) return fieldset @@ -368,7 +461,7 @@ def from_nemo(cls, filenames, variables, dimensions, indices=None, mesh='spheric @classmethod def from_c_grid_dataset(cls, filenames, variables, dimensions, indices=None, mesh='spherical', allow_time_extrapolation=None, time_periodic=False, - tracer_interp_method='cgrid_tracer', **kwargs): + tracer_interp_method='cgrid_tracer', field_chunksize='auto', **kwargs): """Initialises FieldSet object from NetCDF files of Curvilinear NEMO fields. :param filenames: Dictionary mapping variables to file(s). The @@ -416,6 +509,7 @@ def from_c_grid_dataset(cls, filenames, variables, dimensions, indices=None, mes This flag overrides the allow_time_interpolation and sets it to False :param tracer_interp_method: Method for interpolation of tracer fields. It is recommended to use 'cgrid_tracer' (default) Note that in the case of from_nemo() and from_cgrid(), the velocity fields are default to 'cgrid_velocity' + :param field_chunksize: size of the chunks in dask loading """ @@ -434,12 +528,13 @@ def from_c_grid_dataset(cls, filenames, variables, dimensions, indices=None, mes kwargs['creation_log'] = 'from_c_grid_dataset' return cls.from_netcdf(filenames, variables, dimensions, mesh=mesh, indices=indices, time_periodic=time_periodic, - allow_time_extrapolation=allow_time_extrapolation, interp_method=interp_method, **kwargs) + allow_time_extrapolation=allow_time_extrapolation, interp_method=interp_method, + field_chunksize=field_chunksize, **kwargs) @classmethod def from_pop(cls, filenames, variables, dimensions, indices=None, mesh='spherical', allow_time_extrapolation=None, time_periodic=False, - tracer_interp_method='bgrid_tracer', **kwargs): + tracer_interp_method='bgrid_tracer', field_chunksize='auto', **kwargs): """Initialises FieldSet object from NetCDF files of POP fields. It is assumed that the velocities in the POP fields is in cm/s. @@ -489,13 +584,15 @@ def from_pop(cls, filenames, variables, dimensions, indices=None, mesh='spherica This flag overrides the allow_time_interpolation and sets it to False :param tracer_interp_method: Method for interpolation of tracer fields. It is recommended to use 'bgrid_tracer' (default) Note that in the case of from_pop() and from_bgrid(), the velocity fields are default to 'bgrid_velocity' + :param field_chunksize: size of the chunks in dask loading """ if 'creation_log' not in kwargs.keys(): kwargs['creation_log'] = 'from_pop' fieldset = cls.from_b_grid_dataset(filenames, variables, dimensions, mesh=mesh, indices=indices, time_periodic=time_periodic, - allow_time_extrapolation=allow_time_extrapolation, tracer_interp_method=tracer_interp_method, **kwargs) + allow_time_extrapolation=allow_time_extrapolation, tracer_interp_method=tracer_interp_method, + field_chunksize=field_chunksize, **kwargs) if hasattr(fieldset, 'U'): fieldset.U.set_scaling_factor(0.01) # cm/s to m/s if hasattr(fieldset, 'V'): @@ -507,7 +604,7 @@ def from_pop(cls, filenames, variables, dimensions, indices=None, mesh='spherica @classmethod def from_b_grid_dataset(cls, filenames, variables, dimensions, indices=None, mesh='spherical', allow_time_extrapolation=None, time_periodic=False, - tracer_interp_method='bgrid_tracer', **kwargs): + tracer_interp_method='bgrid_tracer', field_chunksize='auto', **kwargs): """Initialises FieldSet object from NetCDF files of Bgrid fields. :param filenames: Dictionary mapping variables to file(s). The @@ -555,6 +652,7 @@ def from_b_grid_dataset(cls, filenames, variables, dimensions, indices=None, mes This flag overrides the allow_time_interpolation and sets it to False :param tracer_interp_method: Method for interpolation of tracer fields. It is recommended to use 'bgrid_tracer' (default) Note that in the case of from_pop() and from_bgrid(), the velocity fields are default to 'bgrid_velocity' + :param field_chunksize: size of the chunks in dask loading """ @@ -575,11 +673,13 @@ def from_b_grid_dataset(cls, filenames, variables, dimensions, indices=None, mes kwargs['creation_log'] = 'from_b_grid_dataset' return cls.from_netcdf(filenames, variables, dimensions, mesh=mesh, indices=indices, time_periodic=time_periodic, - allow_time_extrapolation=allow_time_extrapolation, interp_method=interp_method, **kwargs) + allow_time_extrapolation=allow_time_extrapolation, interp_method=interp_method, + field_chunksize=field_chunksize, **kwargs) @classmethod def from_parcels(cls, basename, uvar='vozocrtx', vvar='vomecrty', indices=None, extra_fields=None, - allow_time_extrapolation=None, time_periodic=False, deferred_load=True, **kwargs): + allow_time_extrapolation=None, time_periodic=False, deferred_load=True, + field_chunksize='auto', **kwargs): """Initialises FieldSet data from NetCDF files using the Parcels FieldSet.write() conventions. :param basename: Base name of the file(s); may contain @@ -600,6 +700,8 @@ def from_parcels(cls, basename, uvar='vozocrtx', vvar='vomecrty', indices=None, fully load them (default: True). It is advised to deferred load the data, since in that case Parcels deals with a better memory management during particle set execution. deferred_load=False is however sometimes necessary for plotting the fields. + :param field_chunksize: size of the chunks in dask loading + """ if extra_fields is None: @@ -618,7 +720,8 @@ def from_parcels(cls, basename, uvar='vozocrtx', vvar='vomecrty', indices=None, for v in extra_fields.keys()]) return cls.from_netcdf(filenames, indices=indices, variables=extra_fields, dimensions=dimensions, allow_time_extrapolation=allow_time_extrapolation, - time_periodic=time_periodic, deferred_load=deferred_load, **kwargs) + time_periodic=time_periodic, deferred_load=deferred_load, + field_chunksize=field_chunksize, **kwargs) @classmethod def from_xarray_dataset(cls, ds, variables, dimensions, mesh='spherical', allow_time_extrapolation=None, @@ -874,6 +977,14 @@ def computeTimeChunk(self, time, dt): if self.compute_on_defer: self.compute_on_defer(self) + # update time varying grid depth + for f in self.get_fields(): + if type(f) in [VectorField, NestedField, SummedField] or not f.grid.defer_load or f.dataFiles is None: + continue + if f.grid.depth_field is not None: + depth_data = f.grid.depth_field.data + f.grid.depth = depth_data if isinstance(depth_data, np.ndarray) else np.array(depth_data) + if abs(nextTime) == np.infty or np.isnan(nextTime): # Second happens when dt=0 return nextTime else: diff --git a/parcels/grid.py b/parcels/grid.py index 260042bea4..1ae31fa771 100644 --- a/parcels/grid.py +++ b/parcels/grid.py @@ -63,7 +63,9 @@ def __init__(self, lon, lat, time, time_origin, mesh): self.periods = 0 self.load_chunk = [] self.chunk_info = None + self.master_chunksize = None self._add_last_periodic_data_timestep = False + self.depth_field = None @staticmethod def create_grid(lon, lat, depth, time, time_origin, mesh, **kwargs): @@ -341,9 +343,10 @@ def __init__(self, lon, lat, depth, time=None, time_origin=None, mesh='flat'): self.zdim = self.depth.shape[-3] self.z4d = len(self.depth.shape) == 4 if self.z4d: - assert self.tdim == self.depth.shape[0], 'depth dimension has the wrong format. It should be [tdim, zdim, ydim, xdim]' - assert self.xdim == self.depth.shape[-1], 'depth dimension has the wrong format. It should be [tdim, zdim, ydim, xdim]' - assert self.ydim == self.depth.shape[-2], 'depth dimension has the wrong format. It should be [tdim, zdim, ydim, xdim]' + # self.depth.shape[0] is 0 for S grids loaded from netcdf file + assert self.tdim == self.depth.shape[0] or self.depth.shape[0] == 0, 'depth dimension has the wrong format. It should be [tdim, zdim, ydim, xdim]' + assert self.xdim == self.depth.shape[-1] or self.depth.shape[-1] == 0, 'depth dimension has the wrong format. It should be [tdim, zdim, ydim, xdim]' + assert self.ydim == self.depth.shape[-2] or self.depth.shape[-2] == 0, 'depth dimension has the wrong format. It should be [tdim, zdim, ydim, xdim]' else: assert self.xdim == self.depth.shape[-1], 'depth dimension has the wrong format. It should be [zdim, ydim, xdim]' assert self.ydim == self.depth.shape[-2], 'depth dimension has the wrong format. It should be [zdim, ydim, xdim]' @@ -471,9 +474,10 @@ def __init__(self, lon, lat, depth, time=None, time_origin=None, mesh='flat'): self.zdim = self.depth.shape[-3] self.z4d = len(self.depth.shape) == 4 if self.z4d: - assert self.tdim == self.depth.shape[0], 'depth dimension has the wrong format. It should be [tdim, zdim, ydim, xdim]' - assert self.xdim == self.depth.shape[-1], 'depth dimension has the wrong format. It should be [tdim, zdim, ydim, xdim]' - assert self.ydim == self.depth.shape[-2], 'depth dimension has the wrong format. It should be [tdim, zdim, ydim, xdim]' + # self.depth.shape[0] is 0 for S grids loaded from netcdf file + assert self.tdim == self.depth.shape[0] or self.depth.shape[0] == 0, 'depth dimension has the wrong format. It should be [tdim, zdim, ydim, xdim]' + assert self.xdim == self.depth.shape[-1] or self.depth.shape[-1] == 0, 'depth dimension has the wrong format. It should be [tdim, zdim, ydim, xdim]' + assert self.ydim == self.depth.shape[-2] or self.depth.shape[-2] == 0, 'depth dimension has the wrong format. It should be [tdim, zdim, ydim, xdim]' else: assert self.xdim == self.depth.shape[-1], 'depth dimension has the wrong format. It should be [zdim, ydim, xdim]' assert self.ydim == self.depth.shape[-2], 'depth dimension has the wrong format. It should be [zdim, ydim, xdim]' diff --git a/parcels/gridset.py b/parcels/gridset.py index 7d7b182199..975ee7cb3d 100644 --- a/parcels/gridset.py +++ b/parcels/gridset.py @@ -1,4 +1,6 @@ import numpy as np +import functools +from parcels.tools.loggers import logger __all__ = ['GridSet'] @@ -15,20 +17,43 @@ def add_grid(self, field): grid = field.grid existing_grid = False for g in self.grids: + if field.field_chunksize != grid.master_chunksize: + logger.warning_once("Field chunksize and Grid master chunksize are not equal - erroneous behaviour expected.") + break + if g == grid: + existing_grid = True + break sameGrid = True + sameDims = True if grid.time_origin != g.time_origin: + sameDims = False continue for attr in ['lon', 'lat', 'depth', 'time']: gattr = getattr(g, attr) gridattr = getattr(grid, attr) if gattr.shape != gridattr.shape or not np.allclose(gattr, gridattr): sameGrid = False + sameDims = False break - if not sameGrid: + if not sameDims: continue - existing_grid = True - field.grid = g - break + sameGrid &= (grid.master_chunksize == g.master_chunksize) or (grid.master_chunksize in [False, None] and g.master_chunksize in [False, None]) + if not sameGrid and sameDims and grid.master_chunksize is not None: + res = False + if (isinstance(grid.master_chunksize, tuple) and isinstance(g.master_chunksize, tuple)) or \ + (isinstance(grid.master_chunksize, dict) and isinstance(g.master_chunksize, dict)): + res |= functools.reduce(lambda i, j: i and j, + map(lambda m, k: m == k, grid.master_chunksize, g.master_chunksize), True) + if res: + sameGrid = True + logger.warning_once("Trying to initialize a shared grid with different chunking sizes - action prohibited. Replacing requested field_chunksize with grid's master chunksize.") + else: + raise ValueError("Conflict between grids of the same gridset: major grid chunksize and requested sibling-grid chunksize as well as their chunk-dimension names are not equal - Please apply the same chunksize to all fields in a shared grid!") + break + if sameGrid: + existing_grid = True + field.grid = g + break if not existing_grid: self.grids.append(grid) diff --git a/parcels/include/index_search.h b/parcels/include/index_search.h index 4e834713d0..2189341bb5 100644 --- a/parcels/include/index_search.h +++ b/parcels/include/index_search.h @@ -44,7 +44,7 @@ typedef struct typedef enum { - SUCCESS=0, EVALUATE=1, REPEAT=2, DELETE=3, ERROR=4, ERROR_INTERPOLATION=41, ERROR_OUT_OF_BOUNDS=5, ERROR_THROUGH_SURFACE=51, ERROR_TIME_EXTRAPOLATION=6 + SUCCESS=0, EVALUATE=1, REPEAT=2, DELETE=3, STOP_EXECUTION=4, ERROR=5, ERROR_INTERPOLATION=51, ERROR_OUT_OF_BOUNDS=6, ERROR_THROUGH_SURFACE=61, ERROR_TIME_EXTRAPOLATION=7 } ErrorCode; typedef enum diff --git a/parcels/kernel.py b/parcels/kernel.py index 2b9f5ef698..95a785e4f6 100644 --- a/parcels/kernel.py +++ b/parcels/kernel.py @@ -342,21 +342,29 @@ def execute_python(self, pset, endtime, dt): dt_pos = 0 break - def execute(self, pset, endtime, dt, recovery=None, output_file=None): + def remove_deleted(self, pset, output_file, endtime): + """Utility to remove all particles that signalled deletion""" + indices = [i for i, p in enumerate(pset.particles) + if p.state in [ErrorCode.Delete]] + if len(indices) > 0 and output_file is not None: + output_file.write(pset[indices], endtime, deleted_only=True) + pset.remove(indices) + + def execute(self, pset, endtime, dt, recovery=None, output_file=None, execute_once=False): """Execute this Kernel over a ParticleSet for several timesteps""" for p in pset.particles: p.reset_state() - if abs(dt) < 1e-6: + if abs(dt) < 1e-6 and not execute_once: logger.warning_once("'dt' is too small, causing numerical accuracy limit problems. Please chose a higher 'dt' and rather scale the 'time' axis of the field accordingly. (related issue #762)") - def remove_deleted(pset): - """Utility to remove all particles that signalled deletion""" - indices = [i for i, p in enumerate(pset.particles) - if p.state in [ErrorCode.Delete]] - if len(indices) > 0 and output_file is not None: - output_file.write(pset[indices], endtime, deleted_only=True) - pset.remove(indices) + # def remove_deleted(pset): + # """Utility to remove all particles that signalled deletion""" + # indices = [i for i, p in enumerate(pset.particles) + # if p.state in [ErrorCode.Delete]] + # if len(indices) > 0 and output_file is not None: + # output_file.write(pset[indices], endtime, deleted_only=True) + # pset.remove(indices) if recovery is None: recovery = {} @@ -376,7 +384,7 @@ def remove_deleted(pset): self.execute_python(pset, endtime, dt) # Remove all particles that signalled deletion - remove_deleted(pset) + self.remove_deleted(pset, output_file=output_file, endtime=endtime) # Identify particles that threw errors error_particles = [p for p in pset.particles if p.state not in [ErrorCode.Success, ErrorCode.Evaluate]] @@ -384,8 +392,13 @@ def remove_deleted(pset): while len(error_particles) > 0: # Apply recovery kernel for p in error_particles: + if p.state == ErrorCode.StopExecution: + return if p.state == ErrorCode.Repeat: p.reset_state() + elif p.state == ErrorCode.Delete: + # p.delete() + pass elif p.state in recovery_map: recovery_kernel = recovery_map[p.state] p.state = ErrorCode.Success @@ -393,11 +406,12 @@ def remove_deleted(pset): if(p.isComputed()): p.reset_state() else: - logger.warning_once('Deleting particle because of bug in #749 and #737') + logger.warning_once('Deleting particle {} because of non-recoverable error'.format(p.id)) + # logger.warning('Deleting particle because of bug in #749 and #737 - particle state: {}'.format(ErrorCode.toString(p.state))) p.delete() # Remove all particles that signalled deletion - remove_deleted(pset) + self.remove_deleted(pset, output_file=output_file, endtime=endtime) # Execute core loop again to continue interrupted particles if self.ptype.uses_jit: diff --git a/parcels/kernel_benchmark.py b/parcels/kernel_benchmark.py new file mode 100644 index 0000000000..07c430ab71 --- /dev/null +++ b/parcels/kernel_benchmark.py @@ -0,0 +1,265 @@ +import _ctypes +import inspect +import math # noqa +import random # noqa +import re +import time +from ast import FunctionDef +from ast import parse +from copy import deepcopy +from ctypes import byref +from ctypes import c_double +from ctypes import c_int +from ctypes import c_void_p +from hashlib import md5 +from os import path +from os import remove +from sys import platform +from sys import version_info + +import numpy as np +import numpy.ctypeslib as npct +try: + from mpi4py import MPI +except: + MPI = None + +from parcels.codegenerator import KernelGenerator +from parcels.codegenerator import LoopGenerator +from parcels.compiler import get_cache_dir +from parcels.field import Field +from parcels.field import FieldOutOfBoundError +from parcels.field import FieldOutOfBoundSurfaceError +from parcels.field import TimeExtrapolationError +from parcels.field import NestedField +from parcels.field import SummedField +from parcels.field import VectorField +from parcels.kernels.advection import AdvectionRK4_3D +from parcels.tools.error import ErrorCode +from parcels.tools.error import recovery_map as recovery_base_map +from parcels.tools.loggers import logger + +from parcels.kernel import Kernel +from parcels.tools.performance_logger import TimingLog + +__all__ = ['Kernel_Benchmark'] + + + +class Kernel_Benchmark(Kernel): + """Kernel object that encapsulates auto-generated code. + + :arg fieldset: FieldSet object providing the field information + :arg ptype: PType object for the kernel particle + :param delete_cfiles: Boolean whether to delete the C-files after compilation in JIT mode (default is True) + + Note: A Kernel is either created from a compiled object + or the necessary information (funcname, funccode, funcvars) is provided. + The py_ast argument may be derived from the code string, but for + concatenation, the merged AST plus the new header definition is required. + """ + + def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, + funccode=None, py_ast=None, funcvars=None, c_include="", delete_cfiles=True): + super(Kernel_Benchmark, self).__init__(fieldset, ptype, pyfunc, funcname, funccode, py_ast, funcvars, c_include, delete_cfiles) + self._compute_timings = TimingLog() + self._io_timings = TimingLog() + self._mem_io_timings = TimingLog() + + @property + def io_timings(self): + return self._io_timings + + @property + def mem_io_timings(self): + return self._mem_io_timings + + @property + def compute_timings(self): + return self._compute_timings + + def __del__(self): + super(Kernel_Benchmark, self).__del__() + + def execute_jit(self, pset, endtime, dt): + """Invokes JIT engine to perform the core update loop""" + self._io_timings.start_timing() + if len(pset.particles) > 0: + assert pset.fieldset.gridset.size == len(pset.particles[0].xi), \ + 'FieldSet has different amount of grids than Particle.xi. Have you added Fields after creating the ParticleSet?' + for g in pset.fieldset.gridset.grids: + g.cstruct = None # This force to point newly the grids from Python to C + # Make a copy of the transposed array to enforce + # C-contiguous memory layout for JIT mode. + for f in pset.fieldset.get_fields(): + if type(f) in [VectorField, NestedField, SummedField]: + continue + if f in self.field_args.values(): + f.chunk_data() + else: + for block_id in range(len(f.data_chunks)): + f.data_chunks[block_id] = None + f.c_data_chunks[block_id] = None + self._io_timings.stop_timing() + self._io_timings.accumulate_timing() + + self._mem_io_timings.start_timing() + for g in pset.fieldset.gridset.grids: + g.load_chunk = np.where(g.load_chunk == 1, 2, g.load_chunk) + if len(g.load_chunk) > 0: # not the case if a field in not called in the kernel + if not g.load_chunk.flags.c_contiguous: + g.load_chunk = g.load_chunk.copy() + if not g.depth.flags.c_contiguous: + g.depth = g.depth.copy() + if not g.lon.flags.c_contiguous: + g.lon = g.lon.copy() + if not g.lat.flags.c_contiguous: + g.lat = g.lat.copy() + + fargs = [byref(f.ctypes_struct) for f in self.field_args.values()] + fargs += [c_double(f) for f in self.const_args.values()] + particle_data = pset._particle_data.ctypes.data_as(c_void_p) + self._mem_io_timings.stop_timing() + self._mem_io_timings.accumulate_timing() + + self._compute_timings.start_timing() + self._function(c_int(len(pset)), particle_data, + c_double(endtime), + c_double(dt), + *fargs) + self._compute_timings.stop_timing() + self._compute_timings.accumulate_timing() + + self._io_timings.advance_iteration() + self._mem_io_timings.advance_iteration() + self._compute_timings.advance_iteration() + + def execute_python(self, pset, endtime, dt): + """Performs the core update loop via Python""" + sign_dt = np.sign(dt) + + # back up variables in case of ErrorCode.Repeat + p_var_back = {} + + for f in self.fieldset.get_fields(): + if type(f) in [VectorField, NestedField, SummedField]: + continue + + self._io_timings.start_timing() + loaded_data = f.data + self._io_timings.stop_timing() + self._io_timings.accumulate_timing() + self._mem_io_timings.start_timing() + f.data = np.array(loaded_data) + self._mem_io_timings.stop_timing() + self._mem_io_timings.accumulate_timing() + + self._compute_timings.start_timing() + for p in pset.particles: + ptype = p.getPType() + # Don't execute particles that aren't started yet + sign_end_part = np.sign(endtime - p.time) + + dt_pos = min(abs(p.dt), abs(endtime - p.time)) + + # ==== numerically stable; also making sure that continuously-recovered particles do end successfully, + # as they fulfil the condition here on entering at the final calculation here. ==== # + if ((sign_end_part != sign_dt) or np.isclose(dt_pos, 0)) and not np.isclose(dt, 0): + if abs(p.time) >= abs(endtime): + p.state = ErrorCode.Success + continue + + while p.state in [ErrorCode.Evaluate, ErrorCode.Repeat] or np.isclose(dt, 0): + + for var in ptype.variables: + p_var_back[var.name] = getattr(p, var.name) + try: + pdt_prekernels = sign_dt * dt_pos + p.dt = pdt_prekernels + state_prev = p.state + res = self.pyfunc(p, pset.fieldset, p.time) + if res is None: + res = ErrorCode.Success + + if res is ErrorCode.Success and p.state != state_prev: + res = p.state + + if res == ErrorCode.Success and not np.isclose(p.dt, pdt_prekernels): + res = ErrorCode.Repeat + + except FieldOutOfBoundError as fse_xy: + res = ErrorCode.ErrorOutOfBounds + p.exception = fse_xy + except FieldOutOfBoundSurfaceError as fse_z: + res = ErrorCode.ErrorThroughSurface + p.exception = fse_z + except TimeExtrapolationError as fse_t: + res = ErrorCode.ErrorTimeExtrapolation + p.exception = fse_t + except Exception as e: + res = ErrorCode.Error + p.exception = e + + # Handle particle time and time loop + if res in [ErrorCode.Success, ErrorCode.Delete]: + # Update time and repeat + p.time += p.dt + p.update_next_dt() + dt_pos = min(abs(p.dt), abs(endtime - p.time)) + + sign_end_part = np.sign(endtime - p.time) + if res != ErrorCode.Delete and not np.isclose(dt_pos, 0) and (sign_end_part == sign_dt): + res = ErrorCode.Evaluate + if sign_end_part != sign_dt: + dt_pos = 0 + + p.state = res + if np.isclose(dt, 0): + break + else: + p.state = res + # Try again without time update + for var in ptype.variables: + if var.name not in ['dt', 'state']: + setattr(p, var.name, p_var_back[var.name]) + dt_pos = min(abs(p.dt), abs(endtime - p.time)) + + sign_end_part = np.sign(endtime - p.time) + if sign_end_part != sign_dt: + dt_pos = 0 + break + self._compute_timings.stop_timing() + self._compute_timings.accumulate_timing() + + self._io_timings.advance_iteration() + self._mem_io_timings.advance_iteration() + self._compute_timings.advance_iteration() + + def remove_deleted(self, pset, output_file, endtime): + """Utility to remove all particles that signalled deletion""" + self._mem_io_timings.start_timing() + super(Kernel_Benchmark, self).remove_deleted(pset=pset, output_file=output_file, endtime=endtime) + self._mem_io_timings.stop_timing() + self._mem_io_timings.accumulate_timing() + self._mem_io_timings.advance_iteration() + + def merge(self, kernel): + funcname = self.funcname + kernel.funcname + func_ast = FunctionDef(name=funcname, args=self.py_ast.args, + body=self.py_ast.body + kernel.py_ast.body, + decorator_list=[], lineno=1, col_offset=0) + delete_cfiles = self.delete_cfiles and kernel.delete_cfiles + return Kernel_Benchmark(self.fieldset, self.ptype, pyfunc=None, + funcname=funcname, funccode=self.funccode + kernel.funccode, + py_ast=func_ast, funcvars=self.funcvars + kernel.funcvars, + delete_cfiles=delete_cfiles) + + def __add__(self, kernel): + if not isinstance(kernel, Kernel): + kernel = Kernel_Benchmark(self.fieldset, self.ptype, pyfunc=kernel) + return self.merge(kernel) + + def __radd__(self, kernel): + if not isinstance(kernel, Kernel): + kernel = Kernel_Benchmark(self.fieldset, self.ptype, pyfunc=kernel) + return kernel.merge(self) diff --git a/parcels/particle.py b/parcels/particle.py index c30e42455d..ff093b13ba 100644 --- a/parcels/particle.py +++ b/parcels/particle.py @@ -7,9 +7,10 @@ from parcels.tools.error import ErrorCode from parcels.tools.loggers import logger - __all__ = ['ScipyParticle', 'JITParticle', 'Variable'] +indicators_64bit = [np.float64, np.int64, c_void_p] + class Variable(object): """Descriptor class that delegates data access to particle data @@ -47,8 +48,7 @@ def __repr__(self): def is64bit(self): """Check whether variable is 64-bit""" - return True if self.dtype == np.float64 or self.dtype == np.int64 \ - or self.dtype == c_void_p else False + return True if self.dtype in indicators_64bit else False class ParticleType(object): diff --git a/parcels/particleset.py b/parcels/particleset.py index a4e89cc889..419666c97e 100644 --- a/parcels/particleset.py +++ b/parcels/particleset.py @@ -16,6 +16,7 @@ from parcels.kernels.advection import AdvectionRK4 from parcels.particle import JITParticle from parcels.particlefile import ParticleFile +from parcels.tools.error import ErrorCode from parcels.tools.loggers import logger try: from mpi4py import MPI @@ -353,7 +354,7 @@ def __iadd__(self, particles): self.add(particles) return self - def __create_progressbar(self, starttime, endtime): + def _create_progressbar_(self, starttime, endtime): pbar = None try: pbar = progressbar.ProgressBar(max_value=abs(endtime - starttime)).start() @@ -481,12 +482,14 @@ def execute(self, pyfunc=AdvectionRK4, endtime=None, runtime=None, dt=1., mintime, maxtime = self.fieldset.gridset.dimrange('time_full') endtime = maxtime if dt >= 0 else mintime + execute_once = False if abs(endtime-_starttime) < 1e-5 or dt == 0 or runtime == 0: dt = 0 runtime = 0 endtime = _starttime logger.warning_once("dt or runtime are zero, or endtime is equal to Particle.time. " "The kernels will be executed once, without incrementing time") + execute_once = True # Initialise particle timestepping for p in self: @@ -519,7 +522,7 @@ def execute(self, pyfunc=AdvectionRK4, endtime=None, runtime=None, dt=1., if verbose_progress is None: walltime_start = time_module.time() if verbose_progress: - pbar = self.__create_progressbar(_starttime, endtime) + pbar = self._create_progressbar_(_starttime, endtime) while (time < endtime and dt > 0) or (time > endtime and dt < 0) or dt == 0: if verbose_progress is None and time_module.time() - walltime_start > 10: # Showing progressbar if runtime > 10 seconds @@ -527,13 +530,13 @@ def execute(self, pyfunc=AdvectionRK4, endtime=None, runtime=None, dt=1., logger.info('Temporary output files are stored in %s.' % output_file.tempwritedir_base) logger.info('You can use "parcels_convert_npydir_to_netcdf %s" to convert these ' 'to a NetCDF file during the run.' % output_file.tempwritedir_base) - pbar = self.__create_progressbar(_starttime, endtime) + pbar = self._create_progressbar_(_starttime, endtime) verbose_progress = True if dt > 0: time = min(next_prelease, next_input, next_output, next_movie, next_callback, endtime) else: time = max(next_prelease, next_input, next_output, next_movie, next_callback, endtime) - self.kernel.execute(self, endtime=time, dt=dt, recovery=recovery, output_file=output_file) + self.kernel.execute(self, endtime=time, dt=dt, recovery=recovery, output_file=output_file, execute_once=execute_once) if abs(time-next_prelease) < tol: pset_new = ParticleSet(fieldset=self.fieldset, time=time, lon=self.repeatlon, lat=self.repeatlat, depth=self.repeatdepth, diff --git a/parcels/particleset_benchmark.py b/parcels/particleset_benchmark.py new file mode 100644 index 0000000000..5bc903acbf --- /dev/null +++ b/parcels/particleset_benchmark.py @@ -0,0 +1,483 @@ +import time as time_module +from datetime import datetime +from datetime import timedelta as delta +import psutil +import os +from platform import system as system_name +import matplotlib.pyplot as plt +import sys + + +import numpy as np + +try: + from mpi4py import MPI +except: + MPI = None + +from parcels.compiler import GNUCompiler +from parcels.kernels.advection import AdvectionRK4 +from parcels.particleset import ParticleSet +from parcels.particle import JITParticle +from parcels.kernel_benchmark import Kernel_Benchmark, Kernel +from parcels.tools.loggers import logger +from parcels.tools.performance_logger import TimingLog, ParamLogging, Asynchronous_ParamLogging + +from resource import getrusage, RUSAGE_SELF + +__all__ = ['ParticleSet_Benchmark'] + +def measure_mem(): + process = psutil.Process(os.getpid()) + pmem = process.memory_info() + pmem_total = pmem.shared + pmem.text + pmem.data + pmem.lib + # print("psutil - res-set: {}; res-shr: {} res-text: {}, res-data: {}, res-lib: {}; res-total: {}".format(pmem.rss, pmem.shared, pmem.text, pmem.data, pmem.lib, pmem_total)) + return pmem_total + +def measure_mem_rss(): + process = psutil.Process(os.getpid()) + pmem = process.memory_info() + pmem_total = pmem.shared + pmem.text + pmem.data + pmem.lib + # print("psutil - res-set: {}; res-shr: {} res-text: {}, res-data: {}, res-lib: {}; res-total: {}".format(pmem.rss, pmem.shared, pmem.text, pmem.data, pmem.lib, pmem_total)) + return pmem.rss + +def measure_mem_usage(): + rsc = getrusage(RUSAGE_SELF) + print("RUSAGE - Max. RES set-size: {}; shr. mem size: {}; ushr. mem size: {}".format(rsc.ru_maxrss, rsc.ru_ixrss, rsc.ru_idrss)) + if system_name() == "Linux": + return rsc.ru_maxrss*1024 + return rsc.ru_maxrss + +USE_ASYNC_MEMLOG = False +USE_RUSE_SYNC_MEMLOG = False # can be faulty + +class ParticleSet_Benchmark(ParticleSet): + + def __init__(self, fieldset, pclass=JITParticle, lon=None, lat=None, depth=None, time=None, repeatdt=None, + lonlatdepth_dtype=None, pid_orig=None, **kwargs): + super(ParticleSet_Benchmark, self).__init__(fieldset, pclass, lon, lat, depth, time, repeatdt, lonlatdepth_dtype, pid_orig, **kwargs) + self.total_log = TimingLog() + self.compute_log = TimingLog() + self.io_log = TimingLog() + self.mem_io_log = TimingLog() + self.plot_log = TimingLog() + self.nparticle_log = ParamLogging() + self.mem_log = ParamLogging() + self.async_mem_log = Asynchronous_ParamLogging() + self.process = psutil.Process(os.getpid()) + + def set_async_memlog_interval(self, interval): + self.async_mem_log.measure_interval = interval + + # @profile + def execute(self, pyfunc=AdvectionRK4, endtime=None, runtime=None, dt=1., + moviedt=None, recovery=None, output_file=None, movie_background_field=None, + verbose_progress=None, postIterationCallbacks=None, callbackdt=None): + """Execute a given kernel function over the particle set for + multiple timesteps. Optionally also provide sub-timestepping + for particle output. + + :param pyfunc: Kernel function to execute. This can be the name of a + defined Python function or a :class:`parcels.kernel.Kernel` object. + Kernels can be concatenated using the + operator + :param endtime: End time for the timestepping loop. + It is either a datetime object or a positive double. + :param runtime: Length of the timestepping loop. Use instead of endtime. + It is either a timedelta object or a positive double. + :param dt: Timestep interval to be passed to the kernel. + It is either a timedelta object or a double. + Use a negative value for a backward-in-time simulation. + :param moviedt: Interval for inner sub-timestepping (leap), which dictates + the update frequency of animation. + It is either a timedelta object or a positive double. + None value means no animation. + :param output_file: :mod:`parcels.particlefile.ParticleFile` object for particle output + :param recovery: Dictionary with additional `:mod:parcels.tools.error` + recovery kernels to allow custom recovery behaviour in case of + kernel errors. + :param movie_background_field: field plotted as background in the movie if moviedt is set. + 'vector' shows the velocity as a vector field. + :param verbose_progress: Boolean for providing a progress bar for the kernel execution loop. + :param postIterationCallbacks: (Optional) Array of functions that are to be called after each iteration (post-process, non-Kernel) + :param callbackdt: (Optional, in conjecture with 'postIterationCallbacks) timestep inverval to (latestly) interrupt the running kernel and invoke post-iteration callbacks from 'postIterationCallbacks' + """ + + # check if pyfunc has changed since last compile. If so, recompile + if self.kernel is None or (self.kernel.pyfunc is not pyfunc and self.kernel is not pyfunc): + # Generate and store Kernel + if isinstance(pyfunc, Kernel): + self.kernel = pyfunc + else: + self.kernel = self.Kernel(pyfunc) + # Prepare JIT kernel execution + if self.ptype.uses_jit: + self.kernel.remove_lib() + cppargs = ['-DDOUBLE_COORD_VARIABLES'] if self.lonlatdepth_dtype == np.float64 else None + self.kernel.compile(compiler=GNUCompiler(cppargs=cppargs)) + self.kernel.load_lib() + + # Convert all time variables to seconds + if isinstance(endtime, delta): + raise RuntimeError('endtime must be either a datetime or a double') + if isinstance(endtime, datetime): + endtime = np.datetime64(endtime) + if isinstance(endtime, np.datetime64): + if self.time_origin.calendar is None: + raise NotImplementedError('If fieldset.time_origin is not a date, execution endtime must be a double') + endtime = self.time_origin.reltime(endtime) + if isinstance(runtime, delta): + runtime = runtime.total_seconds() + if isinstance(dt, delta): + dt = dt.total_seconds() + outputdt = output_file.outputdt if output_file else np.infty + if isinstance(outputdt, delta): + outputdt = outputdt.total_seconds() + if isinstance(moviedt, delta): + moviedt = moviedt.total_seconds() + if isinstance(callbackdt, delta): + callbackdt = callbackdt.total_seconds() + + assert runtime is None or runtime >= 0, 'runtime must be positive' + assert outputdt is None or outputdt >= 0, 'outputdt must be positive' + assert moviedt is None or moviedt >= 0, 'moviedt must be positive' + + # Set particle.time defaults based on sign of dt, if not set at ParticleSet construction + for p in self: + if np.isnan(p.time): + mintime, maxtime = self.fieldset.gridset.dimrange('time_full') + p.time = mintime if dt >= 0 else maxtime + + # Derive _starttime and endtime from arguments or fieldset defaults + if runtime is not None and endtime is not None: + raise RuntimeError('Only one of (endtime, runtime) can be specified') + # ====================================== # + # ==== EXPENSIVE LIST COMPREHENSION ==== # + # ====================================== # + _starttime = min([p.time for p in self]) if dt >= 0 else max([p.time for p in self]) + if self.repeatdt is not None and self.repeat_starttime is None: + self.repeat_starttime = _starttime + if runtime is not None: + endtime = _starttime + runtime * np.sign(dt) + elif endtime is None: + mintime, maxtime = self.fieldset.gridset.dimrange('time_full') + endtime = maxtime if dt >= 0 else mintime + + execute_once = False + if abs(endtime-_starttime) < 1e-5 or dt == 0 or runtime == 0: + dt = 0 + runtime = 0 + endtime = _starttime + logger.warning_once("dt or runtime are zero, or endtime is equal to Particle.time. " + "The kernels will be executed once, without incrementing time") + execute_once = True + + # Initialise particle timestepping + for p in self: + p.dt = dt + + # First write output_file, because particles could have been added + if output_file: + output_file.write(self, _starttime) + if moviedt: + self.show(field=movie_background_field, show_time=_starttime, animation=True) + + if moviedt is None: + moviedt = np.infty + if callbackdt is None: + interupt_dts = [np.infty, moviedt, outputdt] + if self.repeatdt is not None: + interupt_dts.append(self.repeatdt) + callbackdt = np.min(np.array(interupt_dts)) + time = _starttime + if self.repeatdt: + next_prelease = self.repeat_starttime + (abs(time - self.repeat_starttime) // self.repeatdt + 1) * self.repeatdt * np.sign(dt) + else: + next_prelease = np.infty if dt > 0 else - np.infty + next_output = time + outputdt if dt > 0 else time - outputdt + next_movie = time + moviedt if dt > 0 else time - moviedt + next_callback = time + callbackdt if dt > 0 else time - callbackdt + next_input = self.fieldset.computeTimeChunk(time, np.sign(dt)) + + tol = 1e-12 + walltime_start = None + if verbose_progress is None: + walltime_start = time_module.time() + pbar = None + if verbose_progress: + pbar = self._create_progressbar_(_starttime, endtime) + + mem_used_start = 0 + if USE_ASYNC_MEMLOG: + self.async_mem_log.measure_func = measure_mem + mem_used_start = measure_mem() + + while (time < endtime and dt > 0) or (time > endtime and dt < 0) or dt == 0: + self.total_log.start_timing() + if USE_ASYNC_MEMLOG: + self.async_mem_log.measure_start_value = mem_used_start + self.async_mem_log.start_partial_measurement() + if verbose_progress is None and time_module.time() - walltime_start > 10: + # Showing progressbar if runtime > 10 seconds + if output_file: + logger.info('Temporary output files are stored in %s.' % output_file.tempwritedir_base) + logger.info('You can use "parcels_convert_npydir_to_netcdf %s" to convert these ' + 'to a NetCDF file during the run.' % output_file.tempwritedir_base) + pbar = self._create_progressbar_(_starttime, endtime) + verbose_progress = True + if dt > 0: + time = min(next_prelease, next_input, next_output, next_movie, next_callback, endtime) + else: + time = max(next_prelease, next_input, next_output, next_movie, next_callback, endtime) + # ==== compute ==== # + if not isinstance(self.kernel, Kernel_Benchmark): + self.compute_log.start_timing() + self.kernel.execute(self, endtime=time, dt=dt, recovery=recovery, output_file=output_file, execute_once=execute_once) + if abs(time-next_prelease) < tol: + # creating new particles equals a memory-io operation + if not isinstance(self.kernel, Kernel_Benchmark): + self.compute_log.stop_timing() + self.compute_log.accumulate_timing() + + self.mem_io_log.start_timing() + pset_new = ParticleSet(fieldset=self.fieldset, time=time, lon=self.repeatlon, + lat=self.repeatlat, depth=self.repeatdepth, + pclass=self.repeatpclass, lonlatdepth_dtype=self.lonlatdepth_dtype, + partitions=False, pid_orig=self.repeatpid, **self.repeatkwargs) + for p in pset_new: + p.dt = dt + self.add(pset_new) + self.mem_io_log.stop_timing() + self.mem_io_log.accumulate_timing() + next_prelease += self.repeatdt * np.sign(dt) + else: + if not isinstance(self.kernel, Kernel_Benchmark): + self.compute_log.stop_timing() + else: + pass + if isinstance(self.kernel, Kernel_Benchmark): + self.compute_log.add_aux_measure(self.kernel.compute_timings.sum()) + self.kernel.compute_timings.reset() + self.io_log.add_aux_measure(self.kernel.io_timings.sum()) + self.kernel.io_timings.reset() + self.mem_io_log.add_aux_measure(self.kernel.mem_io_timings.sum()) + self.kernel.mem_io_timings.reset() + self.compute_log.accumulate_timing() + self.nparticle_log.advance_iteration(len(self)) + # ==== end compute ==== # + if abs(time-next_output) < tol: # ==== IO ==== # + if output_file: + self.io_log.start_timing() + output_file.write(self, time) + self.io_log.stop_timing() + self.io_log.accumulate_timing() + next_output += outputdt * np.sign(dt) + if abs(time-next_movie) < tol: # ==== Plotting ==== # + self.plot_log.start_timing() + self.show(field=movie_background_field, show_time=time, animation=True) + self.plot_log.stop_timing() + self.plot_log.accumulate_timing() + next_movie += moviedt * np.sign(dt) + # ==== insert post-process here to also allow for memory clean-up via external func ==== # + if abs(time-next_callback) < tol: + # ==== assuming post-processing functions largely use memory than hard computation ... ==== # + self.mem_io_log.start_timing() + if postIterationCallbacks is not None: + for extFunc in postIterationCallbacks: + extFunc() + self.mem_io_log.stop_timing() + self.mem_io_log.accumulate_timing() + next_callback += callbackdt * np.sign(dt) + if time != endtime: # ==== IO ==== # + self.io_log.start_timing() + next_input = self.fieldset.computeTimeChunk(time, dt) + self.io_log.stop_timing() + self.io_log.accumulate_timing() + if dt == 0: + break + if verbose_progress: # ==== Plotting ==== # + self.plot_log.start_timing() + pbar.update(abs(time - _starttime)) + self.plot_log.stop_timing() + self.plot_log.accumulate_timing() + self.total_log.stop_timing() + self.total_log.accumulate_timing() + mem_B_used_total = 0 + # if MPI: + # mpi_comm = MPI.COMM_WORLD + # mem_B_used = self.process.memory_info().rss + # mem_B_used_total = mpi_comm.reduce(mem_B_used, op=MPI.SUM, root=0) + # else: + # mem_B_used_total = self.process.memory_info().rss + # mem_B_used_total = self.process.memory_info().rss + mem_B_used_total = 0 + if USE_RUSE_SYNC_MEMLOG: + mem_B_used_total = measure_mem_usage() + else: + mem_B_used_total = measure_mem_rss() + self.mem_log.advance_iteration(mem_B_used_total) + if USE_ASYNC_MEMLOG: + self.async_mem_log.stop_partial_measurement() # does 'advance_iteration' internally + + self.compute_log.advance_iteration() + self.io_log.advance_iteration() + self.mem_io_log.advance_iteration() + self.plot_log.advance_iteration() + self.total_log.advance_iteration() + + if output_file: + self.io_log.start_timing() + output_file.write(self, time) + self.io_log.stop_timing() + self.io_log.accumulate_timing() + if verbose_progress: + self.plot_log.start_timing() + pbar.finish() + self.plot_log.stop_timing() + self.plot_log.accumulate_timing() + + # ==== Those lines include the timing for file I/O of the set. ==== # + # ==== Disabled as it doesn't have anything to do with advection. ==== # + # self.nparticle_log.advance_iteration(self.size) + # self.compute_log.advance_iteration() + # self.io_log.advance_iteration() + # self.mem_log.advance_iteration(self.process.memory_info().rss) + # self.mem_io_log.advance_iteration() + # self.plot_log.advance_iteration() + # self.total_log.advance_iteration() + + def Kernel(self, pyfunc, c_include="", delete_cfiles=True): + """Wrapper method to convert a `pyfunc` into a :class:`parcels.kernel_benchmark.Kernel` object + based on `fieldset` and `ptype` of the ParticleSet + :param delete_cfiles: Boolean whether to delete the C-files after compilation in JIT mode (default is True) + """ + return Kernel_Benchmark(self.fieldset, self.ptype, pyfunc=pyfunc, c_include=c_include, + delete_cfiles=delete_cfiles) + + def plot_and_log(self, total_times = None, compute_times = None, io_times = None, plot_times = None, memory_used = None, nparticles = None, target_N = 1, imageFilePath = "", odir = os.getcwd(), xlim_range=None, ylim_range=None): + # == do something with the log-arrays == # + if total_times is None or type(total_times) not in [list, dict, np.ndarray]: + total_times = self.total_log.get_values() + if not isinstance(total_times, np.ndarray): + total_times = np.array(total_times) + if compute_times is None or type(compute_times) not in [list, dict, np.ndarray]: + compute_times = self.compute_log.get_values() + if not isinstance(compute_times, np.ndarray): + compute_times = np.array(compute_times) + mem_io_times = None + if io_times is None or type(io_times) not in [list, dict, np.ndarray]: + io_times = self.io_log.get_values() + mem_io_times = self.mem_io_log.get_values() + if not isinstance(io_times, np.ndarray): + io_times = np.array(io_times) + if mem_io_times is not None: + mem_io_times = np.array(mem_io_times) + io_times += mem_io_times + if plot_times is None or type(plot_times) not in [list, dict, np.ndarray]: + plot_times = self.plot_log.get_values() + if not isinstance(plot_times, np.ndarray): + plot_times = np.array(plot_times) + if memory_used is None or type(memory_used) not in [list, dict, np.ndarray]: + memory_used = self.mem_log.get_params() + if not isinstance(memory_used, np.ndarray): + memory_used = np.array(memory_used) + if nparticles is None or type(nparticles) not in [list, dict, np.ndarray]: + nparticles = [] + if not isinstance(nparticles, np.ndarray): + nparticles = np.array(nparticles, dtype=np.int32) + + memory_used_async = None + if USE_ASYNC_MEMLOG: + memory_used_async = np.array(self.async_mem_log.get_params(), dtype=np.int64) + + t_scaler = 1. * 10./1.0 + npart_scaler = 1.0 / 1000.0 + mem_scaler = 1.0 / (1024 * 1024 * 1024) + plot_t = (total_times * t_scaler).tolist() + plot_ct = (compute_times * t_scaler).tolist() + plot_iot = (io_times * t_scaler).tolist() + plot_drawt = (plot_times * t_scaler).tolist() + plot_npart = (nparticles * npart_scaler).tolist() + plot_mem = [] + if memory_used is not None and len(memory_used) > 1: + plot_mem = (memory_used * mem_scaler).tolist() + + plot_mem_async = None + if USE_ASYNC_MEMLOG: + plot_mem_async = (memory_used_async * mem_scaler).tolist() + + do_iot_plot = True + do_drawt_plot = False + do_mem_plot = True + do_mem_plot_async = True + do_npart_plot = True + assert (len(plot_t) == len(plot_ct)) + if len(plot_t) != len(plot_iot): + print("plot_t and plot_iot have different lengths ({} vs {})".format(len(plot_t), len(plot_iot))) + do_iot_plot = False + if len(plot_t) != len(plot_drawt): + print("plot_t and plot_drawt have different lengths ({} vs {})".format(len(plot_t), len(plot_iot))) + do_drawt_plot = False + if len(plot_t) != len(plot_mem): + print("plot_t and plot_mem have different lengths ({} vs {})".format(len(plot_t), len(plot_mem))) + do_mem_plot = False + if len(plot_t) != len(plot_npart): + print("plot_t and plot_npart have different lengths ({} vs {})".format(len(plot_t), len(plot_npart))) + do_npart_plot = False + x = np.arange(start=0, stop=len(plot_t)) + + fig, ax = plt.subplots(1, 1, figsize=(21, 12)) + ax.plot(x, plot_t, 's-', label="total time_spent [100ms]") + ax.plot(x, plot_ct, 'o-', label="compute-time spent [100ms]") + if do_iot_plot: + ax.plot(x, plot_iot, 'o-', label="io-time spent [100ms]") + if do_drawt_plot: + ax.plot(x, plot_drawt, 'o-', label="draw-time spent [100ms]") + if (memory_used is not None) and do_mem_plot: + ax.plot(x, plot_mem, '--', label="memory_used (cumulative) [1 GB]") + if USE_ASYNC_MEMLOG: + if (memory_used_async is not None) and do_mem_plot_async: + ax.plot(x, plot_mem_async, ':', label="memory_used [async] (cum.) [1GB]") + if do_npart_plot: + ax.plot(x, plot_npart, '-', label="sim. particles [# 1000]") + if xlim_range is not None: + plt.xlim(list(xlim_range)) # [0, 730] + if ylim_range is not None: + plt.ylim(list(ylim_range)) # [0, 120] + plt.legend() + ax.set_xlabel('iteration') + plt.savefig(os.path.join(odir, imageFilePath), dpi=600, format='png') + + sys.stdout.write("cumulative total runtime: {}\n".format(total_times.sum())) + sys.stdout.write("cumulative compute time: {}\n".format(compute_times.sum())) + sys.stdout.write("cumulative I/O time: {}\n".format(io_times.sum())) + sys.stdout.write("cumulative plot time: {}\n".format(plot_times.sum())) + + csv_file = os.path.splitext(imageFilePath)[0]+".csv" + with open(os.path.join(odir, csv_file), 'w') as f: + nparticles_t0 = 0 + nparticles_tN = 0 + if nparticles is not None: + nparticles_t0 = nparticles[0] + nparticles_tN = nparticles[-1] + ncores = 1 + if MPI: + mpi_comm = MPI.COMM_WORLD + ncores = mpi_comm.Get_size() + header_string = "target_N, start_N, final_N, avg_N, ncores, avg_kt_total[s], avg_kt_compute[s], avg_kt_io[s], avg_kt_plot[s], cum_t_total[s], cum_t_compute[s], com_t_io[s], cum_t_plot[s], max_mem[MB]\n" + f.write(header_string) + data_string = "{}, {}, {}, {}, {}, ".format(target_N, nparticles_t0, nparticles_tN, nparticles.mean(), ncores) + data_string += "{:2.10f}, {:2.10f}, {:2.10f}, {:2.10f}, ".format(total_times.mean(), compute_times.mean(), io_times.mean(), plot_times.mean()) + max_mem_sync = 0 + if memory_used is not None and len(memory_used) > 1: + memory_used = np.floor(memory_used / (1024*1024)) + memory_used = memory_used.astype(dtype=np.uint32) + max_mem_sync = memory_used.max() + max_mem_async = 0 + if USE_ASYNC_MEMLOG: + if memory_used_async is not None and len(memory_used_async) > 1: + memory_used_async = np.floor(memory_used_async / (1024*1024)) + memory_used_async = memory_used_async.astype(dtype=np.int64) + max_mem_async = memory_used_async.max() + max_mem = max(max_mem_sync, max_mem_async) + data_string += "{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {}".format(total_times.sum(), compute_times.sum(), io_times.sum(), plot_times.sum(), max_mem) + f.write(data_string) diff --git a/parcels/scripts/get_examples.py b/parcels/scripts/get_examples.py index de6437747f..43327870d8 100644 --- a/parcels/scripts/get_examples.py +++ b/parcels/scripts/get_examples.py @@ -37,10 +37,14 @@ "U_purely_zonal-ORCA025_grid_U.nc4", "V_purely_zonal-ORCA025_grid_V.nc4", "mesh_mask.nc4"]] + ["NemoNorthSeaORCA025-N006_data/" + fn for fn in [ - "ORCA025-N06_20000104d05U.nc", "ORCA025-N06_20000109d05U.nc", - "ORCA025-N06_20000104d05V.nc", "ORCA025-N06_20000109d05V.nc", - "ORCA025-N06_20000104d05W.nc", "ORCA025-N06_20000109d05W.nc", + "ORCA025-N06_20000104d05U.nc", "ORCA025-N06_20000109d05U.nc", "ORCA025-N06_20000114d05U.nc", "ORCA025-N06_20000119d05U.nc", "ORCA025-N06_20000124d05U.nc", "ORCA025-N06_20000129d05U.nc", + "ORCA025-N06_20000104d05V.nc", "ORCA025-N06_20000109d05V.nc", "ORCA025-N06_20000114d05V.nc", "ORCA025-N06_20000119d05V.nc", "ORCA025-N06_20000124d05V.nc", "ORCA025-N06_20000129d05V.nc", + "ORCA025-N06_20000104d05W.nc", "ORCA025-N06_20000109d05W.nc", "ORCA025-N06_20000114d05W.nc", "ORCA025-N06_20000119d05W.nc", "ORCA025-N06_20000124d05W.nc", "ORCA025-N06_20000129d05W.nc", "coordinates.nc"]] + + ["POPSouthernOcean_data/" + fn for fn in ["t.x1_SAMOC_flux.169000.nc", "t.x1_SAMOC_flux.169001.nc", + "t.x1_SAMOC_flux.169002.nc", "t.x1_SAMOC_flux.169003.nc", + "t.x1_SAMOC_flux.169004.nc", "t.x1_SAMOC_flux.169005.nc"]] + + ["SWASH_data/" + fn for fn in ["field_0065532.nc", "field_0065537.nc", "field_0065542.nc", "field_0065548.nc", "field_0065552.nc", "field_0065557.nc"]] + ["WOA_data/" + fn for fn in ["woa18_decav_t%.2d_04.nc" % m for m in range(1, 13)]]) diff --git a/parcels/tools/__init__.py b/parcels/tools/__init__.py index 04da2d8d40..26f2f8f792 100644 --- a/parcels/tools/__init__.py +++ b/parcels/tools/__init__.py @@ -3,3 +3,4 @@ from .interpolation_utils import * # noqa from .loggers import * # noqa from .timer import * # noqa +from .performance_logger import * # noga \ No newline at end of file diff --git a/parcels/tools/error.py b/parcels/tools/error.py index ca303fc9d0..4345cf87d0 100644 --- a/parcels/tools/error.py +++ b/parcels/tools/error.py @@ -11,11 +11,26 @@ class ErrorCode(object): Evaluate = 1 Repeat = 2 Delete = 3 - Error = 4 - ErrorInterpolation = 41 - ErrorOutOfBounds = 5 - ErrorThroughSurface = 51 - ErrorTimeExtrapolation = 6 + StopExecution = 4 + Error = 5 + ErrorInterpolation = 51 + ErrorOutOfBounds = 6 + ErrorThroughSurface = 61 + ErrorTimeExtrapolation = 7 + + @classmethod + def toString(cls, code): + names = {cls.Success: 'Success', + cls.Evaluate: 'Evaluate', + cls.Repeat: 'Repeat', + cls.Delete: 'Delete', + cls.StopExecution: 'Stop', + cls.Error: 'Error', + cls.ErrorInterpolation: 'InterpolationError', + cls.ErrorOutOfBounds: 'OutOfBoundsError', + cls.ErrorThroughSurface: 'ThroughSurfaceError', + cls.ErrorTimeExtrapolation: 'TimeExtrapolationError'} + return names[code] class FieldSamplingError(RuntimeError): diff --git a/parcels/tools/performance_logger.py b/parcels/tools/performance_logger.py new file mode 100644 index 0000000000..cd13634545 --- /dev/null +++ b/parcels/tools/performance_logger.py @@ -0,0 +1,321 @@ +import time as time_module +import numpy as np + +try: + from mpi4py import MPI +except: + MPI = None + +from threading import Thread +from threading import Event +from time import sleep + +class TimingLog(): + stime = 0 + etime = 0 + mtime = 0 + _samples = [] + _times_steps = [] + _iter = 0 + + def __init__(self): + self.stime = 0 + self.etime = 0 + self.mtime = 0 + self._samples = [] + self._times_steps = [] + self._iter = 0 + + @property + def timing(self): + return self._times_steps + + @property + def samples(self): + return self._samples + + def __len__(self): + return len(self._samples) + + def get_values(self): + return self._times_steps + + def get_value(self, index): + N = len(self._times_steps) + result = 0 + if N > 0: + result = self._times_steps[min(max(index, 0), N - 1)] + return result + + def start_timing(self): + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank == 0: + # self.stime = MPI.Wtime() + # self.stime = time_module.perf_counter() + self.stime = time_module.process_time() + else: + self.stime = time_module.process_time() + + def stop_timing(self): + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank == 0: + # self.etime = MPI.Wtime() + # self.etime = time_module.perf_counter() + self.etime = time_module.process_time() + else: + self.etime = time_module.process_time() + + def accumulate_timing(self): + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank == 0: + self.mtime += (self.etime-self.stime) + else: + self.mtime = 0 + else: + self.mtime += (self.etime-self.stime) + + def advance_iteration(self): + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank == 0: + self._times_steps.append(self.mtime) + self._samples.append(self._iter) + self._iter += 1 + self.mtime = 0 + else: + self._times_steps.append(self.mtime) + self._samples.append(self._iter) + self._iter += 1 + self.mtime = 0 + + def add_aux_measure(self, value): + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank == 0: + self.mtime += value + else: + self.mtime += 0 + else: + self.mtime += value + + def sum(self): + result = 0 + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank == 0: + result = np.array(self._times_steps).sum() + else: + result = np.array(self._times_steps).sum() + return result + + def reset(self): + if self._times_steps is not None and len(self._times_steps) > 0: + del self._times_steps[:] + if self._samples is not None and len(self._samples) > 0: + del self._samples[:] + self.stime = 0 + self.etime = 0 + self.mtime = 0 + self._samples = [] + self._times_steps = [] + self._iter = 0 + + +class ParamLogging(): + _samples = [] + _params = [] + _iter = 0 + + def __init__(self): + self._samples = [] + self._params = [] + self._iter = 0 + + @property + def samples(self): + return self._samples + + @property + def params(self): + return self._params + + def get_params(self): + return self._params + + def get_param(self, index): + N = len(self._params) + result = 0 + if N > 0: + result = self._params[min(max(index, 0), N-1)] + return result + + def __len__(self): + return len(self._samples) + + def advance_iteration(self, param): + if MPI: + # mpi_comm = MPI.COMM_WORLD + # mpi_rank = mpi_comm.Get_rank() + + self._params.append(param) + self._samples.append(self._iter) + self._iter += 1 + # if mpi_rank == 0: + # self._params.append(param) + # self._samples.append(self._iter) + # self._iter += 1 + else: + self._params.append(param) + self._samples.append(self._iter) + self._iter += 1 + +class Asynchronous_ParamLogging(): + _samples = [] + _params = [] + _iter = 0 + _measure_func = None + _measure_start_value = None # for differential measurements + _measure_partial_values = [] + _measure_interval = 0.25 # 250 ms + _event = None + _thread = None + differential_measurement = False + + def __init__(self): + self._samples = [] + self._params = [] + self._measure_partial_values = [] + self._iter = 0 + self._measure_func = None + self._measure_start_value = None + self._measure_interval = 0.25 # 250 ms + self._event = None + self._thread = None + self.differential_measurement = False + + def __del__(self): + del self._samples[:] + del self._params[:] + del self._measure_partial_values[:] + + @property + def samples(self): + return self._samples + + @property + def params(self): + return self._params + + @property + def measure_func(self): + return self._measure_func + + @measure_func.setter + def measure_func(self, function): + self._measure_func = function + + @property + def measure_interval(self): + return self._measure_interval + + @measure_interval.setter + def measure_interval(self, interval): + """ + Set measure interval in seconds + :param interval: interval in seconds (fractional possible) + :return: None + """ + self._measure_interval = interval + + @property + def measure_start_value(self): + return self._measure_start_value + + @measure_start_value.setter + def measure_start_value(self, value): + self._measure_start_value = value + + def async_run(self): + if self.differential_measurement: + self.async_run_diff_measurement() + else: + pass + + def async_run_diff_measurement(self): + if self._measure_start_value is None: + self._measure_start_value = self._measure_func() + self._measure_partial_values.append(0) + while not self._event.is_set(): + self._measure_partial_values.append( self._measure_func()-self._measure_start_value ) + sleep(self._measure_interval) + + def async_run_measurement(self): + while not self._event.is_set(): + self._measure_partial_values.append( self.measure_func() ) + sleep(self.measure_interval) + + def start_partial_measurement(self): + assert self._measure_func is not None, "Measurement function is None - invalid. Exiting ..." + assert self._thread is None, "Measurement already running - double-start invalid. Exiting ..." + if len(self._measure_partial_values) > 0: + del self._measure_partial_values[:] + self._measure_partial_values = [] + self._event = Event() + self._thread = Thread(target=self.async_run_measurement) + self._thread.start() + + def stop_partial_measurement(self): + """ + function to stop the measurement. The function also internally advances the iteration with the mean (or max) + of the measured partial values. + :return: None + """ + self._event.set() + self._thread.join() + sleep(self._measure_interval) + del self._thread + self._thread = None + self._measure_start_value = None + # param_partial_mean = np.array(self._measure_partial_values).mean() + param_partial_mean = np.array(self._measure_partial_values).max() + self.advance_iteration(param_partial_mean) + + def get_params(self): + return self._params + + def get_param(self, index): + N = len(self._params) + result = 0 + if N > 0: + result = self._params[min(max(index, 0), N-1)] + return result + + def __len__(self): + return len(self._samples) + + def advance_iteration(self, param): + if MPI: + # mpi_comm = MPI.COMM_WORLD + # mpi_rank = mpi_comm.Get_rank() + + self._params.append(param) + self._samples.append(self._iter) + self._iter += 1 + # if mpi_rank == 0: + # self._params.append(param) + # self._samples.append(self._iter) + # self._iter += 1 + else: + self._params.append(param) + self._samples.append(self._iter) + self._iter += 1 + diff --git a/parcels/tools/perlin2d.py b/parcels/tools/perlin2d.py new file mode 100644 index 0000000000..dc7afa5fa7 --- /dev/null +++ b/parcels/tools/perlin2d.py @@ -0,0 +1,77 @@ +import numpy as np +from numpy.random import MT19937 +from numpy.random import RandomState, SeedSequence +from scipy import ndimage +from time import time + +def generate_perlin_noise_2d(shape, res): + def f(t): + return 6*t**5 - 15*t**4 + 10*t**3 + + delta = (res[0] / shape[0], res[1] / shape[1]) + d = (shape[0] // res[0], shape[1] // res[1]) + grid = np.mgrid[0:res[0]:delta[0],0:res[1]:delta[1]].transpose(1, 2, 0) % 1 + # Gradients + angles = 2*np.pi*np.random.rand(res[0]+1, res[1]+1) + gradients = np.dstack((np.cos(angles), np.sin(angles))) + g00 = gradients[0:-1,0:-1].repeat(d[0], 0).repeat(d[1], 1) + g10 = gradients[1: ,0:-1].repeat(d[0], 0).repeat(d[1], 1) + g01 = gradients[0:-1,1: ].repeat(d[0], 0).repeat(d[1], 1) + g11 = gradients[1: ,1: ].repeat(d[0], 0).repeat(d[1], 1) + # Ramps + n00 = np.sum(np.dstack((grid[:,:,0] , grid[:,:,1] )) * g00, 2) + n10 = np.sum(np.dstack((grid[:,:,0]-1, grid[:,:,1] )) * g10, 2) + n01 = np.sum(np.dstack((grid[:,:,0] , grid[:,:,1]-1)) * g01, 2) + n11 = np.sum(np.dstack((grid[:,:,0]-1, grid[:,:,1]-1)) * g11, 2) + # Interpolation + t = f(grid) + n0 = n00*(1-t[:,:,0]) + t[:,:,0]*n10 + n1 = n01*(1-t[:,:,0]) + t[:,:,0]*n11 + return np.sqrt(2)*((1-t[:,:,1])*n0 + t[:,:,1]*n1) + +def generate_fractal_noise_2d(shape, res, octaves=1, persistence=0.5): + rs = RandomState( MT19937(SeedSequence(int(round(time() * 1000)))) ) + noise = np.zeros(shape) + frequency = 1 + amplitude = 1 + for m_i in range(octaves): + noise += amplitude * generate_perlin_noise_2d(shape, (frequency*res[0], frequency*res[1])) + frequency *= 2 + amplitude *= persistence + return noise + +def generate_fractal_noise_temporal2d(shape, tsteps, res, octaves=1, persistence=0.5, max_shift=((-1, 1), (-1, 1))): + rs = RandomState( MT19937(SeedSequence(int(round(time() * 1000)))) ) + noise = np.zeros(shape) + frequency = 1 + amplitude = 1 + for m_i in range(octaves): + noise += amplitude * generate_perlin_noise_2d(shape, (frequency*res[0], frequency*res[1])) + frequency *= 2 + amplitude *= persistence + + ishape = (tsteps, )+shape + result = np.zeros(ishape) + timage = np.zeros(noise.shape) + for i in range(0, tsteps): + result[i,:,:] = noise + sx = np.random.randint(max_shift[0][0], max_shift[0][1], dtype=np.int32) + sy = np.random.randint(max_shift[1][0], max_shift[1][1], dtype=np.int32) + ndimage.shift(noise, (sx, sy), timage, order=3, mode='mirror') + noise = timage + return result + +if __name__ == '__main__': + import matplotlib.pyplot as plt + + np.random.seed(0) + noise = generate_perlin_noise_2d((256, 256), (8, 8)) + plt.imshow(noise, cmap='gray', interpolation='lanczos') + plt.colorbar() + + np.random.seed(0) + noise = generate_fractal_noise_2d((256, 256), (8, 8), 5) + plt.figure() + plt.imshow(noise, cmap='gray', interpolation='lanczos') + plt.colorbar() + plt.show() diff --git a/parcels/tools/perlin3d.py b/parcels/tools/perlin3d.py new file mode 100644 index 0000000000..7424e97ff8 --- /dev/null +++ b/parcels/tools/perlin3d.py @@ -0,0 +1,89 @@ +import numpy as np +from numpy.random import MT19937 +from numpy.random import RandomState, SeedSequence +from scipy import ndimage +from time import time + +def generate_perlin_noise_3d(shape, res): + def f(t): + return 6*t**5 - 15*t**4 + 10*t**3 + + delta = (res[0] / shape[0], res[1] / shape[1], res[2] / shape[2]) + d = (shape[0] // res[0], shape[1] // res[1], shape[2] // res[2]) + grid = np.mgrid[0:res[0]:delta[0],0:res[1]:delta[1],0:res[2]:delta[2]] + grid = grid.transpose(1, 2, 3, 0) % 1 + # Gradients + theta = 2*np.pi*np.random.rand(res[0]+1, res[1]+1, res[2]+1) + phi = 2*np.pi*np.random.rand(res[0]+1, res[1]+1, res[2]+1) + gradients = np.stack((np.sin(phi)*np.cos(theta), np.sin(phi)*np.sin(theta), np.cos(phi)), axis=3) + g000 = gradients[0:-1,0:-1,0:-1].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g100 = gradients[1: ,0:-1,0:-1].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g010 = gradients[0:-1,1: ,0:-1].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g110 = gradients[1: ,1: ,0:-1].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g001 = gradients[0:-1,0:-1,1: ].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g101 = gradients[1: ,0:-1,1: ].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g011 = gradients[0:-1,1: ,1: ].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g111 = gradients[1: ,1: ,1: ].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + # Ramps + n000 = np.sum(np.stack((grid[:,:,:,0] , grid[:,:,:,1] , grid[:,:,:,2] ), axis=3) * g000, 3) + n100 = np.sum(np.stack((grid[:,:,:,0]-1, grid[:,:,:,1] , grid[:,:,:,2] ), axis=3) * g100, 3) + n010 = np.sum(np.stack((grid[:,:,:,0] , grid[:,:,:,1]-1, grid[:,:,:,2] ), axis=3) * g010, 3) + n110 = np.sum(np.stack((grid[:,:,:,0]-1, grid[:,:,:,1]-1, grid[:,:,:,2] ), axis=3) * g110, 3) + n001 = np.sum(np.stack((grid[:,:,:,0] , grid[:,:,:,1] , grid[:,:,:,2]-1), axis=3) * g001, 3) + n101 = np.sum(np.stack((grid[:,:,:,0]-1, grid[:,:,:,1] , grid[:,:,:,2]-1), axis=3) * g101, 3) + n011 = np.sum(np.stack((grid[:,:,:,0] , grid[:,:,:,1]-1, grid[:,:,:,2]-1), axis=3) * g011, 3) + n111 = np.sum(np.stack((grid[:,:,:,0]-1, grid[:,:,:,1]-1, grid[:,:,:,2]-1), axis=3) * g111, 3) + # Interpolation + t = f(grid) + n00 = n000*(1-t[:,:,:,0]) + t[:,:,:,0]*n100 + n10 = n010*(1-t[:,:,:,0]) + t[:,:,:,0]*n110 + n01 = n001*(1-t[:,:,:,0]) + t[:,:,:,0]*n101 + n11 = n011*(1-t[:,:,:,0]) + t[:,:,:,0]*n111 + n0 = (1-t[:,:,:,1])*n00 + t[:,:,:,1]*n10 + n1 = (1-t[:,:,:,1])*n01 + t[:,:,:,1]*n11 + return ((1-t[:,:,:,2])*n0 + t[:,:,:,2]*n1) + +def generate_fractal_noise_3d(shape, res, octaves=1, persistence=0.5): + rs = RandomState( MT19937(SeedSequence(int(round(time() * 1000)))) ) + noise = np.zeros(shape) + frequency = 1 + amplitude = 1 + for _ in range(octaves): + noise += amplitude * generate_perlin_noise_3d(shape, (frequency*res[0], frequency*res[1], frequency*res[2])) + frequency *= 2 + amplitude *= persistence + return noise + +def generate_fractal_noise_temporal3d(shape, tsteps, res, octaves=1, persistence=0.5, max_shift=(1, 1, 1)): + rs = RandomState( MT19937(SeedSequence(int(round(time() * 1000)))) ) + noise = np.zeros(shape) + frequency = 1 + amplitude = 1 + for _ in range(octaves): + noise += amplitude * generate_perlin_noise_3d(shape, (frequency*res[0], frequency*res[1], frequency*res[2])) + frequency *= 2 + amplitude *= persistence + + ishape = (tsteps, )+shape + result = np.zeros(ishape) + timage = np.zeros(noise.shape) + for i in range(0, tsteps): + result[i,:,:] = noise + sx = np.random.randint(-max_shift[0], max_shift[0], dtype=np.int32) + sy = np.random.randint(-max_shift[1], max_shift[1], dtype=np.int32) + sz = np.random.randint(-max_shift[2], max_shift[2], dtype=np.int32) + ndimage.shift(noise, (sx, sy, sz), timage, order=3, mode='mirror') + noise = timage + return result + +if __name__ == '__main__': + import matplotlib.pyplot as plt + import matplotlib.animation as animation + + np.random.seed(0) + noise = generate_fractal_noise_3d((32, 256, 256), (1, 4, 4), 4) + + fig = plt.figure() + images = [[plt.imshow(layer, cmap='gray', interpolation='lanczos', animated=True)] for layer in noise] + animation = animation.ArtistAnimation(fig, images, interval=50, blit=True) + plt.show() diff --git a/performance/benchmark_CMEMS.py b/performance/benchmark_CMEMS.py new file mode 100644 index 0000000000..6959eed652 --- /dev/null +++ b/performance/benchmark_CMEMS.py @@ -0,0 +1,375 @@ +""" +Author: Dr. Christian Kehl +Date: 11-02-2020 +""" + +from parcels import AdvectionEE, AdvectionRK45, AdvectionRK4_3D +from parcels import FieldSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, ErrorCode +from parcels.particleset_benchmark import ParticleSet_Benchmark as ParticleSet +from parcels import rng as random +from parcels.field import VectorField, NestedField, SummedField +# from parcels import plotTrajectoriesFile_loadedField +from datetime import timedelta as delta +from datetime import datetime +from argparse import ArgumentParser +import numpy as np +import fnmatch +# import dask as da +# import dask.array as daArray +from glob import glob +import time as ostime +import math +import matplotlib.pyplot as plt +import sys +import os +import psutil +import gc +try: + from mpi4py import MPI +except: + MPI = None + +with_GC = False + +import warnings +import xarray as xr +warnings.simplefilter("ignore", category=xr.SerializationWarning) + +ptype = {'scipy': ScipyParticle, 'jit': JITParticle} +method = {'RK4': AdvectionRK4, 'EE': AdvectionEE, 'RK45': AdvectionRK45} +global_t_0 = 0 +odir = "" + + +def create_CMEMS_fieldset(datahead, periodic_wrap): + ddir = os.path.join(datahead, "CMEMS/GLOBAL_REANALYSIS_PHY_001_030/") + files = sorted(glob(ddir+"mercatorglorys12v1_gl12_mean_201607*.nc")) + variables = {'U': 'uo', 'V': 'vo'} + dimensions = {'lon': 'longitude', 'lat': 'latitude', 'time': 'time'} + # chs = False + chs = 'auto' + if periodic_wrap: + return FieldSet.from_netcdf(files, variables, dimensions, field_chunksize=chs, time_periodic=delta(days=31)) + else: + return FieldSet.from_netcdf(files, variables, dimensions, field_chunksize=chs, allow_time_extrapolation=True) + + +class AgeParticle_JIT(JITParticle): + age = Variable('age', dtype=np.float64, initial=0.0) + life_expectancy = Variable('life_expectancy', dtype=np.float64, initial=np.finfo(np.float64).max) + initialized_dynamic = Variable('initialized_dynamic', dtype=np.int32, initial=0) + +class AgeParticle_SciPy(ScipyParticle): + age = Variable('age', dtype=np.float64, initial=0.0) + life_expectancy = Variable('life_expectancy', dtype=np.float64, initial=np.finfo(np.float64).max) + initialized_dynamic = Variable('initialized_dynamic', dtype=np.int32, initial=0) + +age_ptype = {'scipy': AgeParticle_SciPy, 'jit': AgeParticle_JIT} + +def periodicBC(particle, fieldSet, time): + if particle.lon > 180.0: + particle.lon -= 360.0 + if particle.lon < -180.0: + particle.lon += 360.0 + particle.lat = min(particle.lat, 90.0) + particle.lat = max(particle.lat, -80.0) + # if particle.lat > 90.0: + # particle.lat -= 170.0 + # if particle.lat < -80.0: + # particle.lat += 170.0 + +def initialize(particle, fieldset, time): + if particle.initialized_dynamic < 1: + np_scaler = math.sqrt(3.0 / 2.0) + particle.life_expectancy = time + random.uniform(.0, (fieldset.life_expectancy-time) * 2.0 / np_scaler) + particle.initialized_dynamic = 1 + +def Age(particle, fieldset, time): + if particle.state == ErrorCode.Evaluate: + particle.age = particle.age + math.fabs(particle.dt) + if particle.age > particle.life_expectancy: + particle.delete() + +def DeleteParticle(particle, fieldset, time): + particle.delete() + +def RenewParticle(particle, fieldset, time): + particle.lat = random.random() * 360.0 -180.0 + particle.lon = random.random() * 170.0 -80.0 + +def perIterGC(): + gc.collect() + +if __name__=='__main__': + parser = ArgumentParser(description="Example of particle advection using in-memory stommel test case") + parser.add_argument("-i", "--imageFileName", dest="imageFileName", type=str, default="mpiChunking_plot_MPI.png", help="image file name of the plot") + parser.add_argument("-b", "--backwards", dest="backwards", action='store_true', default=False, help="enable/disable running the simulation backwards") + parser.add_argument("-p", "--periodic", dest="periodic", action='store_true', default=False, help="enable/disable periodic wrapping (else: extrapolation)") + parser.add_argument("-r", "--release", dest="release", action='store_true', default=False, help="continuously add particles via repeatdt (default: False)") + parser.add_argument("-rt", "--releasetime", dest="repeatdt", type=int, default=720, help="repeating release rate of added particles in Minutes (default: 720min = 12h)") + parser.add_argument("-a", "--aging", dest="aging", action='store_true', default=False, help="Removed aging particles dynamically (default: False)") + parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=int, default=1, help="runtime in days (default: 1)") + parser.add_argument("-x", "--xarray", dest="use_xarray", action='store_true', default=False, help="use xarray as data backend") + parser.add_argument("-w", "--writeout", dest="write_out", action='store_true', default=False, help="write data in outfile") + parser.add_argument("-d", "--delParticle", dest="delete_particle", action='store_true', default=False, help="switch to delete a particle (True) or reset a particle (default: False).") + parser.add_argument("-A", "--animate", dest="animate", action='store_true', default=False, help="animate the particle trajectories during the run or not (default: False).") + parser.add_argument("-V", "--visualize", dest="visualize", action='store_true', default=False, help="Visualize particle trajectories at the end (default: False). Requires -w in addition to take effect.") + parser.add_argument("-N", "--n_particles", dest="nparticles", type=str, default="2**6", help="number of particles to generate and advect (default: 2e6)") + parser.add_argument("-sN", "--start_n_particles", dest="start_nparticles", type=str, default="96", help="(optional) number of particles generated per release cycle (if --rt is set) (default: 96)") + parser.add_argument("-m", "--mode", dest="compute_mode", choices=['jit','scipy'], default="jit", help="computation mode = [JIT, SciPp]") + parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + args = parser.parse_args() + + imageFileName=args.imageFileName + periodicFlag=args.periodic + backwardSimulation = args.backwards + repeatdtFlag=args.release + repeatRateMinutes=args.repeatdt + time_in_days = args.time_in_days + use_xarray = args.use_xarray + agingParticles = args.aging + with_GC = args.useGC + + Nparticle = int(float(eval(args.nparticles))) + target_N = Nparticle + addParticleN = 1 + np_scaler = 3.0 / 2.0 + cycle_scaler = 3.0 / 2.0 + start_N_particles = int(float(eval(args.start_nparticles))) + if MPI: + mpi_comm = MPI.COMM_WORLD + if mpi_comm.Get_rank() == 0: + if agingParticles and not repeatdtFlag: + sys.stdout.write("N: {} ( {} )\n".format(Nparticle, int(Nparticle * ((3.0 / 2.0)**2.0)))) + else: + sys.stdout.write("N: {}\n".format(Nparticle)) + else: + if agingParticles and not repeatdtFlag: + sys.stdout.write("N: {} ( {} )\n".format(Nparticle, int(Nparticle * ((3.0 / 2.0)**2.0)))) + else: + sys.stdout.write("N: {}\n".format(Nparticle)) + + dt_minutes = 60 + nowtime = datetime.now() + random.seed(nowtime.microsecond) + + branch = "benchmarking" + computer_env = "local/unspecified" + scenario = "CMEMS" + headdir = "" + odir = "" + dirread_pal = "" + datahead = "" + dirread_top = "" + dirread_top_bgc = "" + if os.uname()[1] in ['science-bs35', 'science-bs36']: # Gemini + # headdir = "/scratch/{}/experiments/palaeo-parcels".format(os.environ['USER']) + headdir = "/scratch/{}/experiments".format("ckehl") + odir = headdir + datahead = "/data/oceanparcels/input_data" + dirread_top = os.path.join(datahead, 'CMEMS/GLOBAL_REANALYSIS_PHY_001_030/') + computer_env = "Gemini" + # elif fnmatch.fnmatchcase(os.uname()[1], "int?.*"): # Cartesius + elif fnmatch.fnmatchcase(os.uname()[1], "*.bullx*"): # Cartesius + CARTESIUS_SCRATCH_USERNAME = 'ckehluu' + headdir = "/scratch/shared/{}/experiments".format(CARTESIUS_SCRATCH_USERNAME) + odir = headdir + datahead = "/projects/0/topios/hydrodynamic_data" + dirread_top = os.path.join(datahead, 'CMEMS/GLOBAL_REANALYSIS_PHY_001_030/') + computer_env = "Cartesius" + else: + headdir = "/var/scratch/experiments" + odir = headdir + dirread_pal = headdir + datahead = "/data" + dirread_top = os.path.join(datahead, 'CMEMS/GLOBAL_REANALYSIS_PHY_001_030/') + print("running {} on {} (uname: {}) - branch '{}' - (target) N: {} - argv: {}".format(scenario, computer_env, os.uname()[1], branch, target_N, sys.argv[1:])) + + if os.path.sep in imageFileName: + head_dir = os.path.dirname(imageFileName) + if head_dir[0] == os.path.sep: + odir = head_dir + else: + odir = os.path.join(odir, head_dir) + imageFileName = os.path.split(imageFileName)[1] + + func_time = [] + mem_used_GB = [] + + np.random.seed(nowtime.microsecond) + fieldset = create_CMEMS_fieldset(datahead, periodic_wrap=periodicFlag) + + if args.compute_mode is 'scipy': + Nparticle = 2**10 + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + #global_t_0 = MPI.Wtime() + global_t_0 = ostime.process_time() + else: + #global_t_0 = ostime.time() + global_t_0 = ostime.process_time() + + simStart = None + for f in fieldset.get_fields(): + if type(f) in [VectorField, NestedField, SummedField]: # or not f.grid.defer_load + continue + else: + if backwardSimulation: + simStart=f.grid.time_full[-1] + else: + simStart = f.grid.time_full[0] + break + + if agingParticles: + if not repeatdtFlag: + Nparticle = int(Nparticle * np_scaler) + fieldset.add_constant('life_expectancy', delta(days=time_in_days).total_seconds()) + if repeatdtFlag: + addParticleN = Nparticle/2.0 + refresh_cycle = (delta(days=time_in_days).total_seconds() / (addParticleN/start_N_particles)) / cycle_scaler + if agingParticles: + refresh_cycle /= cycle_scaler + repeatRateMinutes = int(refresh_cycle/60.0) if repeatRateMinutes == 720 else repeatRateMinutes + + if backwardSimulation: + # ==== backward simulation ==== # + if agingParticles: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * 360.0 -180.0, lat=np.random.rand(start_N_particles, 1) * 160.0 - 80.0, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * 360.0 -180.0, lat=np.random.rand(int(Nparticle/2.0), 1) * 160.0 - 80.0, time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * 360.0 -180.0, lat=np.random.rand(Nparticle, 1) * 160.0 - 80.0, time=simStart) + else: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * 360.0 -180.0, lat=np.random.rand(start_N_particles, 1) * 160.0 - 80.0, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * 360.0 -180.0, lat=np.random.rand(int(Nparticle/2.0), 1) * 160.0 - 80.0, time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * 360.0 -180.0, lat=np.random.rand(Nparticle, 1) * 160.0 - 80.0, time=simStart) + else: + # ==== forward simulation ==== # + if agingParticles: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * 360.0 -180.0, lat=np.random.rand(start_N_particles, 1) * 160.0 - 80.0, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * 360.0 -180.0, lat=np.random.rand(int(Nparticle/2.0), 1) * 160.0 - 80.0, time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * 360.0 -180.0, lat=np.random.rand(Nparticle, 1) * 160.0 - 80.0, time=simStart) + else: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * 360.0 -180.0, lat=np.random.rand(start_N_particles, 1) * 160.0 - 80.0, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * 360.0 -180.0, lat=np.random.rand(int(Nparticle/2.0), 1) * 160.0 - 80.0, time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * 360.0 -180.0, lat=np.random.rand(Nparticle, 1) * 160.0 - 80.0, time=simStart) + + + output_file = None + out_fname = "benchmark_CMEMS" + if args.write_out: + if MPI and (MPI.COMM_WORLD.Get_size()>1): + out_fname += "_MPI" + else: + out_fname += "_noMPI" + if periodicFlag: + out_fname += "_p" + out_fname += "_n"+str(Nparticle) + if backwardSimulation: + out_fname += "_bwd" + else: + out_fname += "_fwd" + if repeatdtFlag: + out_fname += "_add" + if agingParticles: + out_fname += "_age" + output_file = pset.ParticleFile(name=os.path.join(odir,out_fname+".nc"), outputdt=delta(hours=24)) + delete_func = RenewParticle + if args.delete_particle: + delete_func=DeleteParticle + postProcessFuncs = [] + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + #starttime = MPI.Wtime() + starttime = ostime.process_time() + else: + #starttime = ostime.time() + starttime = ostime.process_time() + kernels = pset.Kernel(AdvectionRK4,delete_cfiles=True) + kernels += pset.Kernel(periodicBC, delete_cfiles=True) + if agingParticles: + kernels += pset.Kernel(initialize, delete_cfiles=True) + kernels += pset.Kernel(Age, delete_cfiles=True) + if with_GC: + postProcessFuncs.append(perIterGC) + if backwardSimulation: + # ==== backward simulation ==== # + if args.animate: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=-dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12), moviedt=delta(hours=6), movie_background_field=fieldset.U) + else: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=-dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12)) + else: + # ==== forward simulation ==== # + if args.animate: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12), moviedt=delta(hours=6), movie_background_field=fieldset.U) + else: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12)) + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + #endtime = MPI.Wtime() + endtime = ostime.process_time() + else: + #endtime = ostime.time() + endtime = ostime.process_time() + + if args.write_out: + output_file.close() + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_comm.Barrier() + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart - 1) + Npart = mpi_comm.reduce(Npart, op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + if size_Npart>0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime-starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time*1000.0)) + else: + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart-1) + if size_Npart > 0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime - starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time * 1000.0)) + + # if args.write_out: + # output_file.close() + # if args.visualize: + # if MPI: + # mpi_comm = MPI.COMM_WORLD + # if mpi_comm.Get_rank() == 0: + # plotTrajectoriesFile_loadedField(os.path.join(odir, out_fname+".nc"), tracerfield=fieldset.U) + # else: + # plotTrajectoriesFile_loadedField(os.path.join(odir, out_fname+".nc"),tracerfield=fieldset.U) + + if MPI: + mpi_comm = MPI.COMM_WORLD + # mpi_comm.Barrier() + Nparticles = mpi_comm.reduce(np.array(pset.nparticle_log.get_params()), op=MPI.SUM, root=0) + Nmem = mpi_comm.reduce(np.array(pset.mem_log.get_params()), op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + pset.plot_and_log(memory_used=Nmem, nparticles=Nparticles, target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 150]) + else: + pset.plot_and_log(target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 150]) diff --git a/performance/benchmark_bickleyjet.py b/performance/benchmark_bickleyjet.py new file mode 100644 index 0000000000..7d04d36d38 --- /dev/null +++ b/performance/benchmark_bickleyjet.py @@ -0,0 +1,426 @@ +""" +Author: Dr. Christian Kehl +Date: 11-02-2020 +""" + +from parcels import AdvectionEE, AdvectionRK45, AdvectionRK4 +from parcels import FieldSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, ErrorCode +from parcels.particleset_benchmark import ParticleSet_Benchmark as ParticleSet +from parcels.particleset import ParticleSet as DryParticleSet +from parcels.field import Field, VectorField, NestedField, SummedField +# from parcels import plotTrajectoriesFile_loadedField +from parcels import rng as random +from datetime import timedelta as delta +import math +from argparse import ArgumentParser +import datetime +import numpy as np +import xarray as xr +import fnmatch +import sys +import gc +import os +import time as ostime +# import matplotlib.pyplot as plt +from parcels.tools import perlin3d +from parcels.tools import perlin2d + +try: + from mpi4py import MPI +except: + MPI = None +with_GC = False + +pset = None +ptype = {'scipy': ScipyParticle, 'jit': JITParticle} +method = {'RK4': AdvectionRK4, 'EE': AdvectionEE, 'RK45': AdvectionRK45} +global_t_0 = 0 +Nparticle = int(math.pow(2,10)) # equals to Nparticle = 1024 +#Nparticle = int(math.pow(2,11)) # equals to Nparticle = 2048 +#Nparticle = int(math.pow(2,12)) # equals to Nparticle = 4096 +#Nparticle = int(math.pow(2,13)) # equals to Nparticle = 8192 +#Nparticle = int(math.pow(2,14)) # equals to Nparticle = 16384 +#Nparticle = int(math.pow(2,15)) # equals to Nparticle = 32768 +#Nparticle = int(math.pow(2,16)) # equals to Nparticle = 65536 +#Nparticle = int(math.pow(2,17)) # equals to Nparticle = 131072 +#Nparticle = int(math.pow(2,18)) # equals to Nparticle = 262144 +#Nparticle = int(math.pow(2,19)) # equals to Nparticle = 524288 + +#a = 1000 * 1e3 +#b = 1000 * 1e3 +#scalefac = 2.0 +#tsteps = 61 +#tscale = 6 + +a = 10 * 1e3 # [a in km -> 10e3] # is going to be overwritten +b = 10 * 1e3 # [b in km -> 10e3] # is going to be overwritten +tsteps = 61 # in steps +tstepsize = 12.0 # unitary +tscale = 12.0*60.0*60.0 # in seconds +# time[days] = (tsteps * tstepsize) * tscale +# jet_rotation_speed = 14.0*24.0*60.0*60.0 # assume 1 rotation every 2 weeks +sx = 1718.9 +sy = 3200.0 +# scalefac = 2.0 * 6.0 # scaling the advection speeds to 12 m/s +scalefac = (40.0 / (1000.0/60.0)) # 40 km/h +scalefac /= 1000.0 + +# we need to modify the kernel.execute / pset.execute so that it returns from the JIT +# in a given time WITHOUT writing to disk via outfie => introduce a pyloop_dt + +def DeleteParticle(particle, fieldset, time): + particle.delete() + +def RenewParticle(particle, fieldset, time): + particle.lat = np.random.rand() * a + particle.lon = np.random.rand() * (-b) + (b/2.0) + +def perIterGC(): + gc.collect() + +def bickleyjet_from_numpy(xdim=540, ydim=320, periodic_wrap=False, write_out=False): + """Bickley Jet Field as implemented in Hadjighasem et al 2017, 10.1063/1.4982720""" + U0 = 0.06266 + r0 = sx + L = r0 / 3.599435028 # 1770. + k1 = 2 * 1 / r0 + k2 = 2 * 2 / r0 + k3 = 2 * 3 / r0 + eps1 = 0.075 + eps2 = 0.4 + eps3 = 0.3 + c3 = 0.461 * U0 + c2 = 0.205 * U0 + c1 = c3 + ((np.sqrt(5)-1)/2.) * (k2/k1) * (c2 - c3) + + global a, b + a, b = np.pi*r0, sy # domain size + lon = np.linspace(0, a, xdim, dtype=np.float32) + lonrange = lon.max()-lon.min() + sys.stdout.write("lon field: {}\n".format(lon.size)) + lat = np.linspace(-b*0.5, b*0.5, ydim, dtype=np.float32) + latrange = lat.max() - lat.min() + sys.stdout.write("lat field: {}\n".format(lat.size)) + totime = (tsteps * tstepsize) * tscale + times = np.linspace(0., totime, tsteps, dtype=np.float64) + sys.stdout.write("time field: {}\n".format(times.size)) + dx, dy = lon[2]-lon[1], lat[2]-lat[1] + + U = np.zeros((times.size, lat.size, lon.size), dtype=np.float32) + V = np.zeros((times.size, lat.size, lon.size), dtype=np.float32) + + for t in range(len(times)): + time_f = times[t] + f1 = eps1 * np.exp(-1j * k1 * c1 * time_f) + f2 = eps2 * np.exp(-1j * k2 * c2 * time_f) + f3 = eps3 * np.exp(-1j * k3 * c3 * time_f) + for j in range(lat.size): + for i in range(lon.size): + x1 = lon[i]-dx/2.0 + x2 = lat[j]-dy/2.0 + F1 = f1 * np.exp(1j * k1 * x1) + F2 = f2 * np.exp(1j * k2 * x1) + F3 = f3 * np.exp(1j * k3 * x1) + G = np.real(np.sum([F1, F2, F3])) + G_x = np.real(np.sum([1j * k1 * F1, 1j * k2 * F2, 1j * k3 * F3])) + U[t, j, i] = U0 / (np.cosh(x2/L)**2) + 2 * U0 * np.sinh(x2/L) / (np.cosh(x2/L)**3) * G + V[t, j, i] = U0 * L * (1./np.cosh(x2/L))**2 * G_x + + U *= scalefac + # U = np.transpose(U, (0, 2, 1)) + V *= scalefac + # V = np.transpose(V, (0, 2, 1)) + + + data = {'U': U, 'V': V} + dimensions = {'time': times, 'lon': lon, 'lat': lat} + fieldset = None + if periodic_wrap: + fieldset = FieldSet.from_data(data, dimensions, mesh='flat', transpose=False, time_periodic=delta(days=1)) + else: + fieldset = FieldSet.from_data(data, dimensions, mesh='flat', transpose=False, allow_time_extrapolation=True) + if write_out: + fieldset.write(filename=write_out) + return fieldset + + +class AgeParticle_JIT(JITParticle): + age = Variable('age', dtype=np.float64, initial=0.0) + life_expectancy = Variable('life_expectancy', dtype=np.float64, initial=np.finfo(np.float64).max) + initialized_dynamic = Variable('initialized_dynamic', dtype=np.int32, initial=0) + +class AgeParticle_SciPy(ScipyParticle): + age = Variable('age', dtype=np.float64, initial=0.0) + life_expectancy = Variable('life_expectancy', dtype=np.float64, initial=np.finfo(np.float64).max) + initialized_dynamic = Variable('initialized_dynamic', dtype=np.int32, initial=0) + +def initialize(particle, fieldset, time): + if particle.initialized_dynamic < 1: + np_scaler = math.sqrt(3.0 / 2.0) + particle.life_expectancy = time + random.uniform(.0, (fieldset.life_expectancy-time) * 2.0 / np_scaler) + # particle.life_expectancy = time + ParcelsRandom.uniform(.0, (fieldset.life_expectancy-time)*math.sqrt(3.0 / 2.0)) + # particle.life_expectancy = time + ParcelsRandom.uniform(.0, fieldset.life_expectancy) * math.sqrt(3.0 / 2.0) + # particle.life_expectancy = time+ParcelsRandom.uniform(.0, fieldset.life_expectancy) * ((3.0/2.0)**2.0) + particle.initialized_dynamic = 1 + +def Age(particle, fieldset, time): + if particle.state == ErrorCode.Evaluate: + particle.age = particle.age + math.fabs(particle.dt) + if particle.age > particle.life_expectancy: + particle.delete() + +age_ptype = {'scipy': AgeParticle_SciPy, 'jit': AgeParticle_JIT} + +if __name__=='__main__': + parser = ArgumentParser(description="Example of particle advection using in-memory stommel test case") + parser.add_argument("-i", "--imageFileName", dest="imageFileName", type=str, default="mpiChunking_plot_MPI.png", help="image file name of the plot") + parser.add_argument("-b", "--backwards", dest="backwards", action='store_true', default=False, help="enable/disable running the simulation backwards") + parser.add_argument("-p", "--periodic", dest="periodic", action='store_true', default=False, help="enable/disable periodic wrapping (else: extrapolation)") + parser.add_argument("-r", "--release", dest="release", action='store_true', default=False, help="continuously add particles via repeatdt (default: False)") + parser.add_argument("-rt", "--releasetime", dest="repeatdt", type=int, default=720, help="repeating release rate of added particles in Minutes (default: 720min = 12h)") + parser.add_argument("-a", "--aging", dest="aging", action='store_true', default=False, help="Removed aging particles dynamically (default: False)") + parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=int, default=1, help="runtime in days (default: 1)") + parser.add_argument("-x", "--xarray", dest="use_xarray", action='store_true', default=False, help="use xarray as data backend") + parser.add_argument("-w", "--writeout", dest="write_out", action='store_true', default=False, help="write data in outfile") + parser.add_argument("-d", "--delParticle", dest="delete_particle", action='store_true', default=False, help="switch to delete a particle (True) or reset a particle (default: False).") + parser.add_argument("-A", "--animate", dest="animate", action='store_true', default=False, help="animate the particle trajectories during the run or not (default: False).") + parser.add_argument("-V", "--visualize", dest="visualize", action='store_true', default=False, help="Visualize particle trajectories at the end (default: False). Requires -w in addition to take effect.") + parser.add_argument("-N", "--n_particles", dest="nparticles", type=str, default="2**6", help="number of particles to generate and advect (default: 2e6)") + parser.add_argument("-sN", "--start_n_particles", dest="start_nparticles", type=str, default="96", help="(optional) number of particles generated per release cycle (if --rt is set) (default: 96)") + parser.add_argument("-m", "--mode", dest="compute_mode", choices=['jit','scipy'], default="jit", help="computation mode = [JIT, SciPp]") + parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + args = parser.parse_args() + + imageFileName=args.imageFileName + periodicFlag=args.periodic + backwardSimulation = args.backwards + repeatdtFlag=args.release + repeatRateMinutes=args.repeatdt + time_in_days = args.time_in_days + use_xarray = args.use_xarray + agingParticles = args.aging + with_GC = args.useGC + Nparticle = int(float(eval(args.nparticles))) + target_N = Nparticle + addParticleN = 1 + # np_scaler = math.sqrt(3.0/2.0) + # np_scaler = (3.0 / 2.0)**2.0 # ** + np_scaler = 3.0 / 2.0 + # cycle_scaler = math.sqrt(3.0/2.0) + # cycle_scaler = (3.0 / 2.0)**2.0 # ** + # cycle_scaler = 3.0 / 2.0 + cycle_scaler = 7.0 / 4.0 + start_N_particles = int(float(eval(args.start_nparticles))) + if MPI: + mpi_comm = MPI.COMM_WORLD + if mpi_comm.Get_rank() == 0: + if agingParticles and not repeatdtFlag: + sys.stdout.write("N: {} ( {} )\n".format(Nparticle, int(Nparticle * np_scaler))) + else: + sys.stdout.write("N: {}\n".format(Nparticle)) + else: + if agingParticles and not repeatdtFlag: + sys.stdout.write("N: {} ( {} )\n".format(Nparticle, int(Nparticle * np_scaler))) + else: + sys.stdout.write("N: {}\n".format(Nparticle)) + + dt_minutes = 60 + #dt_minutes = 20 + #random.seed(123456) + nowtime = datetime.datetime.now() + random.seed(nowtime.microsecond) + + branch = "benchmarking" + computer_env = "local/unspecified" + scenario = "bickleyjet" + odir = "" + if os.uname()[1] in ['science-bs35', 'science-bs36']: # Gemini + # odir = "/scratch/{}/experiments".format(os.environ['USER']) + odir = "/scratch/{}/experiments".format("ckehl") + computer_env = "Gemini" + elif fnmatch.fnmatchcase(os.uname()[1], "*.bullx*"): # Cartesius + CARTESIUS_SCRATCH_USERNAME = 'ckehluu' + odir = "/scratch/shared/{}/experiments".format(CARTESIUS_SCRATCH_USERNAME) + computer_env = "Cartesius" + else: + odir = "/var/scratch/experiments" + print("running {} on {} (uname: {}) - branch '{}' - (target) N: {} - argv: {}".format(scenario, computer_env, os.uname()[1], branch, target_N, sys.argv[1:])) + + if os.path.sep in imageFileName: + head_dir = os.path.dirname(imageFileName) + if head_dir[0] == os.path.sep: + odir = head_dir + else: + odir = os.path.join(odir, head_dir) + imageFileName = os.path.split(imageFileName)[1] + + func_time = [] + mem_used_GB = [] + + np.random.seed(0) + fieldset = None + if use_xarray: + fieldset = bickleyjet_from_numpy(periodic_wrap=periodicFlag) + else: + field_fpath = False + if args.write_out: + field_fpath = os.path.join(odir,"bickleyjet") + fieldset = bickleyjet_from_numpy(periodic_wrap=periodicFlag, write_out=field_fpath) + + if args.compute_mode is 'scipy': + Nparticle = 2**10 + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + global_t_0 = ostime.process_time() + else: + global_t_0 = ostime.process_time() + + simStart = None + for f in fieldset.get_fields(): + if type(f) in [VectorField, NestedField, SummedField]: # or not f.grid.defer_load + continue + else: + if backwardSimulation: + simStart=f.grid.time_full[-1] + else: + simStart = f.grid.time_full[0] + break + + if agingParticles: + if not repeatdtFlag: + Nparticle = int(Nparticle * np_scaler) + fieldset.add_constant('life_expectancy', delta(days=time_in_days).total_seconds()) + if repeatdtFlag: + addParticleN = Nparticle/2.0 + refresh_cycle = (delta(days=time_in_days).total_seconds() / (addParticleN/start_N_particles)) / cycle_scaler + if agingParticles: + refresh_cycle /= cycle_scaler + repeatRateMinutes = int(refresh_cycle/60.0) if repeatRateMinutes == 720 else repeatRateMinutes + + if backwardSimulation: + # ==== backward simulation ==== # + if agingParticles: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * (-b) + (b/2.0), time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(addParticleN), 1) * a, lat=np.random.rand(int(addParticleN), 1) * (-b) + (b/2.0), time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * (-b) + (b/2.0), time=simStart) + else: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * (-b) + (b/2.0), time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(addParticleN), 1) * a, lat=np.random.rand(int(addParticleN), 1) * (-b) + (b/2.0), time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * (-b) + (b/2.0), time=simStart) + else: + # ==== forward simulation ==== # + if agingParticles: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * (-b) + (b/2.0), time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(addParticleN), 1) * a, lat=np.random.rand(int(addParticleN), 1) * (-b) + (b/2.0), time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * (-b) + (b/2.0), time=simStart) + else: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * (-b) + (b/2.0), time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(addParticleN), 1) * a, lat=np.random.rand(int(addParticleN), 1) * (-b) + (b/2.0), time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * (-b) + (b/2.0), time=simStart) + + output_file = None + out_fname = "benchmark_bickleyjet" + if args.write_out: + if MPI and (MPI.COMM_WORLD.Get_size()>1): + out_fname += "_MPI" + else: + out_fname += "_noMPI" + out_fname += "_n"+str(Nparticle) + if backwardSimulation: + out_fname += "_bwd" + else: + out_fname += "_fwd" + if repeatdtFlag: + out_fname += "_add" + if agingParticles: + out_fname += "_age" + output_file = pset.ParticleFile(name=os.path.join(odir,out_fname+".nc"), outputdt=delta(hours=24)) + delete_func = RenewParticle + if args.delete_particle: + delete_func=DeleteParticle + postProcessFuncs = [] + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + starttime = ostime.process_time() + else: + starttime = ostime.process_time() + kernels = pset.Kernel(AdvectionRK4,delete_cfiles=True) + if agingParticles: + kernels += pset.Kernel(initialize, delete_cfiles=True) + kernels += pset.Kernel(Age, delete_cfiles=True) + if with_GC: + postProcessFuncs.append(perIterGC) + if backwardSimulation: + # ==== backward simulation ==== # + if args.animate: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=-dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12), moviedt=delta(hours=6), movie_background_field=fieldset.U) + else: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=-dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12)) + else: + # ==== forward simulation ==== # + if args.animate: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12), moviedt=delta(hours=6), movie_background_field=fieldset.U) + else: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12)) + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + endtime = ostime.process_time() + else: + endtime = ostime.process_time() + + if args.write_out: + output_file.close() + + if not args.dryrun: + if MPI: + mpi_comm = MPI.COMM_WORLD + # mpi_comm.Barrier() + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart-1) + Npart = mpi_comm.reduce(Npart, op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + if size_Npart>0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime-starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time*1000.0)) + else: + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart-1) + if size_Npart > 0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime - starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time * 1000.0)) + + if MPI: + mpi_comm = MPI.COMM_WORLD + Nparticles = mpi_comm.reduce(np.array(pset.nparticle_log.get_params()), op=MPI.SUM, root=0) + Nmem = mpi_comm.reduce(np.array(pset.mem_log.get_params()), op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + pset.plot_and_log(memory_used=Nmem, nparticles=Nparticles, target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 150]) + else: + pset.plot_and_log(target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 150]) + + diff --git a/performance/benchmark_deep_migration_NPacific.py b/performance/benchmark_deep_migration_NPacific.py new file mode 100644 index 0000000000..17b5fa2973 --- /dev/null +++ b/performance/benchmark_deep_migration_NPacific.py @@ -0,0 +1,453 @@ +from parcels import FieldSet, JITParticle, ScipyParticle, AdvectionRK4_3D, AdvectionRK4, ErrorCode, ParticleFile, Variable, Field, NestedField, VectorField, timer +from parcels import ParticleSet_Benchmark +from parcels.kernels import seawaterdensity +from argparse import ArgumentParser +from datetime import timedelta as delta +# from datetime import datetime +import time as ostime +import os +import sys +import math +import numpy as np +import scipy.linalg +# from numpy import * +from glob import glob +import fnmatch +import warnings +import pickle +import gc +warnings.filterwarnings("ignore") + +try: + from mpi4py import MPI +except: + MPI = None + + +global_t_0 = 0 +# Fieldset grid is 30x30 deg in North Pacific +minlat = 20 +maxlat = 50 +minlon = -175 +maxlon = -145 + +# Release particles on a 10x10 deg grid in middle of the 30x30 fieldset grid and 1m depth +lat_release0 = np.tile(np.linspace(30,39,10),[10,1]) +lat_release = lat_release0.T +lon_release = np.tile(np.linspace(-165,-156,10),[10,1]) +z_release = np.tile(1,[10,10]) + +# Choose: +simdays = 50.0 * 365.0 +#simdays = 5 +time0 = 0 +simhours = 1 +simmins = 30 +secsdt = 30 +hrsoutdt = 5 + +#--------- Choose below: NOTE- MUST ALSO MANUALLY CHANGE IT IN THE KOOI KERNAL BELOW ----- +rho_pl = 920. # density of plastic (kg m-3): DEFAULT FOR FIG 1 in Kooi: 920 but full range is: 840, 920, 940, 1050, 1380 (last 2 are initially non-buoyant) +r_pl = "1e-04" # radius of plastic (m): DEFAULT FOR FIG 1: 10-3 to 10-6 included but full range is: 10 mm to 0.1 um or 10-2 to 10-7 + + +def Kooi(particle,fieldset,time): + #------ CHOOSE ----- + rho_pl = 920. # density of plastic (kg m-3): DEFAULT FOR FIG 1: 920 but full range is: 840, 920, 940, 1050, 1380 (last 2 are initially non-buoyant) + r_pl = 1e-04 # radius of plastic (m): DEFAULT FOR FIG 1: 10-3 to 10-6 included but full range is: 10 mm to 0.1 um or 10-2 to 10-7 + + # Nitrogen to cell ratios for ambient algal concentrations ('aa') and algal growth ('mu_aa') from NEMO output (no longer using N:C:AA (Redfield ratio), directly N:AA from Menden-Deuer and Lessard 2000) + min_N2cell = 2656.0e-09 #[mgN cell-1] (from Menden-Deuer and Lessard 2000) + max_N2cell = 11.0e-09 #[mgN cell-1] + med_N2cell = 356.04e-09 #[mgN cell-1] THIS is used below + + # Ambient algal concentration from MEDUSA's non-diatom + diatom phytoplankton + n0 = particle.nd_phy+particle.d_phy # [mmol N m-3] in MEDUSA + n = n0*14.007 # conversion from [mmol N m-3] to [mg N m-3] (atomic weight of 1 mol of N = 14.007 g) + n2 = n/med_N2cell # conversion from [mg N m-3] to [no. m-3] + + if n2<0.: + aa = 0. + else: + aa = n2 # [no m-3] to compare to Kooi model + + # Primary productivity (algal growth) only above euphotic zone, condition same as in Kooi et al. 2017 + if particle.depth 5e9: + w = 1000. + elif dn <0.05: + w = (d**2.) *1.71E-4 + else: + w = 10.**(-3.76715 + (1.92944*math.log10(d)) - (0.09815*math.log10(d)**2.) - (0.00575*math.log10(d)**3.) + (0.00056*math.log10(d)**4.)) + + if z >= 4000.: + vs = 0 + elif z < 1. and delta_rho < 0: + vs = 0 + elif delta_rho > 0: + vs = (g * kin_visc * w * delta_rho)**(1./3.) + else: + a_del_rho = delta_rho*-1. + vs = -1.*(g * kin_visc * w * a_del_rho)**(1./3.) # m s-1 + + particle.depth += vs * particle.dt + particle.vs = vs + z = particle.depth + dt = particle.dt + +""" Defining the particle class """ + +class plastic_particle(JITParticle): #ScipyParticle): # + u = Variable('u', dtype=np.float32,to_write=False) + v = Variable('v', dtype=np.float32,to_write=False) + w = Variable('w', dtype=np.float32,to_write=False) + temp = Variable('temp',dtype=np.float32,to_write=False) + density = Variable('density',dtype=np.float32,to_write=True) + #aa = Variable('aa',dtype=np.float32,to_write=True) + #d_tpp = Variable('d_tpp',dtype=np.float32,to_write=False) # mu_aa + #nd_tpp = Variable('nd_tpp',dtype=np.float32,to_write=False) + tpp3 = Variable('tpp3',dtype=np.float32,to_write=False) + euph_z = Variable('euph_z',dtype=np.float32,to_write=False) + d_phy = Variable('d_phy',dtype=np.float32,to_write=False) + nd_phy = Variable('nd_phy',dtype=np.float32,to_write=False) + kin_visc = Variable('kin_visc',dtype=np.float32,to_write=False) + sw_visc = Variable('sw_visc',dtype=np.float32,to_write=False) + a = Variable('a',dtype=np.float32,to_write=False) + vs = Variable('vs',dtype=np.float32,to_write=True) + +"""functions and kernals""" + +def DeleteParticle(particle, fieldset, time): + """Kernel for deleting particles if they are out of bounds.""" + # print('particle is deleted') #print(particle.lon, particle.lat, particle.depth) + particle.delete() + +def perIterGC(): + gc.collect() + +def getclosest_ij(lats,lons,latpt,lonpt): + """Function to find the index of the closest point to a certain lon/lat value.""" + dist_sq = (lats-latpt)**2 + (lons-lonpt)**2 # find squared distance of every point on grid + minindex_flattened = dist_sq.argmin() # 1D index of minimum dist_sq element + return np.unravel_index(minindex_flattened, lats.shape) # Get 2D index for latvals and lonvals arrays from 1D index + +def AdvectionRK4_3D_vert(particle, fieldset, time): # adapting AdvectionRK4_3D kernal to only vertical velocity + """Advection of particles using fourth-order Runge-Kutta integration including vertical velocity. + Function needs to be converted to Kernel object before execution""" + (w1) = fieldset.W[time, particle.depth, particle.lat, particle.lon] + #lon1 = particle.lon + u1*.5*particle.dt + #lat1 = particle.lat + v1*.5*particle.dt + dep1 = particle.depth + w1*.5*particle.dt + (w2) = fieldset.W[time + .5 * particle.dt, dep1, particle.lat, particle.lon] + #lon2 = particle.lon + u2*.5*particle.dt + #lat2 = particle.lat + v2*.5*particle.dt + dep2 = particle.depth + w2*.5*particle.dt + (w3) = fieldset.W[time + .5 * particle.dt, dep2, particle.lat, particle.lon] + #lon3 = particle.lon + u3*particle.dt + #lat3 = particle.lat + v3*particle.dt + dep3 = particle.depth + w3*particle.dt + (w4) = fieldset.W[time + particle.dt, dep3, particle.lat, particle.lon] + #particle.lon += particle.lon #(u1 + 2*u2 + 2*u3 + u4) / 6. * particle.dt + #particle.lat += particle.lat #lats[1,1] #(v1 + 2*v2 + 2*v3 + v4) / 6. * particle.dt + particle.depth += (w1 + 2*w2 + 2*w3 + w4) / 6. * particle.dt + +def Profiles(particle, fieldset, time): + particle.temp = fieldset.cons_temperature[time, particle.depth,particle.lat,particle.lon] + particle.d_phy= fieldset.d_phy[time, particle.depth,particle.lat,particle.lon] + particle.nd_phy= fieldset.nd_phy[time, particle.depth,particle.lat,particle.lon] + #particle.d_tpp = fieldset.d_tpp[time,particle.depth,particle.lat,particle.lon] + #particle.nd_tpp = fieldset.nd_tpp[time,particle.depth,particle.lat,particle.lon] + particle.tpp3 = fieldset.tpp3[time,particle.depth,particle.lat,particle.lon] + particle.euph_z = fieldset.euph_z[time,particle.depth,particle.lat,particle.lon] + particle.kin_visc = fieldset.KV[time,particle.depth,particle.lat,particle.lon] + particle.sw_visc = fieldset.SV[time,particle.depth,particle.lat,particle.lon] + particle.w = fieldset.W[time,particle.depth,particle.lat,particle.lon] + +if __name__ == "__main__": + parser = ArgumentParser(description="Example of particle advection using in-memory stommel test case") + parser.add_argument("-i", "--imageFileName", dest="imageFileName", type=str, default="mpiChunking_plot_MPI.png", help="image file name of the plot") + parser.add_argument("-p", "--periodic", dest="periodic", action='store_true', default=False, help="enable/disable periodic wrapping (else: extrapolation)") + # parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=int, default=365, help="runtime in days (default: 365)") + parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=str, default="1*365", help="runtime in days (default: 1*365)") + parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + args = parser.parse_args() + + imageFileName=args.imageFileName + periodicFlag=args.periodic + time_in_days = int(float(eval(args.time_in_days))) + with_GC = args.useGC + + branch = "benchmarking" + computer_env = "local/unspecified" + scenario = "deep_migration" + headdir = "" + odir = "" + datahead = "" + dirread_top = "" + dirread_top_bgc = "" + dirread_mesh = "" + if os.uname()[1] in ['science-bs35', 'science-bs36']: # Gemini + # headdir = "/scratch/{}/experiments/palaeo-parcels".format(os.environ['USER']) + headdir = "/scratch/{}/experiments/deep_migration_behaviour".format("ckehl") + odir = os.path.join(headdir,"BENCHres") + datahead = "/data/oceanparcels/input_data" + dirread = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/means/') + dirread_bgc = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/means/') + dirread_mesh = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/domain/') + computer_env = "Gemini" + # elif fnmatch.fnmatchcase(os.uname()[1], "int?.*"): # Cartesius + elif fnmatch.fnmatchcase(os.uname()[1], "*.bullx*"): # Cartesius + CARTESIUS_SCRATCH_USERNAME = 'ckehluu' + headdir = "/scratch/shared/{}/experiments/deep_migration_behaviour".format(CARTESIUS_SCRATCH_USERNAME) + odir = os.path.join(headdir, "BENCHres") + datahead = "/projects/0/topios/hydrodynamic_data" + dirread = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/means/') + dirread_bgc = os.path.join(datahead, 'NEMO-MEDUSA_BGC/ORCA0083-N006/means/') + dirread_mesh = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/domain/') + computer_env = "Cartesius" + else: + headdir = "/var/scratch/dlobelle" + odir = os.path.join(headdir, "BENCHres") + datahead = "/data" + dirread = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/means/') + dirread_bgc = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/means/') + dirread_mesh = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/domain/') + print("running {} on {} (uname: {}) - branch '{}' - (target) N: {} - argv: {}".format(scenario, computer_env, os.uname()[1], branch, lon_release.shape[0], sys.argv[1:])) + + # ==== CARTESIUS ==== # + # dirread = '/projects/0/topios/hydrodynamic_data/NEMO-MEDUSA/ORCA0083-N006/means/' + # dirread_bgc = '/projects/0/topios/hydrodynamic_data/NEMO-MEDUSA_BGC/ORCA0083-N006/means/' + # dirread_mesh = '/projects/0/topios/hydrodynamic_data/NEMO-MEDUSA/ORCA0083-N006/domain/' + # ==== GEMINI ==== # + # dirread = '/data/oceanparcels/input_data/NEMO-MEDUSA/ORCA0083-N006/means/' + # dirread_bgc = '/data/oceanparcels/input_data/NEMO-MEDUSA/ORCA0083-N006/means/' + # dirread_mesh = '/data/oceanparcels/input_data/NEMO-MEDUSA/ORCA0083-N006/domain/' + # dirwrite = '/scratch/ckehl/experiments/deep_migration_behaviour/NEMOres/tests/' + # ==== ====== ==== # + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + # global_t_0 = ostime.time() + # global_t_0 = MPI.Wtime() + global_t_0 = ostime.process_time() + else: + # global_t_0 = ostime.time() + global_t_0 = ostime.process_time() + + ufiles = sorted(glob(dirread+'ORCA0083-N06_2000*d05U.nc')) #0105d05 + vfiles = sorted(glob(dirread+'ORCA0083-N06_2000*d05V.nc')) + wfiles = sorted(glob(dirread+'ORCA0083-N06_2000*d05W.nc')) + pfiles = sorted(glob(dirread_bgc+'ORCA0083-N06_2000*d05P.nc')) + ppfiles = sorted(glob(dirread_bgc+'ORCA0083-N06_2000*d05D.nc')) + tsfiles = sorted(glob(dirread+'ORCA0083-N06_2000*d05T.nc')) + mesh_mask = dirread_mesh+'coordinates.nc' + + filenames = {'U': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': ufiles}, #'depth': wfiles, + 'V': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': vfiles}, + 'W': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': wfiles}, + 'd_phy': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': pfiles}, + 'nd_phy': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': pfiles}, + 'euph_z': {'lon': mesh_mask, 'lat': mesh_mask, 'data': ppfiles}, + #'d_tpp': {'lon': mesh_mask, 'lat': mesh_mask, 'data': ppfiles}, # 'depth': wfiles, + #'nd_tpp': {'lon': mesh_mask, 'lat': mesh_mask, 'data': ppfiles}, + 'tpp3': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': ppfiles}, + 'cons_temperature': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': tsfiles}, + 'abs_salinity': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': tsfiles}} + + + variables = {'U': 'uo', + 'V': 'vo', + 'W': 'wo', + 'd_phy': 'PHD', + 'nd_phy': 'PHN', + 'euph_z': 'MED_XZE', + #'d_tpp': 'ML_PRD', # units: mmolN/m2/d + #'nd_tpp': 'ML_PRN', # units: mmolN/m2/d + 'tpp3': 'TPP3', # units: mmolN/m3/d + 'cons_temperature': 'potemp', + 'abs_salinity': 'salin'} + + dimensions = {'U': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}, #time_centered + 'V': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}, + 'W': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw', 'time': 'time_counter'}, + 'd_phy': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw','time': 'time_counter'}, + 'nd_phy': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw','time': 'time_counter'}, + 'euph_z': {'lon': 'glamf', 'lat': 'gphif','time': 'time_counter'}, + #'d_tpp': {'lon': 'glamf', 'lat': 'gphif','time': 'time_counter'}, # 'depth': 'depthw', + #'nd_tpp': {'lon': 'glamf', 'lat': 'gphif','time': 'time_counter'}, + 'tpp3': {'lon': 'glamf', 'lat': 'gphif','depth': 'depthw', 'time': 'time_counter'}, + 'cons_temperature': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw','time': 'time_counter'}, + 'abs_salinity': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw','time': 'time_counter'}} + + chs = {'time_counter': 1, 'depthu': 75, 'depthv': 75, 'depthw': 75, 'deptht': 75, 'y': 200, 'x': 200} + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, allow_time_extrapolation=False, field_chunksize=chs, time_periodic=delta(days=365)) + depths = fieldset.U.depth + + outfile = 'Kooi+NEMO_3D_grid10by10_rho'+str(int(rho_pl))+'_r'+ r_pl+'_'+str(simdays)+'days_'+str(secsdt)+'dtsecs_'+str(hrsoutdt)+'hrsoutdt' + dirwrite = os.path.join(odir, "rho_"+str(int(rho_pl))+"kgm-3") + if not os.path.exists(dirwrite): + os.mkdir(dirwrite) + + # Kinematic viscosity and dynamic viscosity not available in MEDUSA so replicating Kooi's profiles at all grid points + # profile_auxin_path = '/home/dlobelle/Kooi_data/data_input/profiles.pickle' + # profile_auxin_path = '/scratch/ckehl/experiments/deep_migration_behaviour/aux_in/profiles.pickle' + profile_auxin_path = os.path.join(headdir, 'aux_in/profiles.pickle') + with open(profile_auxin_path, 'rb') as f: + depth,T_z,S_z,rho_z,upsilon_z,mu_z = pickle.load(f) + + v_lon = np.array([minlon,maxlon]) + v_lat = np.array([minlat,maxlat]) + + kv_or = np.transpose(np.tile(np.array(upsilon_z),(len(v_lon),len(v_lat),1)), (2,0,1)) # kinematic viscosity + sv_or = np.transpose(np.tile(np.array(mu_z),(len(v_lon),len(v_lat),1)), (2,0,1)) # dynamic viscosity of seawater + KV = Field('KV', kv_or, lon=v_lon, lat=v_lat, depth=depths, mesh='spherical', field_chunksize=False)#,transpose="True") #,fieldtype='U') + SV = Field('SV', sv_or, lon=v_lon, lat=v_lat, depth=depths, mesh='spherical', field_chunksize=False)#,transpose="True") #,fieldtype='U') + fieldset.add_field(KV, 'KV') + fieldset.add_field(SV, 'SV') + + """ Defining the particle set """ + pset = ParticleSet_Benchmark.from_list(fieldset=fieldset, # the fields on which the particles are advected + pclass=plastic_particle, # the type of particles (JITParticle or ScipyParticle) + lon= lon_release, #-160., # a vector of release longitudes + lat= lat_release, #36., + time = time0, + depth = z_release) #[1.] + + # perflog = PerformanceLog() + # perflog.pset = pset + #postProcessFuncs = [perflog.advance,] + + """ Kernal + Execution""" + postProcessFuncs = [] + if with_GC: + postProcessFuncs.append(perIterGC) + # kernels = pset.Kernel(AdvectionRK4_3D) + pset.Kernel(seawaterdensity.polyTEOS10_bsq) + pset.Kernel(Profiles) + pset.Kernel(Kooi) + kernels = pset.Kernel(AdvectionRK4_3D_vert) + pset.Kernel(seawaterdensity.polyTEOS10_bsq) + pset.Kernel(Profiles) + pset.Kernel(Kooi) + pfile= ParticleFile(os.path.join(dirwrite, outfile), pset, outputdt=delta(hours = hrsoutdt)) + + starttime = 0 + endtime = 0 + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + # global_t_0 = ostime.time() + # starttime = MPI.Wtime() + starttime = ostime.process_time() + else: + #starttime = ostime.time() + starttime = ostime.process_time() + + # postIterationCallbacks = postProcessFuncs, callbackdt = delta(hours=hrsoutdt) + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(seconds = secsdt), output_file=pfile, verbose_progress=True, recovery={ErrorCode.ErrorOutOfBounds: DeleteParticle}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours = hrsoutdt)) + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + # global_t_0 = ostime.time() + # endtime = MPI.Wtime() + endtime = ostime.process_time() + else: + # endtime = ostime.time() + endtime = ostime.process_time() + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_comm.Barrier() + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart - 1) + Npart = mpi_comm.reduce(Npart, op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + if size_Npart>0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime-starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time*1000.0)) + else: + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart-1) + if size_Npart > 0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime - starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time * 1000.0)) + + pfile.close() + + if MPI: + mpi_comm = MPI.COMM_WORLD + # mpi_comm.Barrier() + Nparticles = mpi_comm.reduce(np.array(pset.nparticle_log.get_params()), op=MPI.SUM, root=0) + Nmem = mpi_comm.reduce(np.array(pset.mem_log.get_params()), op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + pset.plot_and_log(memory_used=Nmem, nparticles=Nparticles, target_N=1, imageFilePath=imageFileName, odir=odir) + else: + pset.plot_and_log(target_N=1, imageFilePath=imageFileName, odir=odir) + + print('Execution finished') diff --git a/performance/benchmark_doublegyre.py b/performance/benchmark_doublegyre.py new file mode 100644 index 0000000000..8733e6a02f --- /dev/null +++ b/performance/benchmark_doublegyre.py @@ -0,0 +1,421 @@ +""" +Author: Dr. Christian Kehl +Date: 11-02-2020 +""" + +from parcels import AdvectionEE, AdvectionRK45, AdvectionRK4 +from parcels import FieldSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, ErrorCode +from parcels.particleset_benchmark import ParticleSet_Benchmark as ParticleSet +from parcels.particleset import ParticleSet as DryParticleSet +from parcels.field import Field, VectorField, NestedField, SummedField +# from parcels import plotTrajectoriesFile_loadedField +from parcels import rng as random +from datetime import timedelta as delta +import math +from argparse import ArgumentParser +import datetime +import numpy as np +import xarray as xr +import fnmatch +import sys +import gc +import os +import time as ostime +# import matplotlib.pyplot as plt +from parcels.tools import perlin3d +from parcels.tools import perlin2d + +try: + from mpi4py import MPI +except: + MPI = None +with_GC = False + +pset = None +ptype = {'scipy': ScipyParticle, 'jit': JITParticle} +method = {'RK4': AdvectionRK4, 'EE': AdvectionEE, 'RK45': AdvectionRK45} +global_t_0 = 0 +Nparticle = int(math.pow(2,10)) # equals to Nparticle = 1024 +#Nparticle = int(math.pow(2,11)) # equals to Nparticle = 2048 +#Nparticle = int(math.pow(2,12)) # equals to Nparticle = 4096 +#Nparticle = int(math.pow(2,13)) # equals to Nparticle = 8192 +#Nparticle = int(math.pow(2,14)) # equals to Nparticle = 16384 +#Nparticle = int(math.pow(2,15)) # equals to Nparticle = 32768 +#Nparticle = int(math.pow(2,16)) # equals to Nparticle = 65536 +#Nparticle = int(math.pow(2,17)) # equals to Nparticle = 131072 +#Nparticle = int(math.pow(2,18)) # equals to Nparticle = 262144 +#Nparticle = int(math.pow(2,19)) # equals to Nparticle = 524288 + +#a = 1000 * 1e3 +#b = 1000 * 1e3 +#scalefac = 2.0 +#tsteps = 61 +#tscale = 6 + +a = 9.6 * 1e3 # [a in km -> 10e3] +b = 4.8 * 1e3 # [b in km -> 10e3] +tsteps = 122 # in steps +tstepsize = 6.0 # unitary +tscale = 12.0*60.0*60.0 # in seconds +# time[days] = (tsteps * tstepsize) * tscale +# gyre_rotation_speed = 60.0*24.0*60.0*60.0 # assume 1 rotation every 8.5 weeks +gyre_rotation_speed = (366.0*24.0*60.0*60.0)/2.0 # assume 1 rotation every 8.5 weeks +# scalefac = 2.0 * 6.0 # scaling the advection speeds to 12 m/s +scalefac = (40.0 / (1000.0/60.0)) # 40 km/h +scalefac /= 1000.0 + +# we need to modify the kernel.execute / pset.execute so that it returns from the JIT +# in a given time WITHOUT writing to disk via outfie => introduce a pyloop_dt + +def DeleteParticle(particle, fieldset, time): + particle.delete() + +def RenewParticle(particle, fieldset, time): + particle.lat = np.random.rand() * (-a) + (a/2.0) + particle.lon = np.random.rand() * (-b) + (b/2.0) + +def perIterGC(): + gc.collect() + +def doublegyre_from_numpy(xdim=960, ydim=480, periodic_wrap=False, write_out=False, steady=False): + """Implemented following Froyland and Padberg (2009), 10.1016/j.physd.2009.03.002""" + A = 0.3 + epsilon = 0.25 + omega = 2 * np.pi + + + lon = np.linspace(-a*0.5, a*0.5, xdim, dtype=np.float32) + lonrange = lon.max()-lon.min() + sys.stdout.write("lon field: {}\n".format(lon.size)) + lat = np.linspace(-b*0.5, b*0.5, ydim, dtype=np.float32) + latrange = lat.max() - lat.min() + sys.stdout.write("lat field: {}\n".format(lat.size)) + totime = (tsteps * tstepsize) * tscale + times = np.linspace(0., totime, tsteps, dtype=np.float64) + sys.stdout.write("time field: {}\n".format(times.size)) + dx, dy = lon[2]-lon[1], lat[2]-lat[1] + + U = np.zeros((times.size, lat.size, lon.size), dtype=np.float32) + V = np.zeros((times.size, lat.size, lon.size), dtype=np.float32) + freqs = np.ones(times.size, dtype=np.float32) + if not steady: + for ti in range(times.shape[0]): + time_f = times[ti] / gyre_rotation_speed + # time_f = np.fmod((times[ti])/gyre_rotation_speed, 2.0) + # time_f = np.fmod((times[ti]/gyre_rotation_speed), 2*np.pi) + # freqs[ti] = omega * np.cos(time_f) * 2.0 + freqs[ti] *= omega * time_f + # freqs[ti] *= time_f + else: + freqs = (freqs * 0.5) * omega + + for ti in range(times.shape[0]): + freq = freqs[ti] + # print(freq) + for i in range(lon.shape[0]): + for j in range(lat.shape[0]): + x1 = ((lon[i]*2.0 + a) / a) # - dx / 2 + x2 = ((lat[j]*2.0 + b) / (2.0*b)) # - dy / 2 + f_xt = (epsilon * np.sin(freq) * x1**2.0) + (1.0 - (2.0 * epsilon * np.sin(freq))) * x1 + U[ti, j, i] = -np.pi * A * np.sin(np.pi * f_xt) * np.cos(np.pi * x2) + V[ti, j, i] = np.pi * A * np.cos(np.pi * f_xt) * np.sin(np.pi * x2) * (2 * epsilon * np.sin(freq) * x1 + 1 - 2 * epsilon * np.sin(freq)) + + U *= scalefac + # U = np.transpose(U, (0, 2, 1)) + V *= scalefac + # V = np.transpose(V, (0, 2, 1)) + + + data = {'U': U, 'V': V} + dimensions = {'time': times, 'lon': lon, 'lat': lat} + fieldset = None + if periodic_wrap: + fieldset = FieldSet.from_data(data, dimensions, mesh='flat', transpose=False, time_periodic=delta(days=1)) + else: + fieldset = FieldSet.from_data(data, dimensions, mesh='flat', transpose=False, allow_time_extrapolation=True) + if write_out: + fieldset.write(filename=write_out) + return fieldset + + +class AgeParticle_JIT(JITParticle): + age = Variable('age', dtype=np.float64, initial=0.0) + life_expectancy = Variable('life_expectancy', dtype=np.float64, initial=np.finfo(np.float64).max) + initialized_dynamic = Variable('initialized_dynamic', dtype=np.int32, initial=0) + +class AgeParticle_SciPy(ScipyParticle): + age = Variable('age', dtype=np.float64, initial=0.0) + life_expectancy = Variable('life_expectancy', dtype=np.float64, initial=np.finfo(np.float64).max) + initialized_dynamic = Variable('initialized_dynamic', dtype=np.int32, initial=0) + +def initialize(particle, fieldset, time): + if particle.initialized_dynamic < 1: + np_scaler = math.sqrt(3.0 / 2.0) + particle.life_expectancy = time + random.uniform(.0, (fieldset.life_expectancy-time) * 2.0 / np_scaler) + # particle.life_expectancy = time + ParcelsRandom.uniform(.0, (fieldset.life_expectancy-time)*math.sqrt(3.0 / 2.0)) + # particle.life_expectancy = time + ParcelsRandom.uniform(.0, fieldset.life_expectancy) * math.sqrt(3.0 / 2.0) + # particle.life_expectancy = time+ParcelsRandom.uniform(.0, fieldset.life_expectancy) * ((3.0/2.0)**2.0) + particle.initialized_dynamic = 1 + +def Age(particle, fieldset, time): + if particle.state == ErrorCode.Evaluate: + particle.age = particle.age + math.fabs(particle.dt) + if particle.age > particle.life_expectancy: + particle.delete() + +age_ptype = {'scipy': AgeParticle_SciPy, 'jit': AgeParticle_JIT} + +if __name__=='__main__': + parser = ArgumentParser(description="Example of particle advection using in-memory stommel test case") + parser.add_argument("-i", "--imageFileName", dest="imageFileName", type=str, default="mpiChunking_plot_MPI.png", help="image file name of the plot") + parser.add_argument("-b", "--backwards", dest="backwards", action='store_true', default=False, help="enable/disable running the simulation backwards") + parser.add_argument("-p", "--periodic", dest="periodic", action='store_true', default=False, help="enable/disable periodic wrapping (else: extrapolation)") + parser.add_argument("-r", "--release", dest="release", action='store_true', default=False, help="continuously add particles via repeatdt (default: False)") + parser.add_argument("-rt", "--releasetime", dest="repeatdt", type=int, default=720, help="repeating release rate of added particles in Minutes (default: 720min = 12h)") + parser.add_argument("-a", "--aging", dest="aging", action='store_true', default=False, help="Removed aging particles dynamically (default: False)") + parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=int, default=1, help="runtime in days (default: 1)") + parser.add_argument("-x", "--xarray", dest="use_xarray", action='store_true', default=False, help="use xarray as data backend") + parser.add_argument("-w", "--writeout", dest="write_out", action='store_true', default=False, help="write data in outfile") + parser.add_argument("-d", "--delParticle", dest="delete_particle", action='store_true', default=False, help="switch to delete a particle (True) or reset a particle (default: False).") + parser.add_argument("-A", "--animate", dest="animate", action='store_true', default=False, help="animate the particle trajectories during the run or not (default: False).") + parser.add_argument("-V", "--visualize", dest="visualize", action='store_true', default=False, help="Visualize particle trajectories at the end (default: False). Requires -w in addition to take effect.") + parser.add_argument("-N", "--n_particles", dest="nparticles", type=str, default="2**6", help="number of particles to generate and advect (default: 2e6)") + parser.add_argument("-sN", "--start_n_particles", dest="start_nparticles", type=str, default="96", help="(optional) number of particles generated per release cycle (if --rt is set) (default: 96)") + parser.add_argument("-m", "--mode", dest="compute_mode", choices=['jit','scipy'], default="jit", help="computation mode = [JIT, SciPp]") + parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + args = parser.parse_args() + + imageFileName=args.imageFileName + periodicFlag=args.periodic + backwardSimulation = args.backwards + repeatdtFlag=args.release + repeatRateMinutes=args.repeatdt + time_in_days = args.time_in_days + use_xarray = args.use_xarray + agingParticles = args.aging + with_GC = args.useGC + Nparticle = int(float(eval(args.nparticles))) + target_N = Nparticle + addParticleN = 1 + # np_scaler = math.sqrt(3.0/2.0) + # np_scaler = (3.0 / 2.0)**2.0 # ** + np_scaler = 3.0 / 2.0 + # cycle_scaler = math.sqrt(3.0/2.0) + # cycle_scaler = (3.0 / 2.0)**2.0 # ** + # cycle_scaler = 3.0 / 2.0 + cycle_scaler = 7.0 / 4.0 + start_N_particles = int(float(eval(args.start_nparticles))) + if MPI: + mpi_comm = MPI.COMM_WORLD + if mpi_comm.Get_rank() == 0: + if agingParticles and not repeatdtFlag: + sys.stdout.write("N: {} ( {} )\n".format(Nparticle, int(Nparticle * np_scaler))) + else: + sys.stdout.write("N: {}\n".format(Nparticle)) + else: + if agingParticles and not repeatdtFlag: + sys.stdout.write("N: {} ( {} )\n".format(Nparticle, int(Nparticle * np_scaler))) + else: + sys.stdout.write("N: {}\n".format(Nparticle)) + + dt_minutes = 60 + #dt_minutes = 20 + #random.seed(123456) + nowtime = datetime.datetime.now() + random.seed(nowtime.microsecond) + + branch = "benchmarking" + computer_env = "local/unspecified" + scenario = "doublegyre" + odir = "" + if os.uname()[1] in ['science-bs35', 'science-bs36']: # Gemini + # odir = "/scratch/{}/experiments".format(os.environ['USER']) + odir = "/scratch/{}/experiments".format("ckehl") + computer_env = "Gemini" + elif fnmatch.fnmatchcase(os.uname()[1], "*.bullx*"): # Cartesius + CARTESIUS_SCRATCH_USERNAME = 'ckehluu' + odir = "/scratch/shared/{}/experiments".format(CARTESIUS_SCRATCH_USERNAME) + computer_env = "Cartesius" + else: + odir = "/var/scratch/experiments" + print("running {} on {} (uname: {}) - branch '{}' - (target) N: {} - argv: {}".format(scenario, computer_env, os.uname()[1], branch, target_N, sys.argv[1:])) + + if os.path.sep in imageFileName: + head_dir = os.path.dirname(imageFileName) + if head_dir[0] == os.path.sep: + odir = head_dir + else: + odir = os.path.join(odir, head_dir) + imageFileName = os.path.split(imageFileName)[1] + + func_time = [] + mem_used_GB = [] + + np.random.seed(0) + fieldset = None + if use_xarray: + fieldset = doublegyre_from_numpy(periodic_wrap=periodicFlag) + else: + field_fpath = False + if args.write_out: + field_fpath = os.path.join(odir,"doublegyre") + # fieldset = doublegyre_from_numpy(xdim=240, ydim=120, periodic_wrap=periodicFlag, write_out=field_fpath) + fieldset = doublegyre_from_numpy(xdim=960, ydim=480, periodic_wrap=periodicFlag, write_out=field_fpath) + + if args.compute_mode is 'scipy': + Nparticle = 2**10 + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + global_t_0 = ostime.process_time() + else: + global_t_0 = ostime.process_time() + + simStart = None + for f in fieldset.get_fields(): + if type(f) in [VectorField, NestedField, SummedField]: # or not f.grid.defer_load + continue + else: + if backwardSimulation: + simStart=f.grid.time_full[-1] + else: + simStart = f.grid.time_full[0] + break + + if agingParticles: + if not repeatdtFlag: + Nparticle = int(Nparticle * np_scaler) + fieldset.add_constant('life_expectancy', delta(days=time_in_days).total_seconds()) + if repeatdtFlag: + addParticleN = Nparticle/2.0 + refresh_cycle = (delta(days=time_in_days).total_seconds() / (addParticleN/start_N_particles)) / cycle_scaler + if agingParticles: + refresh_cycle /= cycle_scaler + repeatRateMinutes = int(refresh_cycle/60.0) if repeatRateMinutes == 720 else repeatRateMinutes + + if backwardSimulation: + # ==== backward simulation ==== # + if agingParticles: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * (-a) + (a/2.0), lat=np.random.rand(start_N_particles, 1) * (-b) + (b/2.0), time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(addParticleN), 1) * (-a) + (a/2.0), lat=np.random.rand(int(addParticleN), 1) * (-b) + (b/2.0), time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * (-a) + (a/2.0), lat=np.random.rand(Nparticle, 1) * (-b) + (b/2.0), time=simStart) + else: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * (-a) + (a/2.0), lat=np.random.rand(start_N_particles, 1) * (-b) + (b/2.0), time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(addParticleN), 1) * (-a) + (a/2.0), lat=np.random.rand(int(addParticleN), 1) * (-b) + (b/2.0), time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * (-a) + (a/2.0), lat=np.random.rand(Nparticle, 1) * (-b) + (b/2.0), time=simStart) + else: + # ==== forward simulation ==== # + if agingParticles: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * (-a) + (a/2.0), lat=np.random.rand(start_N_particles, 1) * (-b) + (b/2.0), time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(addParticleN), 1) * (-a) + (a/2.0), lat=np.random.rand(int(addParticleN), 1) * (-b) + (b/2.0), time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * (-a) + (a/2.0), lat=np.random.rand(Nparticle, 1) * (-b) + (b/2.0), time=simStart) + else: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * (-a) + (a/2.0), lat=np.random.rand(start_N_particles, 1) * (-b) + (b/2.0), time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(addParticleN), 1) * (-a) + (a/2.0), lat=np.random.rand(int(addParticleN), 1) * (-b) + (b/2.0), time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * (-a) + (a/2.0), lat=np.random.rand(Nparticle, 1) * (-b) + (b/2.0), time=simStart) + + output_file = None + out_fname = "benchmark_doublegyre" + if args.write_out: + if MPI and (MPI.COMM_WORLD.Get_size()>1): + out_fname += "_MPI" + else: + out_fname += "_noMPI" + out_fname += "_n"+str(Nparticle) + if backwardSimulation: + out_fname += "_bwd" + else: + out_fname += "_fwd" + if repeatdtFlag: + out_fname += "_add" + if agingParticles: + out_fname += "_age" + output_file = pset.ParticleFile(name=os.path.join(odir,out_fname+".nc"), outputdt=delta(hours=24)) + delete_func = RenewParticle + if args.delete_particle: + delete_func=DeleteParticle + postProcessFuncs = [] + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + starttime = ostime.process_time() + else: + starttime = ostime.process_time() + kernels = pset.Kernel(AdvectionRK4,delete_cfiles=True) + if agingParticles: + kernels += pset.Kernel(initialize, delete_cfiles=True) + kernels += pset.Kernel(Age, delete_cfiles=True) + if with_GC: + postProcessFuncs.append(perIterGC) + if backwardSimulation: + # ==== backward simulation ==== # + if args.animate: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=-dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12), moviedt=delta(hours=6), movie_background_field=fieldset.U) + else: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=-dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12)) + else: + # ==== forward simulation ==== # + if args.animate: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12), moviedt=delta(hours=6), movie_background_field=fieldset.U) + else: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12)) + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + endtime = ostime.process_time() + else: + endtime = ostime.process_time() + + if args.write_out: + output_file.close() + + if not args.dryrun: + if MPI: + mpi_comm = MPI.COMM_WORLD + # mpi_comm.Barrier() + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart-1) + Npart = mpi_comm.reduce(Npart, op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + if size_Npart>0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime-starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time*1000.0)) + else: + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart-1) + if size_Npart > 0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime - starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time * 1000.0)) + + if MPI: + mpi_comm = MPI.COMM_WORLD + Nparticles = mpi_comm.reduce(np.array(pset.nparticle_log.get_params()), op=MPI.SUM, root=0) + Nmem = mpi_comm.reduce(np.array(pset.mem_log.get_params()), op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + pset.plot_and_log(memory_used=Nmem, nparticles=Nparticles, target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 150]) + else: + pset.plot_and_log(target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 150]) + + diff --git a/performance/benchmark_galapagos_backwards.py b/performance/benchmark_galapagos_backwards.py new file mode 100644 index 0000000000..401e17df48 --- /dev/null +++ b/performance/benchmark_galapagos_backwards.py @@ -0,0 +1,222 @@ +from parcels import FieldSet, JITParticle, AdvectionRK4, ErrorCode, Variable +from parcels import ParticleSet_Benchmark +from datetime import timedelta as delta +from glob import glob +import numpy as np +import itertools +import matplotlib.pyplot as plt +import xarray as xr +import warnings +import math +import sys +import os +import gc +from argparse import ArgumentParser +import fnmatch +import time as ostime +#import dask +warnings.simplefilter("ignore", category=xr.SerializationWarning) + +try: + from mpi4py import MPI +except: + MPI = None + + +def create_galapagos_fieldset(datahead, periodic_wrap, use_stokes): + # dask.config.set({'array.chunk-size': '16MiB'}) + ddir = os.path.join(datahead,"NEMO-MEDUSA/ORCA0083-N006/") + ufiles = sorted(glob(ddir+'means/ORCA0083-N06_20[00-10]*d05U.nc')) + vfiles = [u.replace('05U.nc', '05V.nc') for u in ufiles] + meshfile = glob(ddir+'domain/coordinates.nc') + nemo_files = {'U': {'lon': meshfile, 'lat': meshfile, 'data': ufiles}, + 'V': {'lon': meshfile, 'lat': meshfile, 'data': vfiles}} + nemo_variables = {'U': 'uo', 'V': 'vo'} + nemo_dimensions = {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'} + period = delta(days=366*11) if periodic_wrap else False # 10 years period + extrapolation = False if periodic_wrap else True + # ==== Because the stokes data is a different grid, we actually need to define the chunking ==== # + # fieldset_nemo = FieldSet.from_nemo(nemofiles, nemovariables, nemodimensions, field_chunksize='auto') + nemo_chs = {'time_counter': 1, 'depthu': 75, 'depthv': 75, 'depthw': 75, 'deptht': 75, 'y': 100, 'x': 100} + fieldset_nemo = FieldSet.from_nemo(nemo_files, nemo_variables, nemo_dimensions, field_chunksize=nemo_chs, time_periodic=period, allow_time_extrapolation=extrapolation) + + if wstokes: + stokes_files = sorted(glob(datahead+"/WaveWatch3data/CFSR/WW3-*_uss.nc")) + stokes_variables = {'U': 'uuss', 'V': 'vuss'} + stokes_dimensions = {'lat': 'latitude', 'lon': 'longitude', 'time': 'time'} + stokes_chs = {'time': 1, 'latitude': 16, 'longitude': 32} + stokes_period = delta(days=366+2*31) if periodic_wrap else False # 14 month period + fieldset_stokes = FieldSet.from_netcdf(stokes_files, stokes_variables, stokes_dimensions, field_chunksize=stokes_chs, time_periodic=period, allow_time_extrapolation=extrapolation) + fieldset_stokes.add_periodic_halo(zonal=True, meridional=False, halosize=5) + + fieldset = FieldSet(U=fieldset_nemo.U+fieldset_stokes.U, V=fieldset_nemo.V+fieldset_stokes.V) + fU = fieldset.U[0] + else: + fieldset = fieldset_nemo + fU = fieldset.U + + return fieldset, fU + + +def perIterGC(): + gc.collect() + + +class GalapagosParticle(JITParticle): + age = Variable('age', initial=0.) + life_expectancy = Variable('life_expectancy', dtype=np.float64, initial=14.0*86400.0) # np.finfo(np.float64).max + + +def Age(particle, fieldset, time): + if particle.state == ErrorCode.Evaluate: + particle.age = particle.age + math.fabs(particle.dt) + if particle.age > particle.life_expectancy: + particle.delete() + + +def DeleteParticle(particle, fieldset, time): + particle.delete() + + +def periodicBC(particle, fieldSet, time): + dlon = -89.0 + 91.8 + dlat = 0.7 + 1.4 + if particle.lon > -89.0: + particle.lon -= dlon + if particle.lon < -91.8: + particle.lon += dlon + if particle.lat > 0.7: + particle.lat -= dlat + if particle.lat < -1.4: + particle.lat += dlat + + +if __name__=='__main__': + parser = ArgumentParser(description="Example of particle advection around an idealised peninsula") + parser.add_argument("-s", "--stokes", dest="stokes", action='store_true', default=False, help="use Stokes' field data") + parser.add_argument("-i", "--imageFileName", dest="imageFileName", type=str, default="mpiChunking_plot_MPI.png", help="image file name of the plot") + parser.add_argument("-p", "--periodic", dest="periodic", action='store_true', default=False, help="enable/disable periodic wrapping (else: extrapolation)") + parser.add_argument("-w", "--writeout", dest="write_out", action='store_true', default=False, help="write data in outfile") + # parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=int, default=365, help="runtime in days (default: 365)") + parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=str, default="1*365", help="runtime in days (default: 1*365)") + parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + args = parser.parse_args() + + wstokes = args.stokes + imageFileName=args.imageFileName + periodicFlag=args.periodic + time_in_days = int(float(eval(args.time_in_days))) + with_GC = args.useGC + + branch = "benchmarking" + computer_env = "local/unspecified" + scenario = "galapagos_backwards" + headdir = "" + odir = "" + datahead = "" + dirread_top = "" + dirread_top_bgc = "" + dirread_mesh = "" + if os.uname()[1] in ['science-bs35', 'science-bs36']: # Gemini + # headdir = "/scratch/{}/experiments/palaeo-parcels".format(os.environ['USER']) + headdir = "/scratch/{}/experiments/galapagos".format("ckehl") + odir = os.path.join(headdir,"BENCHres") + datahead = "/data/oceanparcels/input_data" + ddir_head = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/') + computer_env = "Gemini" + # elif fnmatch.fnmatchcase(os.uname()[1], "int?.*"): # Cartesius + elif fnmatch.fnmatchcase(os.uname()[1], "*.bullx*"): # Cartesius + CARTESIUS_SCRATCH_USERNAME = 'ckehluu' + headdir = "/scratch/shared/{}/experiments/galapagos".format(CARTESIUS_SCRATCH_USERNAME) + odir = os.path.join(headdir, "BENCHres") + datahead = "/projects/0/topios/hydrodynamic_data" + ddir_head = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/') + computer_env = "Cartesius" + else: + headdir = "/var/scratch/galapagos" + odir = os.path.join(headdir, "BENCHres") + datahead = "/data" + ddir_head = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/') + print("running {} on {} (uname: {}) - branch '{}' - argv: {}".format(scenario, computer_env, os.uname()[1], branch, sys.argv[1:])) + + + + + fieldset, fU = create_galapagos_fieldset(datahead, True, wstokes) + fname = os.path.join(odir,"galapagosparticles_bwd_wstokes_v2.nc") if wstokes else os.path.join(odir,"galapagosparticles_bwd_v2.nc") + + galapagos_extent = [-91.8, -89, -1.4, 0.7] + startlon, startlat = np.meshgrid(np.arange(galapagos_extent[0], galapagos_extent[1], 0.2), + np.arange(galapagos_extent[2], galapagos_extent[3], 0.2)) + + print("|lon| = {}; |lat| = {}".format(startlon.shape[0], startlat.shape[0])) + + pset = ParticleSet_Benchmark(fieldset=fieldset, pclass=GalapagosParticle, lon=startlon, lat=startlat, time=fU.grid.time[-1], repeatdt=delta(days=7)) + """ Kernal + Execution""" + postProcessFuncs = [] + if with_GC: + postProcessFuncs.append(perIterGC) + output_fpath = None + outfile = None + if args.write_out: + output_fpath = fname + outfile = pset.ParticleFile(name=output_fpath, outputdt=delta(days=1)) + kernel = pset.Kernel(AdvectionRK4)+pset.Kernel(Age)+pset.Kernel(periodicBC) + + starttime = 0 + endtime = 0 + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + # global_t_0 = ostime.time() + # starttime = MPI.Wtime() + starttime = ostime.process_time() + else: + #starttime = ostime.time() + starttime = ostime.process_time() + + pset.execute(kernel, dt=delta(hours=-1), output_file=outfile, recovery={ErrorCode.ErrorOutOfBounds: DeleteParticle}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(days=1)) + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + # global_t_0 = ostime.time() + # endtime = MPI.Wtime() + endtime = ostime.process_time() + else: + # endtime = ostime.time() + endtime = ostime.process_time() + + if args.write_out: + outfile.close() + + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart - 1) + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_comm.Barrier() + Npart = mpi_comm.reduce(Npart, op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + if size_Npart>0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime-starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time*1000.0)) + else: + if size_Npart > 0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime - starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time * 1000.0)) + + if MPI: + mpi_comm = MPI.COMM_WORLD + # mpi_comm.Barrier() + Nparticles = mpi_comm.reduce(np.array(pset.nparticle_log.get_params()), op=MPI.SUM, root=0) + Nmem = mpi_comm.reduce(np.array(pset.mem_log.get_params()), op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + pset.plot_and_log(memory_used=Nmem, nparticles=Nparticles, target_N=1, imageFilePath=imageFileName, odir=odir) + else: + pset.plot_and_log(target_N=1, imageFilePath=imageFileName, odir=odir) diff --git a/performance/benchmark_palaeo_Y2K.py b/performance/benchmark_palaeo_Y2K.py new file mode 100644 index 0000000000..4150891ee2 --- /dev/null +++ b/performance/benchmark_palaeo_Y2K.py @@ -0,0 +1,452 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Oct 13 15:31:22 2017 + +@author: nooteboom +""" +import itertools + +from parcels import (FieldSet, JITParticle, AdvectionRK4_3D, + Field, ErrorCode, ParticleFile, Variable) +# from parcels import ParticleSet +from parcels import ParticleSet_Benchmark + +from argparse import ArgumentParser +from datetime import timedelta as delta +from datetime import datetime +import numpy as np +import math +from glob import glob +import sys +import pandas as pd + +import os +import time as ostime +import matplotlib.pyplot as plt +import fnmatch + +# import dask +import gc + +try: + from mpi4py import MPI +except: + MPI = None + +import warnings +import xarray as xr +warnings.simplefilter("ignore", category=xr.SerializationWarning) + +global_t_0 = 0 +odir = "" + + +def set_nemo_fieldset(ufiles, vfiles, wfiles, tfiles, pfiles, dfiles, ifiles, bfile, mesh_mask='/scratch/ckehl/experiments/palaeo-parcels/NEMOdata/domain/coordinates.nc', periodicFlag=False): + filenames = { 'U': {'lon': mesh_mask, + 'lat': mesh_mask, + 'depth': [ufiles[0]], + 'data':ufiles}, + 'V' : {'lon': mesh_mask, + 'lat': mesh_mask, + 'depth': [ufiles[0]], + 'data':vfiles}, + 'W' : {'lon': mesh_mask, + 'lat': mesh_mask, + 'depth': [ufiles[0]], + 'data':wfiles}, + 'S' : {'lon': mesh_mask, + 'lat': mesh_mask, + 'depth': [tfiles[0]], + 'data':tfiles}, + 'T' : {'lon': mesh_mask, + 'lat': mesh_mask, + 'depth': [tfiles[0]], + 'data':tfiles}, + 'NO3':{'lon': mesh_mask, + 'lat': mesh_mask, + 'depth': [pfiles[0]], + 'data':pfiles}, + 'PP':{'lon': mesh_mask, + 'lat': mesh_mask, + 'depth': [dfiles[0]], + 'data':dfiles}, + 'ICE':{'lon': mesh_mask, + 'lat': mesh_mask, + 'depth': [ifiles[0]], + 'data':ifiles}, + 'ICEPRES':{'lon': mesh_mask, + 'lat': mesh_mask, + 'depth': [ifiles[0]], + 'data':ifiles}, + 'CO2':{'lon': mesh_mask, + 'lat': mesh_mask, + 'depth': [dfiles[0]], + 'data':dfiles}, + } + if mesh_mask: + filenames['mesh_mask'] = mesh_mask + variables = {'U': 'uo', + 'V': 'vo', + 'W': 'wo', + 'T': 'sst', + 'S': 'sss', + 'NO3': 'DIN', + 'PP': 'TPP3', + 'ICE': 'sit', + 'ICEPRES': 'ice_pres', + 'CO2': 'TCO2' } + + dimensions = {'U': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthu', 'time': 'time_counter'}, # + 'V': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthu', 'time': 'time_counter'}, # + 'W': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthu', 'time': 'time_counter'}, # + 'T': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, + 'S': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, + 'NO3': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, + 'PP': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, + 'ICE': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, + 'ICEPRES': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, + 'CO2': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'} } #, + bfiles = {'lon': mesh_mask, 'lat': mesh_mask, 'data': [bfile, ]} + bvariables = ('B', 'Bathymetry') + bdimensions = {'lon': 'glamf', 'lat': 'gphif'} + bchs = False + + # ==== depth-split need to be 75 so all (2D- and 3D) fields are chunked the same way + chs = {'time_counter': 1, 'depthu': 80, 'depthv': 80, 'depthw': 80, 'deptht': 80, 'y': 200, 'x': 200} + # chs = {'U': {'depthu': 80, 'depthv': 80, 'depthw': 80, 'deptht': 80, 'y': 64, 'x': 128, 'time_counter': 1}, + # 'V': {'depthu': 80, 'depthv': 80, 'depthw': 80, 'deptht': 80, 'y': 64, 'x': 128, 'time_counter': 1}, + # 'W': {'depthu': 80, 'depthv': 80, 'depthw': 80, 'deptht': 80, 'y': 64, 'x': 128, 'time_counter': 1}, + # 'T': {'x': 128, 'y': 64, 'deptht': 80, 'time_counter': 1}, # tfiles + # 'S': {'x': 128, 'y': 64, 'deptht': 80, 'time_counter': 1}, # tfiles + # 'NO3': {'x': 128, 'y': 64, 'deptht': 80, 'time_counter': 1}, # pfiles + # 'PP': {'x': 128, 'y': 64, 'deptht': 80, 'time_counter': 1}, # dfiles + # 'ICE': False, # ifiles + # 'ICEPRES': False, # ifiles + # 'CO2': {'x': 128, 'y': 64, 'deptht': 80, 'time_counter': 1}, # dfiles + # } + + #chs = (1, 75, 200, 200) + + #dask.config.set({'array.chunk-size': '6MiB'}) + #chs = 'auto' + nchs = { + 'U': {'lon': ('x', 64), 'lat': ('y', 32), 'depth': ('depthu', 25), 'time': ('time_counter', 1)}, # ufiles + 'V': {'lon': ('x', 64), 'lat': ('y', 32), 'depth': ('depthv', 25), 'time': ('time_counter', 1)}, # vfiles + 'W': {'lon': ('x', 64), 'lat': ('y', 32), 'depth': ('depthw', 25), 'time': ('time_counter', 1)}, # wfiles + 'T': {'lon': ('x', 64), 'lat': ('y', 32), 'depth': ('deptht', 25), 'time': ('time_counter', 1)}, # tfiles + 'S': {'lon': ('x', 64), 'lat': ('y', 32), 'depth': ('deptht', 25), 'time': ('time_counter', 1)}, # tfiles + 'NO3': {'lon': ('x', 64), 'lat': ('y', 32), 'depth': ('deptht', 25), 'time': ('time_counter', 1)}, # pfiles + 'PP': {'lon': ('x', 64), 'lat': ('y', 32), 'depth': ('deptht', 25), 'time': ('time_counter', 1)}, # dfiles + 'ICE': {'lon': ('x', 64), 'lat': ('y', 32), 'time': ('time_counter', 1)}, # ifiles + 'ICEPRES': {'lon': ('x', 64), 'lat': ('y', 32), 'time': ('time_counter', 1)}, # ifiles + 'CO2': {'lon': ('x', 64), 'lat': ('y', 32), 'depth': ('deptht', 25), 'time': ('time_counter', 1)}, # dfiles + } + + if mesh_mask: # and isinstance(bfile, list) and len(bfile) > 0: + if not periodicFlag: + try: + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, allow_time_extrapolation=True, field_chunksize=chs) + # Tfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + # Sfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + # NO3field = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + # PPfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + # ICEfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + # ICEPRESfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + # CO2field = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + Bfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + except (SyntaxError, ): + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, allow_time_extrapolation=True, chunksize=nchs) + Bfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', chunksize=bchs) + else: + try: + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, time_periodic=delta(days=366), field_chunksize=chs) + Bfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + except (SyntaxError, ): + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, time_periodic=delta(days=366), chunksize=nchs) + Bfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', chunksize=bchs) + fieldset.add_field(Bfield, 'B') + fieldset.U.vmax = 10 + fieldset.V.vmax = 10 + fieldset.W.vmax = 10 + return fieldset + else: + filenames.pop('B') + variables.pop('B') + dimensions.pop('B') + if not periodicFlag: + try: + fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, allow_time_extrapolation=True, field_chunksize=chs) + except (SyntaxError, ): + fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, allow_time_extrapolation=True, chunksize=nchs) + else: + try: + fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, time_periodic=delta(days=366), field_chunksize=chs) + except (SyntaxError, ): + fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, time_periodic=delta(days=366), chunksize=nchs) + fieldset.U.vmax = 10 + fieldset.V.vmax = 10 + fieldset.W.vmax = 10 + return fieldset + + +def periodicBC(particle, fieldSet, time): + if particle.lon > 180: + particle.lon -= 360 + if particle.lon < -180: + particle.lon += 360 + +def Sink(particle, fieldset, time): + if(particle.depth>fieldset.dwellingdepth): + particle.depth = particle.depth + fieldset.sinkspeed * particle.dt + elif(particle.depth<=fieldset.dwellingdepth and particle.depth>1): + particle.depth = fieldset.surface + particle.temp = fieldset.T[time+particle.dt, fieldset.surface, particle.lat, particle.lon] + particle.salin = fieldset.S[time+particle.dt, fieldset.surface, particle.lat, particle.lon] + particle.PP = fieldset.PP[time+particle.dt, fieldset.surface, particle.lat, particle.lon] + particle.NO3 = fieldset.NO3[time+particle.dt, fieldset.surface, particle.lat, particle.lon] + particle.ICE = fieldset.ICE[time+particle.dt, fieldset.surface, particle.lat, particle.lon] + particle.ICEPRES = fieldset.ICEPRES[time+particle.dt, fieldset.surface, particle.lat, particle.lon] + particle.CO2 = fieldset.CO2[time+particle.dt, fieldset.surface, particle.lat, particle.lon] + particle.delete() + +def Age(particle, fieldset, time): + if particle.state == ErrorCode.Evaluate: + particle.age = particle.age + math.fabs(particle.dt) + +def DeleteParticle(particle, fieldset, time): + particle.delete() + +def perIterGC(): + gc.collect() + +def initials(particle, fieldset, time): + if particle.age==0.: + particle.depth = fieldset.B[time, fieldset.surface, particle.lat, particle.lon] + if(particle.depth > 5800.): + particle.age = (particle.depth - 5799.)*fieldset.sinkspeed + particle.depth = 5799. + particle.lon0 = particle.lon + particle.lat0 = particle.lat + particle.depth0 = particle.depth + + +class DinoParticle(JITParticle): + temp = Variable('temp', dtype=np.float32, initial=np.nan) + age = Variable('age', dtype=np.float32, initial=0.) + salin = Variable('salin', dtype=np.float32, initial=np.nan) + lon0 = Variable('lon0', dtype=np.float32, initial=0.) + lat0 = Variable('lat0', dtype=np.float32, initial=0.) + depth0 = Variable('depth0',dtype=np.float32, initial=0.) + PP = Variable('PP',dtype=np.float32, initial=np.nan) + NO3 = Variable('NO3',dtype=np.float32, initial=np.nan) + ICE = Variable('ICE',dtype=np.float32, initial=np.nan) + ICEPRES = Variable('ICEPRES',dtype=np.float32, initial=np.nan) + CO2 = Variable('CO2',dtype=np.float32, initial=np.nan) + +if __name__ == "__main__": + parser = ArgumentParser(description="Example of particle advection using in-memory stommel test case") + parser.add_argument("-i", "--imageFileName", dest="imageFileName", type=str, default="mpiChunking_plot_MPI.png", help="image file name of the plot") + parser.add_argument("-p", "--periodic", dest="periodic", action='store_true', default=False, help="enable/disable periodic wrapping (else: extrapolation)") + parser.add_argument("-sp", "--sinking_speed", dest="sp", type=float, default=11.0, help="set the simulation sinking speed in [m/day] (default: 11.0)") + parser.add_argument("-dd", "--dwelling_depth", dest="dd", type=float, default=10.0, help="set the dwelling depth (i.e. ocean surface depth) in [m] (default: 10.0)") + # parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=int, default=365, help="runtime in days (default: 365)") + parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=str, default="1*365", help="runtime in days (default: 1*365)") + parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + parser.add_argument("-pr", "--profiling", dest="profiling", action='store_true', default=False, help="tells that the profiling of the script is activates") + args = parser.parse_args() + + sp = args.sp # The sinkspeed m/day + dd = args.dd # The dwelling depth + imageFileName=args.imageFileName + periodicFlag=args.periodic + time_in_days = int(float(eval(args.time_in_days))) + time_in_years = int(time_in_days/366.0) + with_GC = args.useGC + + branch = "benchmarking" + computer_env = "local/unspecified" + scenario = "palaeo-parcels" + headdir = "" + odir = "" + dirread_pal = "" + datahead = "" + dirread_top = "" + dirread_top_bgc = "" + if os.uname()[1] in ['science-bs35', 'science-bs36']: # Gemini + # headdir = "/scratch/{}/experiments/palaeo-parcels".format(os.environ['USER']) + headdir = "/scratch/{}/experiments/palaeo-parcels".format("ckehl") + odir = os.path.join(headdir,"BENCHres") + dirread_pal = os.path.join(headdir,'NEMOdata') + datahead = "/data/oceanparcels/input_data" + dirread_top = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/') + dirread_top_bgc = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/') + computer_env = "Gemini" + # elif fnmatch.fnmatchcase(os.uname()[1], "int?.*"): # Cartesius + elif fnmatch.fnmatchcase(os.uname()[1], "*.bullx*"): # Cartesius + CARTESIUS_SCRATCH_USERNAME = 'ckehluu' + headdir = "/scratch/shared/{}/experiments/palaeo-parcels".format(CARTESIUS_SCRATCH_USERNAME) + odir = os.path.join(headdir, "BENCHres") + dirread_pal = os.path.join(headdir,'NEMOdata') + datahead = "/projects/0/topios/hydrodynamic_data" + dirread_top = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/') + dirread_top_bgc = os.path.join(datahead, 'NEMO-MEDUSA_BGC/ORCA0083-N006/') + computer_env = "Cartesius" + else: + headdir = "/var/scratch/nooteboom" + odir = os.path.join(headdir, "BENCHres") + dirread_pal = headdir + datahead = "/data" + dirread_top = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/') + dirread_top_bgc = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/') + + + # dirread_pal = '/projects/0/palaeo-parcels/NEMOdata/' + + outfile = 'grid_dd' + str(int(dd)) + outfile += '_sp' + str(int(sp)) + if periodicFlag: + outfile += '_p' + if time_in_years != 1: + outfile += '_' + str(time_in_years) + 'y' + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_size = mpi_comm.Get_size() + outfile += '_n' + str(mpi_size) + if args.profiling: + outfile += '_prof' + if with_GC: + outfile += '_wGC' + else: + outfile += '_woGC' + dirwrite = os.path.join(odir, "sp%d_dd%d" % (int(sp),int(dd))) + if not os.path.exists(dirwrite): + os.mkdir(dirwrite) + + latsz = np.array(pd.read_csv(os.path.join(headdir,"TF_locationsSurfaceSamples_forPeter.csv")).Latitude.tolist()) + lonsz = np.array(pd.read_csv(os.path.join(headdir,"TF_locationsSurfaceSamples_forPeter.csv")).Longitude.tolist()) + numlocs = np.logical_and(latsz<1000, lonsz<1000) + latsz = latsz[numlocs] + lonsz = lonsz[numlocs] + + assert ~(np.isnan(latsz)).any(), 'locations should not contain any NaN values' + dep = dd * np.ones(latsz.shape) + + # timesz = np.array([datetime(2000, 12, 25) - delta(days=x) for x in range(0,int(365),3)]) + timesz = np.array([datetime(2000, 12, 31) - delta(days=x) for x in range(0, time_in_days, 3)]) + # timesz = np.array([datetime(2000, 12, 25) - delta(days=x) for x in range(0, time_in_days, 3)]) + times = np.empty(shape=(0)) + depths = np.empty(shape=(0)) + lons = np.empty(shape=(0)) + lats = np.empty(shape=(0)) + for i in range(len(timesz)): + lons = np.append(lons,lonsz) + lats = np.append(lats, latsz) + depths = np.append(depths, np.zeros(len(lonsz), dtype=np.float32)) + times = np.append(times, np.full(len(lonsz),timesz[i])) + + print("running {} on {} (uname: {}) - branch '{}' - (target) N: {} - argv: {}".format(scenario, computer_env, os.uname()[1], branch, lons.shape[0], sys.argv[1:])) + + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + # global_t_0 = ostime.time() + # global_t_0 = MPI.Wtime() + global_t_0 = ostime.process_time() + else: + # global_t_0 = ostime.time() + global_t_0 = ostime.process_time() + + ufiles = sorted(glob(dirread_top + 'means/ORCA0083-N06_2000????d05U.nc')) + vfiles = sorted(glob(dirread_top + 'means/ORCA0083-N06_2000????d05V.nc')) + wfiles = sorted(glob(dirread_top + 'means/ORCA0083-N06_2000????d05W.nc')) + tfiles = sorted(glob(dirread_top + 'means/ORCA0083-N06_2000????d05T.nc')) + pfiles = sorted(glob(dirread_top_bgc + 'means/ORCA0083-N06_2000????d05P.nc')) + dfiles = sorted(glob(dirread_top_bgc + 'means/ORCA0083-N06_2000????d05D.nc')) + ifiles = sorted(glob(dirread_top + 'means/ORCA0083-N06_2000????d05I.nc')) + bfile = dirread_top + 'domain/bathymetry_ORCA12_V3.3.nc' + + fieldset = set_nemo_fieldset(ufiles, vfiles, wfiles, tfiles, pfiles, dfiles, ifiles, bfile, os.path.join(dirread_pal, "domain/coordinates.nc"), periodicFlag=periodicFlag) + fieldset.add_periodic_halo(zonal=True) + fieldset.add_constant('dwellingdepth', np.float(dd)) + fieldset.add_constant('sinkspeed', sp/86400.) + fieldset.add_constant('maxage', 300000.*86400) + fieldset.add_constant('surface', 2.5) + + pset = ParticleSet_Benchmark.from_list(fieldset=fieldset, pclass=DinoParticle, lon=lons.tolist(), lat=lats.tolist(), depth=depths.tolist(), time = times) + + """ Kernal + Execution""" + postProcessFuncs = [] + if with_GC: + postProcessFuncs.append(perIterGC) + pfile = pset.ParticleFile(os.path.join(dirwrite, outfile), convert_at_end=True, write_ondelete=True) + kernels = pset.Kernel(initials) + Sink + Age + pset.Kernel(AdvectionRK4_3D) + Age + + starttime = 0 + endtime = 0 + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + # starttime = ostime.time() + # starttime = MPI.Wtime() + starttime = ostime.process_time() + else: + # starttime = ostime.time() + starttime = ostime.process_time() + + # pset.execute(kernels, runtime=delta(days=365*9), dt=delta(minutes=-20), output_file=pfile, verbose_progress=False, + # recovery={ErrorCode.ErrorOutOfBounds: DeleteParticle}, postIterationCallbacks=postProcessFuncs) + # postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12) + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(hours=-12), output_file=pfile, verbose_progress=False, recovery={ErrorCode.ErrorOutOfBounds: DeleteParticle}, postIterationCallbacks=postProcessFuncs, callbackdt=np.infty) + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + # endtime = ostime.time() + # endtime = MPI.Wtime() + endtime = ostime.process_time() + else: + #endtime = ostime.time() + endtime = ostime.process_time() + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_comm.Barrier() + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart - 1) + Npart = mpi_comm.reduce(Npart, op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + if size_Npart>0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime-starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time*1000.0)) + else: + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart-1) + if size_Npart > 0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime - starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time * 1000.0)) + + pfile.close() + + if MPI: + mpi_comm = MPI.COMM_WORLD + # mpi_comm.Barrier() + Nparticles = mpi_comm.reduce(np.array(pset.nparticle_log.get_params()), op=MPI.SUM, root=0) + Nmem = mpi_comm.reduce(np.array(pset.mem_log.get_params()), op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + pset.plot_and_log(memory_used=Nmem, nparticles=Nparticles, target_N=1, imageFilePath=imageFileName, odir=odir) + else: + pset.plot_and_log(target_N=1, imageFilePath=imageFileName, odir=odir) + + print('Execution finished') + + + + diff --git a/performance/benchmark_perlin.py b/performance/benchmark_perlin.py new file mode 100644 index 0000000000..67622b4909 --- /dev/null +++ b/performance/benchmark_perlin.py @@ -0,0 +1,461 @@ +""" +Author: Dr. Christian Kehl +Date: 11-02-2020 +""" + +from parcels import AdvectionEE, AdvectionRK45, AdvectionRK4 +from parcels import FieldSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, ErrorCode +from parcels.particleset_benchmark import ParticleSet_Benchmark as ParticleSet +from parcels.field import Field, VectorField, NestedField, SummedField +# from parcels import plotTrajectoriesFile_loadedField +from parcels import rng as random +from datetime import timedelta as delta +import math +from argparse import ArgumentParser +import datetime +import numpy as np +import xarray as xr +import fnmatch +import sys +import gc +import os +import time as ostime +import matplotlib.pyplot as plt +from parcels.tools import perlin3d +from parcels.tools import perlin2d + +try: + from mpi4py import MPI +except: + MPI = None +with_GC = False + +pset = None +ptype = {'scipy': ScipyParticle, 'jit': JITParticle} +method = {'RK4': AdvectionRK4, 'EE': AdvectionEE, 'RK45': AdvectionRK45} +global_t_0 = 0 +Nparticle = int(math.pow(2,10)) # equals to Nparticle = 1024 +#Nparticle = int(math.pow(2,11)) # equals to Nparticle = 2048 +#Nparticle = int(math.pow(2,12)) # equals to Nparticle = 4096 +#Nparticle = int(math.pow(2,13)) # equals to Nparticle = 8192 +#Nparticle = int(math.pow(2,14)) # equals to Nparticle = 16384 +#Nparticle = int(math.pow(2,15)) # equals to Nparticle = 32768 +#Nparticle = int(math.pow(2,16)) # equals to Nparticle = 65536 +#Nparticle = int(math.pow(2,17)) # equals to Nparticle = 131072 +#Nparticle = int(math.pow(2,18)) # equals to Nparticle = 262144 +#Nparticle = int(math.pow(2,19)) # equals to Nparticle = 524288 + +noctaves=3 +#noctaves=4 # formerly +perlinres=(1,32,8) +shapescale=(4,8,8) +#shapescale=(8,6,6) # formerly +perlin_persistence=0.6 +img_shape = (int(math.pow(2,noctaves))*perlinres[1]*shapescale[1], int(math.pow(2,noctaves))*perlinres[2]*shapescale[2]) +sx = img_shape[0]/1000.0 +sy = img_shape[1]/1000.0 +a = (10.0 * img_shape[0]) +b = (10.0 * img_shape[1]) +tsteps = 61 +tscale = 6 +scalefac = (40.0 / (1000.0/60.0)) # 40 km/h +scalefac /= 1000.0 + +# Idea for 4D: perlin3D creates a time-consistent 3D field +# Thus, we can use skimage to create shifted/rotated/morphed versions +# for the depth domain, so that perlin4D = [depth][time][lat][lon]. +# then, we can do a transpose-op in numpy, to get [time][depth][lat][lon] + +# we need to modify the kernel.execute / pset.execute so that it returns from the JIT +# in a given time WITHOUT writing to disk via outfie => introduce a pyloop_dt + +def DeleteParticle(particle, fieldset, time): + particle.delete() + +def RenewParticle(particle, fieldset, time): + particle.lat = np.random.rand() * a + particle.lon = np.random.rand() * b + +def perIterGC(): + gc.collect() + +def perlin_fieldset_from_numpy(periodic_wrap=False, write_out=False): + """Simulate a current from structured random noise (i.e. Perlin noise). + we use the external package 'perlin-numpy' as field generator, see: + https://github.com/pvigier/perlin-numpy + + Perlin noise was introduced in the literature here: + Perlin, Ken (July 1985). "An Image Synthesizer". SIGGRAPH Comput. Graph. 19 (97–8930), p. 287–296. + doi:10.1145/325165.325247, https://dl.acm.org/doi/10.1145/325334.325247 + """ + + # Coordinates of the test fieldset (on A-grid in deg) + lon = np.linspace(-a*0.5, a*0.5, img_shape[0], dtype=np.float32) + # sys.stdout.write("lon field: {}\n".format(lon.size)) + lat = np.linspace(-b*0.5, b*0.5, img_shape[1], dtype=np.float32) + # sys.stdout.write("lat field: {}\n".format(lat.size)) + totime = tsteps*tscale*24.0*60.0*60.0 + time = np.linspace(0., totime, tsteps, dtype=np.float64) + # sys.stdout.write("time field: {}\n".format(time.size)) + + # Define arrays U (zonal), V (meridional) + U = perlin2d.generate_fractal_noise_temporal2d(img_shape, tsteps, (perlinres[1], perlinres[2]), noctaves, perlin_persistence, max_shift=((-1, 2), (-1, 2))) + U = np.transpose(U, (0,2,1)) + # U = np.swapaxes(U, 1, 2) + # print("U-statistics - min: {:10.7f}; max: {:10.7f}; avg. {:10.7f}; std_dev: {:10.7f}".format(U.min(initial=0), U.max(initial=0), U.mean(), U.std())) + V = perlin2d.generate_fractal_noise_temporal2d(img_shape, tsteps, (perlinres[1], perlinres[2]), noctaves, perlin_persistence, max_shift=((-1, 2), (-1, 2))) + V = np.transpose(V, (0,2,1)) + # V = np.swapaxes(V, 1, 2) + # print("V-statistics - min: {:10.7f}; max: {:10.7f}; avg. {:10.7f}; std_dev: {:10.7f}".format(V.min(initial=0), V.max(initial=0), V.mean(), V.std())) + + # U = perlin3d.generate_fractal_noise_3d(img_shape, perlinres, noctaves, perlin_persistence) * scalefac + # U = np.transpose(U, (0,2,1)) + # sys.stdout.write("U field shape: {} - [tdim][ydim][xdim]=[{}][{}][{}]\n".format(U.shape, time.shape[0], lat.shape[0], lon.shape[0])) + # V = perlin3d.generate_fractal_noise_3d(img_shape, perlinres, noctaves, perlin_persistence) * scalefac + # V = np.transpose(V, (0,2,1)) + # sys.stdout.write("V field shape: {} - [tdim][ydim][xdim]=[{}][{}][{}]\n".format(V.shape, time.shape[0], lat.shape[0], lon.shape[0])) + + U *= scalefac + V *= scalefac + + data = {'U': U, 'V': V} + dimensions = {'time': time, 'lon': lon, 'lat': lat} + fieldset = None + if periodic_wrap: + fieldset = FieldSet.from_data(data, dimensions, mesh='flat', transpose=False, time_periodic=delta(days=366)) + else: + fieldset = FieldSet.from_data(data, dimensions, mesh='flat', transpose=False, allow_time_extrapolation=True) + if write_out: + fieldset.write(filename=write_out) + return fieldset + + +def perlin_fieldset_from_xarray(periodic_wrap=False): + """Simulate a current from structured random noise (i.e. Perlin noise). + we use the external package 'perlin-numpy' as field generator, see: + https://github.com/pvigier/perlin-numpy + + Perlin noise was introduced in the literature here: + Perlin, Ken (July 1985). "An Image Synthesizer". SIGGRAPH Comput. Graph. 19 (97–8930), p. 287–296. + doi:10.1145/325165.325247, https://dl.acm.org/doi/10.1145/325334.325247 + """ + img_shape = (perlinres[0]*shapescale[0], int(math.pow(2,noctaves))*perlinres[1]*shapescale[1], int(math.pow(2,noctaves))*perlinres[2]*shapescale[2]) + + # Coordinates of the test fieldset (on A-grid in deg) + lon = np.linspace(0, a, img_shape[1], dtype=np.float32) + lat = np.linspace(0, b, img_shape[2], dtype=np.float32) + totime = img_shape[0] * 24.0 * 60.0 * 60.0 + time = np.linspace(0., totime, img_shape[0], dtype=np.float32) + + # Define arrays U (zonal), V (meridional), W (vertical) and P (sea + # surface height) all on A-grid + U = perlin3d.generate_fractal_noise_3d(img_shape, perlinres, noctaves, perlin_persistence) * scalefac + U = np.transpose(U, (0,2,1)) + V = perlin3d.generate_fractal_noise_3d(img_shape, perlinres, noctaves, perlin_persistence) * scalefac + V = np.transpose(V, (0,2,1)) + + dimensions = {'time': time, 'lon': lon, 'lat': lat} + dims = ('time', 'lat', 'lon') + data = {'Uxr': xr.DataArray(U, coords=dimensions, dims=dims), + 'Vxr': xr.DataArray(V, coords=dimensions, dims=dims)} #,'Pxr': xr.DataArray(P, coords=dimensions, dims=dims) + ds = xr.Dataset(data) + + variables = {'U': 'Uxr', 'V': 'Vxr'} + dimensions = {'time': 'time', 'lat': 'lat', 'lon': 'lon'} + if periodic_wrap: + return FieldSet.from_xarray_dataset(ds, variables, dimensions, mesh='flat', time_periodic=delta(days=1)) + else: + return FieldSet.from_xarray_dataset(ds, variables, dimensions, mesh='flat', allow_time_extrapolation=True) + +class AgeParticle_JIT(JITParticle): + age = Variable('age', dtype=np.float64, initial=0.0) + life_expectancy = Variable('life_expectancy', dtype=np.float64, initial=np.finfo(np.float64).max) + initialized_dynamic = Variable('initialized_dynamic', dtype=np.int32, initial=0) + +class AgeParticle_SciPy(ScipyParticle): + age = Variable('age', dtype=np.float64, initial=0.0) + life_expectancy = Variable('life_expectancy', dtype=np.float64, initial=np.finfo(np.float64).max) + initialized_dynamic = Variable('initialized_dynamic', dtype=np.int32, initial=0) + +def initialize(particle, fieldset, time): + if particle.initialized_dynamic < 1: + np_scaler = math.sqrt(3.0 / 2.0) + # np_scaler = 3.0 / 2.0 + # np_scaler = (3.0 / 2.0)**2.0 + particle.life_expectancy = time + random.uniform(.0, (fieldset.life_expectancy-time) * 2.0 / np_scaler) + # particle.life_expectancy = time + random.uniform(.0, fieldset.life_expectancy) * np_scaler + particle.initialized_dynamic = 1 + +def Age(particle, fieldset, time): + if particle.state == ErrorCode.Evaluate: + particle.age = particle.age + math.fabs(particle.dt) + if particle.age > particle.life_expectancy: + particle.delete() + +age_ptype = {'scipy': AgeParticle_SciPy, 'jit': AgeParticle_JIT} + +if __name__=='__main__': + parser = ArgumentParser(description="Example of particle advection using in-memory stommel test case") + parser.add_argument("-i", "--imageFileName", dest="imageFileName", type=str, default="mpiChunking_plot_MPI.png", help="image file name of the plot") + parser.add_argument("-b", "--backwards", dest="backwards", action='store_true', default=False, help="enable/disable running the simulation backwards") + parser.add_argument("-p", "--periodic", dest="periodic", action='store_true', default=False, help="enable/disable periodic wrapping (else: extrapolation)") + parser.add_argument("-r", "--release", dest="release", action='store_true', default=False, help="continuously add particles via repeatdt (default: False)") + parser.add_argument("-rt", "--releasetime", dest="repeatdt", type=int, default=720, help="repeating release rate of added particles in Minutes (default: 720min = 12h)") + parser.add_argument("-a", "--aging", dest="aging", action='store_true', default=False, help="Removed aging particles dynamically (default: False)") + parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=int, default=1, help="runtime in days (default: 1)") + parser.add_argument("-x", "--xarray", dest="use_xarray", action='store_true', default=False, help="use xarray as data backend") + parser.add_argument("-w", "--writeout", dest="write_out", action='store_true', default=False, help="write data in outfile") + parser.add_argument("-d", "--delParticle", dest="delete_particle", action='store_true', default=False, help="switch to delete a particle (True) or reset a particle (default: False).") + parser.add_argument("-A", "--animate", dest="animate", action='store_true', default=False, help="animate the particle trajectories during the run or not (default: False).") + parser.add_argument("-V", "--visualize", dest="visualize", action='store_true', default=False, help="Visualize particle trajectories at the end (default: False). Requires -w in addition to take effect.") + parser.add_argument("-N", "--n_particles", dest="nparticles", type=str, default="2**6", help="number of particles to generate and advect (default: 2e6)") + parser.add_argument("-sN", "--start_n_particles", dest="start_nparticles", type=str, default="96", help="(optional) number of particles generated per release cycle (if --rt is set) (default: 96)") + parser.add_argument("-m", "--mode", dest="compute_mode", choices=['jit','scipy'], default="jit", help="computation mode = [JIT, SciPp]") + parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + args = parser.parse_args() + + imageFileName=args.imageFileName + periodicFlag=args.periodic + backwardSimulation = args.backwards + repeatdtFlag=args.release + repeatRateMinutes=args.repeatdt + time_in_days = args.time_in_days + use_xarray = args.use_xarray + agingParticles = args.aging + with_GC = args.useGC + Nparticle = int(float(eval(args.nparticles))) + target_N = Nparticle + addParticleN = 1 + # np_scaler = math.sqrt(3.0/2.0) + # np_scaler = (3.0 / 2.0)**2.0 # ** + np_scaler = 3.0 / 2.0 + # cycle_scaler = math.sqrt(3.0/2.0) + # cycle_scaler = (3.0 / 2.0)**2.0 # ** + # cycle_scaler = 3.0 / 2.0 + cycle_scaler = 7.0 / 4.0 + start_N_particles = int(float(eval(args.start_nparticles))) + if MPI: + mpi_comm = MPI.COMM_WORLD + if mpi_comm.Get_rank() == 0: + if agingParticles and not repeatdtFlag: + sys.stdout.write("N: {} ( {} )\n".format(Nparticle, int(Nparticle * np_scaler))) + else: + sys.stdout.write("N: {}\n".format(Nparticle)) + else: + if agingParticles and not repeatdtFlag: + sys.stdout.write("N: {} ( {} )\n".format(Nparticle, int(Nparticle * np_scaler))) + else: + sys.stdout.write("N: {}\n".format(Nparticle)) + + dt_minutes = 60 + #dt_minutes = 20 + #random.seed(123456) + nowtime = datetime.datetime.now() + random.seed(nowtime.microsecond) + + branch = "benchmarking" + computer_env = "local/unspecified" + scenario = "perlin" + odir = "" + if os.uname()[1] in ['science-bs35', 'science-bs36']: # Gemini + # odir = "/scratch/{}/experiments".format(os.environ['USER']) + odir = "/scratch/{}/experiments".format("ckehl") + computer_env = "Gemini" + # elif fnmatch.fnmatchcase(os.uname()[1], "int?.*"): # Cartesius + elif fnmatch.fnmatchcase(os.uname()[1], "*.bullx*"): # Cartesius + CARTESIUS_SCRATCH_USERNAME = 'ckehluu' + odir = "/scratch/shared/{}/experiments".format(CARTESIUS_SCRATCH_USERNAME) + computer_env = "Cartesius" + else: + odir = "/var/scratch/experiments" + print("running {} on {} (uname: {}) - branch '{}' - (target) N: {} - argv: {}".format(scenario, computer_env, os.uname()[1], branch, target_N, sys.argv[1:])) + + if os.path.sep in imageFileName: + head_dir = os.path.dirname(imageFileName) + if head_dir[0] == os.path.sep: + odir = head_dir + else: + odir = os.path.join(odir, head_dir) + imageFileName = os.path.split(imageFileName)[1] + + func_time = [] + mem_used_GB = [] + + np.random.seed(0) + fieldset = None + if use_xarray: + fieldset = perlin_fieldset_from_xarray(periodic_wrap=periodicFlag) + else: + field_fpath = False + if args.write_out: + field_fpath = os.path.join(odir,"perlin") + fieldset = perlin_fieldset_from_numpy(periodic_wrap=periodicFlag, write_out=field_fpath) + + if args.compute_mode is 'scipy': + Nparticle = 2**10 + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + global_t_0 = ostime.process_time() + else: + global_t_0 = ostime.process_time() + + simStart = None + for f in fieldset.get_fields(): + if type(f) in [VectorField, NestedField, SummedField]: # or not f.grid.defer_load + continue + else: + if backwardSimulation: + simStart=f.grid.time_full[-1] + else: + simStart = f.grid.time_full[0] + break + + if agingParticles: + if not repeatdtFlag: + Nparticle = int(Nparticle * np_scaler) + fieldset.add_constant('life_expectancy', delta(days=time_in_days).total_seconds()) + if repeatdtFlag: + addParticleN = Nparticle/2.0 + refresh_cycle = (delta(days=time_in_days).total_seconds() / (addParticleN/start_N_particles)) / cycle_scaler + if agingParticles: + refresh_cycle /= cycle_scaler + repeatRateMinutes = int(refresh_cycle/60.0) if repeatRateMinutes == 720 else repeatRateMinutes + + if backwardSimulation: + # ==== backward simulation ==== # + if agingParticles: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) + else: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) + else: + # ==== forward simulation ==== # + if agingParticles: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) + else: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) + + output_file = None + out_fname = "benchmark_perlin" + if args.write_out: + if MPI and (MPI.COMM_WORLD.Get_size()>1): + out_fname += "_MPI" + else: + out_fname += "_noMPI" + if periodicFlag: + out_fname += "_p" + out_fname += "_n"+str(Nparticle) + if backwardSimulation: + out_fname += "_bwd" + else: + out_fname += "_fwd" + if repeatdtFlag: + out_fname += "_add" + if agingParticles: + out_fname += "_age" + output_file = pset.ParticleFile(name=os.path.join(odir,out_fname+".nc"), outputdt=delta(hours=24)) + delete_func = RenewParticle + if args.delete_particle: + delete_func=DeleteParticle + postProcessFuncs = [] + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + starttime = ostime.process_time() + else: + starttime = ostime.process_time() + kernels = pset.Kernel(AdvectionRK4,delete_cfiles=True) + if agingParticles: + kernels += pset.Kernel(initialize, delete_cfiles=True) + kernels += pset.Kernel(Age, delete_cfiles=True) + if with_GC: + postProcessFuncs.append(perIterGC) + if backwardSimulation: + # ==== backward simulation ==== # + if args.animate: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=-dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12), moviedt=delta(hours=6), movie_background_field=fieldset.U) + else: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=-dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12)) + else: + # ==== forward simulation ==== # + if args.animate: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12), moviedt=delta(hours=6), movie_background_field=fieldset.U) + else: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12)) + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + endtime = ostime.process_time() + else: + endtime = ostime.process_time() + + if args.write_out: + output_file.close() + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_comm.Barrier() + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart - 1) + Npart = mpi_comm.reduce(Npart, op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + if size_Npart>0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime-starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time*1000.0)) + else: + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart-1) + if size_Npart > 0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime - starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time * 1000.0)) + + # if args.write_out: + # output_file.close() + # if args.visualize: + # if MPI: + # mpi_comm = MPI.COMM_WORLD + # if mpi_comm.Get_rank() == 0: + # plotTrajectoriesFile_loadedField(os.path.join(odir, out_fname+".nc"), tracerfield=fieldset.U) + # else: + # plotTrajectoriesFile_loadedField(os.path.join(odir, out_fname+".nc"),tracerfield=fieldset.U) + + if MPI: + mpi_comm = MPI.COMM_WORLD + # mpi_comm.Barrier() + Nparticles = mpi_comm.reduce(np.array(pset.nparticle_log.get_params()), op=MPI.SUM, root=0) + Nmem = mpi_comm.reduce(np.array(pset.mem_log.get_params()), op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + pset.plot_and_log(memory_used=Nmem, nparticles=Nparticles, target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 120]) + else: + pset.plot_and_log(target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 150]) + + diff --git a/performance/benchmark_stommel.py b/performance/benchmark_stommel.py new file mode 100644 index 0000000000..a336cacf2d --- /dev/null +++ b/performance/benchmark_stommel.py @@ -0,0 +1,505 @@ +""" +Author: Dr. Christian Kehl +Date: 11-02-2020 +""" + +from parcels import AdvectionEE, AdvectionRK45, AdvectionRK4 +from parcels import FieldSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, ErrorCode +from parcels.particleset_benchmark import ParticleSet_Benchmark as ParticleSet +# from parcels.kernel_benchmark import Kernel_Benchmark as Kernel +from parcels.field import VectorField, NestedField, SummedField +# from parcels import plotTrajectoriesFile_loadedField +from datetime import timedelta as delta +import math +from argparse import ArgumentParser +import datetime +import numpy as np +import xarray as xr +# import pytest +import fnmatch +import gc +import os +import time as ostime +import matplotlib.pyplot as plt + +from parcels import rng as random + +import sys +try: + from mpi4py import MPI +except: + MPI = None +with_GC = False + +pset = None + + +# ptype = {'scipy': ScipyParticle, 'jit': JITParticle} +method = {'RK4': AdvectionRK4, 'EE': AdvectionEE, 'RK45': AdvectionRK45} +global_t_0 = 0 +Nparticle = int(math.pow(2,10)) # equals to Nparticle = 1024 +#Nparticle = int(math.pow(2,11)) # equals to Nparticle = 2048 +#Nparticle = int(math.pow(2,12)) # equals to Nparticle = 4096 +#Nparticle = int(math.pow(2,13)) # equals to Nparticle = 8192 +#Nparticle = int(math.pow(2,14)) # equals to Nparticle = 16384 +#Nparticle = int(math.pow(2,15)) # equals to Nparticle = 32768 +#Nparticle = int(math.pow(2,16)) # equals to Nparticle = 65536 +#Nparticle = int(math.pow(2,17)) # equals to Nparticle = 131072 +#Nparticle = int(math.pow(2,18)) # equals to Nparticle = 262144 +#Nparticle = int(math.pow(2,19)) # equals to Nparticle = 524288 + +a = 10000 * 1e3 +b = 10000 * 1e3 +scalefac = 0.05 # to scale for physically meaningful velocities + +def DeleteParticle(particle, fieldset, time): + particle.delete() + +def RenewParticle(particle, fieldset, time): + particle.lat = np.random.rand() * a + particle.lon = np.random.rand() * b + +def perIterGC(): + gc.collect() + +def stommel_fieldset_from_numpy(xdim=200, ydim=200, periodic_wrap=False, write_out=False): + """Simulate a periodic current along a western boundary, with significantly + larger velocities along the western edge than the rest of the region + + The original test description can be found in: N. Fabbroni, 2009, + Numerical Simulation of Passive tracers dispersion in the sea, + Ph.D. dissertation, University of Bologna + http://amsdottorato.unibo.it/1733/1/Fabbroni_Nicoletta_Tesi.pdf + """ + + # Coordinates of the test fieldset (on A-grid in deg) + lon = np.linspace(0, a, xdim, dtype=np.float32) + lat = np.linspace(0, b, ydim, dtype=np.float32) + totime = 366*24.0*60.0*60.0 + time = np.linspace(0., totime, ydim, dtype=np.float64) + + # Define arrays U (zonal), V (meridional), W (vertical) and P (sea + # surface height) all on A-grid + U = np.zeros((lon.size, lat.size, time.size), dtype=np.float32) + V = np.zeros((lon.size, lat.size, time.size), dtype=np.float32) + P = np.zeros((lon.size, lat.size, time.size), dtype=np.float32) + + beta = 2e-11 + r = 1/(11.6*86400) + es = r/(beta*a) + + for i in range(lon.size): + for j in range(lat.size): + xi = lon[i] / a + yi = lat[j] / b + P[i, j, 0] = (1 - math.exp(-xi/es) - xi) * math.pi * np.sin(math.pi*yi)*scalefac + U[i, j, 0] = -(1 - math.exp(-xi/es) - xi) * math.pi**2 * np.cos(math.pi*yi)*scalefac + V[i, j, 0] = (math.exp(-xi/es)/es - 1) * math.pi * np.sin(math.pi*yi)*scalefac + + for t in range(1, time.size): + for i in range(lon.size): + for j in range(lat.size): + P[i, j, t] = P[i, j - 1, t - 1] + U[i, j, t] = U[i, j - 1, t - 1] + V[i, j, t] = V[i, j - 1, t - 1] + + data = {'U': U, 'V': V, 'P': P} + dimensions = {'time': time, 'lon': lon, 'lat': lat} + fieldset = None + if periodic_wrap: + fieldset = FieldSet.from_data(data, dimensions, mesh='flat', transpose=True, time_periodic=delta(days=366)) + else: + fieldset = FieldSet.from_data(data, dimensions, mesh='flat', transpose=True, allow_time_extrapolation=True) + if write_out: + fieldset.write(filename=write_out) + return fieldset + + +def stommel_fieldset_from_xarray(xdim=200, ydim=200, periodic_wrap=False): + """Simulate a periodic current along a western boundary, with significantly + larger velocities along the western edge than the rest of the region + + The original test description can be found in: N. Fabbroni, 2009, + Numerical Simulation of Passive tracers dispersion in the sea, + Ph.D. dissertation, University of Bologna + http://amsdottorato.unibo.it/1733/1/Fabbroni_Nicoletta_Tesi.pdf + """ + # Coordinates of the test fieldset (on A-grid in deg) + lon = np.linspace(0., a, xdim, dtype=np.float32) + lat = np.linspace(0., b, ydim, dtype=np.float32) + totime = 366*24.0*60.0*60.0 + time = np.linspace(0., totime, ydim, dtype=np.float64) + # Define arrays U (zonal), V (meridional), W (vertical) and P (sea + # surface height) all on A-grid + U = np.zeros((lon.size, lat.size, time.size), dtype=np.float32) + V = np.zeros((lon.size, lat.size, time.size), dtype=np.float32) + P = np.zeros((lon.size, lat.size, time.size), dtype=np.float32) + + beta = 2e-11 + r = 1/(11.6*86400) + es = r/(beta*a) + + for i in range(lat.size): + for j in range(lon.size): + xi = lon[j] / a + yi = lat[i] / b + P[i, j, 0] = (1 - math.exp(-xi/es) - xi) * math.pi * np.sin(math.pi*yi)*scalefac + U[i, j, 0] = -(1 - math.exp(-xi/es) - xi) * math.pi**2 * np.cos(math.pi*yi)*scalefac + V[i, j, 0] = (math.exp(-xi/es)/es - 1) * math.pi * np.sin(math.pi*yi)*scalefac + + for t in range(1, time.size): + for i in range(lon.size): + for j in range(lat.size): + P[i, j, t] = P[i, j - 1, t - 1] + U[i, j, t] = U[i, j - 1, t - 1] + V[i, j, t] = V[i, j - 1, t - 1] + + dimensions = {'time': time, 'lon': lon, 'lat': lat} + dims = ('lon', 'lat', 'time') + data = {'Uxr': xr.DataArray(U, coords=dimensions, dims=dims), + 'Vxr': xr.DataArray(V, coords=dimensions, dims=dims), + 'Pxr': xr.DataArray(P, coords=dimensions, dims=dims)} + ds = xr.Dataset(data) + + pvariables = {'U': 'Uxr', 'V': 'Vxr', 'P': 'Pxr'} + pdimensions = {'time': 'time', 'lat': 'lat', 'lon': 'lon'} + if periodic_wrap: + return FieldSet.from_xarray_dataset(ds, pvariables, pdimensions, mesh='flat', time_periodic=delta(days=3661)) + else: + return FieldSet.from_xarray_dataset(ds, pvariables, pdimensions, mesh='flat', allow_time_extrapolation=True) + + +class StommelParticleJ(JITParticle): + p = Variable('p', dtype=np.float32, initial=.0) + p0 = Variable('p0', dtype=np.float32, initial=.0) + +class StommelParticleS(ScipyParticle): + p = Variable('p', dtype=np.float32, initial=.0) + p0 = Variable('p0', dtype=np.float32, initial=.0) + +class AgeParticle_JIT(StommelParticleJ): + age = Variable('age', dtype=np.float64, initial=0.0) + life_expectancy = Variable('life_expectancy', dtype=np.float64, initial=np.finfo(np.float64).max) + initialized_dynamic = Variable('initialized_dynamic', dtype=np.int32, initial=0) + +class AgeParticle_SciPy(StommelParticleS): + age = Variable('age', dtype=np.float64, initial=0.0) + life_expectancy = Variable('life_expectancy', dtype=np.float64, initial=np.finfo(np.float64).max) + initialized_dynamic = Variable('initialized_dynamic', dtype=np.int32, initial=0) + +def initialize(particle, fieldset, time): + if particle.initialized_dynamic < 1: + np_scaler = math.sqrt(3.0 / 2.0) + particle.life_expectancy = time+random.uniform(.0, (fieldset.life_expectancy-time) * 2.0 / np_scaler) + particle.initialized_dynamic = 1 + +def Age(particle, fieldset, time): + if particle.state == ErrorCode.Evaluate: + particle.age = particle.age + math.fabs(particle.dt) + + if particle.age > particle.life_expectancy: + particle.delete() + +ptype = {'scipy': StommelParticleS, 'jit': StommelParticleJ} +age_ptype = {'scipy': AgeParticle_SciPy, 'jit': AgeParticle_JIT} + +if __name__=='__main__': + parser = ArgumentParser(description="Example of particle advection using in-memory stommel test case") + parser.add_argument("-i", "--imageFileName", dest="imageFileName", type=str, default="mpiChunking_plot_MPI.png", help="image file name of the plot") + parser.add_argument("-b", "--backwards", dest="backwards", action='store_true', default=False, help="enable/disable running the simulation backwards") + parser.add_argument("-p", "--periodic", dest="periodic", action='store_true', default=False, help="enable/disable periodic wrapping (else: extrapolation)") + parser.add_argument("-r", "--release", dest="release", action='store_true', default=False, help="continuously add particles via repeatdt (default: False)") + parser.add_argument("-rt", "--releasetime", dest="repeatdt", type=int, default=720, help="repeating release rate of added particles in Minutes (default: 720min = 12h)") + parser.add_argument("-a", "--aging", dest="aging", action='store_true', default=False, help="Removed aging particles dynamically (default: False)") + parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=int, default=1, help="runtime in days (default: 1)") + parser.add_argument("-x", "--xarray", dest="use_xarray", action='store_true', default=False, help="use xarray as data backend") + parser.add_argument("-w", "--writeout", dest="write_out", action='store_true', default=False, help="write data in outfile") + parser.add_argument("-d", "--delParticle", dest="delete_particle", action='store_true', default=False, help="switch to delete a particle (True) or reset a particle (default: False).") + parser.add_argument("-A", "--animate", dest="animate", action='store_true', default=False, help="animate the particle trajectories during the run or not (default: False).") + parser.add_argument("-V", "--visualize", dest="visualize", action='store_true', default=False, help="Visualize particle trajectories at the end (default: False). Requires -w in addition to take effect.") + parser.add_argument("-N", "--n_particles", dest="nparticles", type=str, default="2**6", help="number of particles to generate and advect (default: 2e6)") + parser.add_argument("-sN", "--start_n_particles", dest="start_nparticles", type=str, default="96", help="(optional) number of particles generated per release cycle (if --rt is set) (default: 96)") + parser.add_argument("-m", "--mode", dest="compute_mode", choices=['jit','scipy'], default="jit", help="computation mode = [JIT, SciPp]") + parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + args = parser.parse_args() + + imageFileName=args.imageFileName + periodicFlag=args.periodic + backwardSimulation = args.backwards + repeatdtFlag=args.release + repeatRateMinutes=args.repeatdt + time_in_days = args.time_in_days + use_xarray = args.use_xarray + agingParticles = args.aging + with_GC = args.useGC + Nparticle = int(float(eval(args.nparticles))) + target_N = Nparticle + addParticleN = 1 + np_scaler = 3.0 / 2.0 + cycle_scaler = 7.0 / 4.0 + start_N_particles = int(float(eval(args.start_nparticles))) + if MPI: + mpi_comm = MPI.COMM_WORLD + if mpi_comm.Get_rank() == 0: + if agingParticles and not repeatdtFlag: + sys.stdout.write("N: {} ( {} )\n".format(Nparticle, int(Nparticle * np_scaler))) + else: + sys.stdout.write("N: {}\n".format(Nparticle)) + else: + if agingParticles and not repeatdtFlag: + sys.stdout.write("N: {} ( {} )\n".format(Nparticle, int(Nparticle * np_scaler))) + else: + sys.stdout.write("N: {}\n".format(Nparticle)) + + dt_minutes = 60 + #dt_minutes = 20 + #random.seed(123456) + nowtime = datetime.datetime.now() + random.seed(nowtime.microsecond) + + branch = "benchmarking" + computer_env = "local/unspecified" + scenario = "stommel" + odir = "" + if os.uname()[1] in ['science-bs35', 'science-bs36']: # Gemini + # odir = "/scratch/{}/experiments".format(os.environ['USER']) + odir = "/scratch/{}/experiments".format("ckehl") + computer_env = "Gemini" + # elif fnmatch.fnmatchcase(os.uname()[1], "int?.*"): # Cartesius + elif fnmatch.fnmatchcase(os.uname()[1], "*.bullx*"): # Cartesius + CARTESIUS_SCRATCH_USERNAME = 'ckehluu' + odir = "/scratch/shared/{}/experiments".format(CARTESIUS_SCRATCH_USERNAME) + computer_env = "Cartesius" + else: + odir = "/var/scratch/experiments" + print("running {} on {} (uname: {}) - branch '{}' - (target) N: {} - argv: {}".format(scenario, computer_env, os.uname()[1], branch, target_N, sys.argv[1:])) + + func_time = [] + mem_used_GB = [] + + np.random.seed(0) + fieldset = None + if use_xarray: + fieldset = stommel_fieldset_from_xarray(200, 200, periodic_wrap=periodicFlag) + else: + field_fpath = False + if args.write_out: + field_fpath = os.path.join(odir,"stommel") + fieldset = stommel_fieldset_from_numpy(200, 200, periodic_wrap=periodicFlag, write_out=field_fpath) + + if args.compute_mode is 'scipy': + Nparticle = 2**10 + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + # global_t_0 = ostime.time() + # global_t_0 = MPI.Wtime() + global_t_0 = ostime.process_time() + else: + # global_t_0 = ostime.time() + global_t_0 = ostime.process_time() + + simStart = None + for f in fieldset.get_fields(): + if type(f) in [VectorField, NestedField, SummedField]: # or not f.grid.defer_load + continue + else: + if backwardSimulation: + simStart=f.grid.time_full[-1] + else: + simStart = f.grid.time_full[0] + break + + if agingParticles: + if not repeatdtFlag: + Nparticle = int(Nparticle * np_scaler) + fieldset.add_constant('life_expectancy', delta(days=time_in_days).total_seconds()) + if repeatdtFlag: + addParticleN = Nparticle/2.0 + refresh_cycle = (delta(days=time_in_days).total_seconds() / (addParticleN/start_N_particles)) / cycle_scaler + if agingParticles: + refresh_cycle /= cycle_scaler + repeatRateMinutes = int(refresh_cycle/60.0) if repeatRateMinutes == 720 else repeatRateMinutes + + if backwardSimulation: + # ==== backward simulation ==== # + if agingParticles: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) + else: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) + else: + # ==== forward simulation ==== # + if agingParticles: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) + else: + if repeatdtFlag: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) + psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + pset.add(psetA) + else: + pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) + + # if agingParticles: + # for p in pset.particles: + # p.life_expectancy = delta(days=time_in_days).total_seconds() + # else: + # for p in pset.particles: + # p.initialized_dynamic = 1 + + # available_n_particles = len(pset) + # life = np.random.uniform(delta(hours=24).total_seconds(), delta(days=time_in_days).total_seconds(), available_n_particles) + # i=0 + # for p in pset.particles: + # p.life_expectancy = life[i] + # i += 1 + + output_file = None + out_fname = "benchmark_stommel" + if args.write_out: + if MPI and (MPI.COMM_WORLD.Get_size()>1): + out_fname += "_MPI" + else: + out_fname += "_noMPI" + out_fname += "_n"+str(Nparticle) + if periodicFlag: + out_fname += "_p" + if backwardSimulation: + out_fname += "_bwd" + else: + out_fname += "_fwd" + if repeatdtFlag: + out_fname += "_add" + if agingParticles: + out_fname += "_age" + output_file = pset.ParticleFile(name=os.path.join(odir,out_fname+".nc"), outputdt=delta(hours=24)) + delete_func = RenewParticle + if args.delete_particle: + delete_func=DeleteParticle + postProcessFuncs = [] + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + # starttime = ostime.time() + # starttime = MPI.Wtime() + starttime = ostime.process_time() + else: + # starttime = ostime.time() + starttime = ostime.process_time() + kernels = pset.Kernel(AdvectionRK4,delete_cfiles=True) + if agingParticles: + kernels += pset.Kernel(initialize, delete_cfiles=True) + kernels += pset.Kernel(Age, delete_cfiles=True) + if with_GC: + postProcessFuncs.append(perIterGC) + if backwardSimulation: + # ==== backward simulation ==== # + if args.animate: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=-dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12), moviedt=delta(hours=6), movie_background_field=fieldset.U) + else: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=-dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12)) + else: + # ==== forward simulation ==== # + if args.animate: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12), moviedt=delta(hours=6), movie_background_field=fieldset.U) + else: + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(minutes=dt_minutes), output_file=output_file, recovery={ErrorCode.ErrorOutOfBounds: delete_func}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12)) + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank==0: + # endtime = ostime.time() + # endtime = MPI.Wtime() + endtime = ostime.process_time() + else: + #endtime = ostime.time() + endtime = ostime.process_time() + + if args.write_out: + output_file.close() + + # if MPI: + # mpi_comm = MPI.COMM_WORLD + # if mpi_comm.Get_rank() == 0: + # dt_time = [] + # for i in range(len(perflog.times_steps)): + # if i==0: + # dt_time.append( (perflog.times_steps[i]-global_t_0) ) + # else: + # dt_time.append( (perflog.times_steps[i]-perflog.times_steps[i-1]) ) + # sys.stdout.write("final # particles: {}\n".format(perflog.Nparticles_step[len(perflog.Nparticles_step)-1])) + # sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime-starttime)) + # avg_time = np.mean(np.array(dt_time, dtype=np.float64)) + # sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time*1000.0)) + # else: + # dt_time = [] + # for i in range(len(perflog.times_steps)): + # if i == 0: + # dt_time.append((perflog.times_steps[i] - global_t_0)) + # else: + # dt_time.append((perflog.times_steps[i] - perflog.times_steps[i - 1])) + # sys.stdout.write("final # particles: {}\n".format(perflog.Nparticles_step[len(perflog.Nparticles_step)-1])) + # sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime - starttime)) + # avg_time = np.mean(np.array(dt_time, dtype=np.float64)) + # sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time * 1000.0)) + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_comm.Barrier() + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart - 1) + Npart = mpi_comm.reduce(Npart, op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + if size_Npart>0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime-starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time*1000.0)) + else: + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart-1) + if size_Npart > 0: + sys.stdout.write("final # particles: {}\n".format( Npart )) + sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime - starttime)) + avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) + sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time * 1000.0)) + + # if args.write_out: + # output_file.close() + # if args.visualize: + # if MPI: + # mpi_comm = MPI.COMM_WORLD + # if mpi_comm.Get_rank() == 0: + # plotTrajectoriesFile_loadedField(os.path.join(odir, out_fname+".nc"), tracerfield=fieldset.U) + # else: + # plotTrajectoriesFile_loadedField(os.path.join(odir, out_fname+".nc"),tracerfield=fieldset.U) + + if MPI: + mpi_comm = MPI.COMM_WORLD + # mpi_comm.Barrier() + Nparticles = mpi_comm.reduce(np.array(pset.nparticle_log.get_params()), op=MPI.SUM, root=0) + Nmem = mpi_comm.reduce(np.array(pset.mem_log.get_params()), op=MPI.SUM, root=0) + if mpi_comm.Get_rank() == 0: + pset.plot_and_log(memory_used=Nmem, nparticles=Nparticles, target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 120]) + else: + pset.plot_and_log(target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 150]) + + diff --git a/tests/test_fieldset.py b/tests/test_fieldset.py index 12874885f0..38b0398820 100644 --- a/tests/test_fieldset.py +++ b/tests/test_fieldset.py @@ -2,6 +2,7 @@ from parcels.field import Field, VectorField from parcels.tools.converters import TimeConverter, _get_cftime_calendars, _get_cftime_datetimes, UnitConverter, GeographicPolar import dask.array as da +import dask from datetime import timedelta as delta import datetime import numpy as np @@ -354,6 +355,11 @@ def test_vector_fields(mode, swapUV): @pytest.mark.parametrize('field_chunksize', [False, 'auto', (1, 32, 32)]) @pytest.mark.parametrize('with_GC', [False, True]) def test_from_netcdf_memory_containment(mode, time_periodic, field_chunksize, with_GC): + if field_chunksize == 'auto': + dask.config.set({'array.chunk-size': '2MiB'}) + else: + dask.config.set({'array.chunk-size': '128MiB'}) + class PerformanceLog(): samples = [] memory_steps = [] @@ -432,7 +438,7 @@ def test_from_netcdf_field_chunking(mode, time_periodic, field_chunksize, deferL @pytest.mark.parametrize('datetype', ['float', 'datetime64']) -def test_timestaps(datetype, tmpdir): +def test_timestamps(datetype, tmpdir): data1, dims1 = generate_fieldset(10, 10, 1, 10) data2, dims2 = generate_fieldset(10, 10, 1, 4) if datetype == 'float': diff --git a/tests/test_fieldset_sampling.py b/tests/test_fieldset_sampling.py index 59b985987a..c9abfc5d63 100644 --- a/tests/test_fieldset_sampling.py +++ b/tests/test_fieldset_sampling.py @@ -402,6 +402,153 @@ def test_sampling_out_of_bounds_time(mode, allow_time_extrapolation, k_sample_p, pset.execute(k_sample_p, runtime=0.1, dt=0.1) +@pytest.mark.parametrize('mode', ['jit', 'scipy']) +@pytest.mark.parametrize('npart', [1, 10]) +@pytest.mark.parametrize('chs', [False, 'auto', (10, 10)]) +def test_sampling_multigrids_non_vectorfield_from_file(mode, npart, tmpdir, chs, filename='test_subsets'): + xdim, ydim = 100, 200 + filepath = tmpdir.join(filename) + U = Field('U', np.zeros((ydim, xdim), dtype=np.float32), + lon=np.linspace(0., 1., xdim, dtype=np.float32), + lat=np.linspace(0., 1., ydim, dtype=np.float32)) + V = Field('V', np.zeros((ydim, xdim), dtype=np.float32), + lon=np.linspace(0., 1., xdim, dtype=np.float32), + lat=np.linspace(0., 1., ydim, dtype=np.float32)) + B = Field('B', np.ones((3*ydim, 4*xdim), dtype=np.float32), + lon=np.linspace(0., 1., 4*xdim, dtype=np.float32), + lat=np.linspace(0., 1., 3*ydim, dtype=np.float32)) + fieldset = FieldSet(U, V) + fieldset.add_field(B, 'B') + fieldset.write(filepath) + fieldset = None + + ufiles = [filepath+'U.nc', ] * 4 + vfiles = [filepath+'V.nc', ] * 4 + bfiles = [filepath+'B.nc', ] * 4 + timestamps = np.arange(0, 4, 1) * 86400.0 + timestamps = np.expand_dims(timestamps, 1) + files = {'U': ufiles, 'V': vfiles, 'B': bfiles} + variables = {'U': 'vozocrtx', 'V': 'vomecrty', 'B': 'B'} + dimensions = {'lon': 'nav_lon', 'lat': 'nav_lat'} + fieldset = FieldSet.from_netcdf(files, variables, dimensions, timestamps=timestamps, allow_time_extrapolation=True, + field_chunksize=chs) + + fieldset.add_constant('sample_depth', 2.5) + assert fieldset.U.grid is fieldset.V.grid + assert fieldset.U.grid is not fieldset.B.grid + + class TestParticle(ptype[mode]): + sample_var = Variable('sample_var', initial=0.) + + pset = ParticleSet.from_line(fieldset, pclass=TestParticle, start=[0.3, 0.3], finish=[0.7, 0.7], size=npart) + + def test_sample(particle, fieldset, time): + particle.sample_var += fieldset.B[time, fieldset.sample_depth, particle.lat, particle.lon] + + kernels = pset.Kernel(AdvectionRK4) + pset.Kernel(test_sample) + pset.execute(kernels, runtime=10, dt=1) + assert np.allclose(np.array([p.sample_var for p in pset]), 10.0) + if mode == 'jit': + assert np.all(np.array([len(p.xi) == fieldset.gridset.size for p in pset])) + assert np.all(np.array([[p.xi[i] >= 0 for i in range(0, len(p.xi))] for p in pset])) + #assert np.all(pset.xi[:, fieldset.B.igrid] < xdim * 4) + #assert np.all(pset.xi[:, 0] < xdim) + #assert pset.yi.shape[0] == len(pset.lon) + #assert pset.yi.shape[1] == fieldset.gridset.size + #assert np.all(pset.yi >= 0) + #assert np.all(pset.yi[:, fieldset.B.igrid] < ydim * 3) + #assert np.all(pset.yi[:, 0] < ydim) + + +@pytest.mark.parametrize('mode', ['jit', 'scipy']) +@pytest.mark.parametrize('npart', [1, 10]) +def test_sampling_multigrids_non_vectorfield(mode, npart): + xdim, ydim = 100, 200 + U = Field('U', np.zeros((ydim, xdim), dtype=np.float32), + lon=np.linspace(0., 1., xdim, dtype=np.float32), + lat=np.linspace(0., 1., ydim, dtype=np.float32)) + V = Field('V', np.zeros((ydim, xdim), dtype=np.float32), + lon=np.linspace(0., 1., xdim, dtype=np.float32), + lat=np.linspace(0., 1., ydim, dtype=np.float32)) + B = Field('B', np.ones((3*ydim, 4*xdim), dtype=np.float32), + lon=np.linspace(0., 1., 4*xdim, dtype=np.float32), + lat=np.linspace(0., 1., 3*ydim, dtype=np.float32)) + fieldset = FieldSet(U, V) + fieldset.add_field(B, 'B') + fieldset.add_constant('sample_depth', 2.5) + assert fieldset.U.grid is fieldset.V.grid + assert fieldset.U.grid is not fieldset.B.grid + + class TestParticle(ptype[mode]): + sample_var = Variable('sample_var', initial=0.) + + pset = ParticleSet.from_line(fieldset, pclass=TestParticle, start=[0.3, 0.3], finish=[0.7, 0.7], size=npart) + + def test_sample(particle, fieldset, time): + particle.sample_var += fieldset.B[time, fieldset.sample_depth, particle.lat, particle.lon] + + kernels = pset.Kernel(AdvectionRK4) + pset.Kernel(test_sample) + pset.execute(kernels, runtime=10, dt=1) + assert np.allclose(np.array([p.sample_var for p in pset]), 10.0) + if mode == 'jit': + assert np.all(np.array([len(p.xi) == fieldset.gridset.size for p in pset])) + assert np.all(np.array([[p.xi[i] >= 0 for i in range(0, len(p.xi))] for p in pset])) + #assert np.all(pset.xi[:, fieldset.B.igrid] < xdim * 4) + #assert np.all(pset.xi[:, 0] < xdim) + #assert pset.yi.shape[0] == len(pset.lon) + #assert pset.yi.shape[1] == fieldset.gridset.size + #assert np.all(pset.yi >= 0) + #assert np.all(pset.yi[:, fieldset.B.igrid] < ydim * 3) + #assert np.all(pset.yi[:, 0] < ydim) + + +@pytest.mark.parametrize('mode', ['jit', 'scipy']) +@pytest.mark.parametrize('ugridfactor', [1, 10]) +def test_sampling_multiple_grid_sizes(mode, ugridfactor): + xdim, ydim = 10, 20 + U = Field('U', np.zeros((ydim*ugridfactor, xdim*ugridfactor), dtype=np.float32), + lon=np.linspace(0., 1., xdim*ugridfactor, dtype=np.float32), + lat=np.linspace(0., 1., ydim*ugridfactor, dtype=np.float32)) + V = Field('V', np.zeros((ydim, xdim), dtype=np.float32), + lon=np.linspace(0., 1., xdim, dtype=np.float32), + lat=np.linspace(0., 1., ydim, dtype=np.float32)) + fieldset = FieldSet(U, V) + pset = ParticleSet(fieldset, pclass=pclass(mode), lon=[0.8], lat=[0.9]) + + if ugridfactor > 1: + assert fieldset.U.grid is not fieldset.V.grid + else: + assert fieldset.U.grid is fieldset.V.grid + pset.execute(AdvectionRK4, runtime=10, dt=1) + assert np.isclose(pset[0].lon, 0.8) + assert np.all((pset[0].xi >= 0) & (pset[0].xi < xdim*ugridfactor)) + + +def test_multiple_grid_addlater_error(): + xdim, ydim = 10, 20 + U = Field('U', np.zeros((ydim, xdim), dtype=np.float32), + lon=np.linspace(0., 1., xdim, dtype=np.float32), + lat=np.linspace(0., 1., ydim, dtype=np.float32)) + V = Field('V', np.zeros((ydim, xdim), dtype=np.float32), + lon=np.linspace(0., 1., xdim, dtype=np.float32), + lat=np.linspace(0., 1., ydim, dtype=np.float32)) + fieldset = FieldSet(U, V) + + pset = ParticleSet(fieldset, pclass=pclass('jit'), lon=[0.8], lat=[0.9]) + + P = Field('P', np.zeros((ydim*10, xdim*10), dtype=np.float32), + lon=np.linspace(0., 1., xdim*10, dtype=np.float32), + lat=np.linspace(0., 1., ydim*10, dtype=np.float32)) + fieldset.add_field(P) + + fail = False + try: + pset.execute(AdvectionRK4, runtime=10, dt=1) + except: + fail = True + assert fail + + @pytest.mark.parametrize('mode', ['jit', 'scipy']) def test_sampling_multiple_grid_sizes(mode): """Sampling test that tests for FieldSet with different grid sizes diff --git a/tests/test_kernel_execution.py b/tests/test_kernel_execution.py index 331a74ab1d..800c53eb7c 100644 --- a/tests/test_kernel_execution.py +++ b/tests/test_kernel_execution.py @@ -5,6 +5,13 @@ ) import numpy as np import pytest +from parcels.tools import logger +from os import getpid + +try: + from mpi4py import MPI +except: + MPI = None ptype = {'scipy': ScipyParticle, 'jit': JITParticle} @@ -226,7 +233,7 @@ def MoveEast1(particle, fieldset, time): particle.lon += add_lon def MoveEast2(particle, fieldset, time): - particle.lon += add_lon # NOQA - no flake8 testing of this line + particle.lon += add_lon # noga - no flake8 testing of this line pset = ParticleSet(fieldset, pclass=ptype[mode], lon=[0.5], lat=[0.5]) pset.execute(pset.Kernel(MoveEast1) + pset.Kernel(MoveEast2), @@ -282,3 +289,38 @@ def test_execution_keep_cfiles_and_nocompilation_warnings(fieldset, delete_cfile assert path.exists(cfile) with open(logfile) as f: assert 'warning' not in f.read(), 'Compilation WARNING in log file' + + + + +def run_test_execution_runtime(fset, mode, start, end, substeps, dt, npart=10): + pset = ParticleSet(fset, pclass=ptype[mode], time=start, + lon=np.linspace(0, 1, npart), + lat=np.linspace(1, 0, npart)) + t_step = abs(end - start) / substeps + for _ in range(substeps): + pset.execute(DoNothing, runtime=t_step, dt=dt) + if MPI is None: + assert np.allclose(np.array([p.time for p in pset]), end) + logger.info( + "Completed run (mode={}; start={}; end={}; substep={}; dt={}) on process (pid={})".format(mode, start, end, substeps, dt, getpid())) + else: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + logger.info("Completed run (mode={}; start={}; end={}; substep={}; dt={}) on processor {} (pid={})".format(mode, start, end, substeps, dt, mpi_rank, getpid())) + + +if __name__ == '__main__': + fset = fieldset() + run_test_execution_runtime(fset, 'jit', 0., 10., 1, 1.) + run_test_execution_runtime(fset, 'jit', 0., 10., 4, 1.) + run_test_execution_runtime(fset, 'jit', 0., 10., 1, 3.) + run_test_execution_runtime(fset, 'jit', 2., 16., 5, 3.) + run_test_execution_runtime(fset, 'jit', 20., 10., 4, -1.) + run_test_execution_runtime(fset, 'jit', 20., -10., 7, -2.) + run_test_execution_runtime(fset, 'scipy', 0., 10., 1, 1.) + run_test_execution_runtime(fset, 'scipy', 0., 10., 4, 1.) + run_test_execution_runtime(fset, 'scipy', 0., 10., 1, 3.) + run_test_execution_runtime(fset, 'scipy', 2., 16., 5, 3.) + run_test_execution_runtime(fset, 'scipy', 20., 10., 4, -1.) + run_test_execution_runtime(fset, 'scipy', 20., -10., 7, -2.) diff --git a/tests/test_particle_sets.py b/tests/test_particle_sets.py index 2e488a1563..bda1e55398 100644 --- a/tests/test_particle_sets.py +++ b/tests/test_particle_sets.py @@ -1,7 +1,15 @@ from parcels import (FieldSet, ParticleSet, Field, ScipyParticle, JITParticle, - Variable, ErrorCode, CurvilinearZGrid) + Variable, ErrorCode, CurvilinearZGrid, AdvectionRK4) import numpy as np import pytest +from parcels.compiler import get_cache_dir +from parcels.tools import logger +from os import path + +try: + from mpi4py import MPI +except: + MPI = None ptype = {'scipy': ScipyParticle, 'jit': JITParticle} @@ -422,3 +430,45 @@ class SampleParticle(ptype['scipy']): assert (np.array([p.lon for p in pset]) <= 1).all() test = np.logical_or(np.array([p.lon for p in pset]) <= 0, np.array([p.lat for p in pset]) >= 51) assert test.all() + + def run_test_pset_add_explicit(fset, mode, npart=100): + lon = np.linspace(0, 1, npart, dtype=np.float64) + lat = np.linspace(1, 0, npart, dtype=np.float64) + outfilepath = path.join(get_cache_dir(), "add_explicit", "pfile_node_add_explicit.nc") + pset = ParticleSet(fieldset=fset, pclass=ptype[mode], lon=lon, lat=lat, lonlatdepth_dtype=np.float64) + pfile = pset.ParticleFile(name=outfilepath, outputdt=1.) + + for i in range(npart): + particle = ParticleSet(pclass=ptype[mode], lon=lon[i], lat=lat[i], + fieldset=fieldset, lonlatdepth_dtype=np.float64) + pset.add(particle) + + for tstep in range(3): + pfile.write(pset, tstep) + pfile.close() + logger.info("# particles: {}".format(pset.size)) + assert (pset.size > 0) + assert (pset.size <= 2 * npart) + + def run_test_pset_node_execute(fset, mode, npart=10000): + lon = np.linspace(0, 1, npart, dtype=np.float64) + lat = np.linspace(1, 0, npart, dtype=np.float64) + outfilepath = path.join(get_cache_dir(), "pfile_node_execute_test.nc") + pset = ParticleSet(fieldset=fset, lon=lon, lat=lat, pclass=ptype[mode], lonlatdepth_dtype=np.float64) + pfile = pset.ParticleFile(name=outfilepath, outputdt=360.0) + for i in range(npart): + particle = ParticleSet(pclass=ptype[mode], lon=lon[i], lat=lat[i], + fieldset=fieldset, lonlatdepth_dtype=np.float64) + pset.add(particle) + pset.execute(AdvectionRK4, runtime=0., dt=(360.0 * 24.0 * 10.0), output_file=pfile) + pfile.close() + logger.info("# particles: {}".format(pset.size)) + assert (pset.size > 0) + assert (pset.size <= 2 * npart) + + if __name__ == '__main__': + fset = fieldset() + run_test_pset_add_explicit(fset, 'jit') + run_test_pset_add_explicit(fset, 'scipy') + run_test_pset_node_execute(fset, 'jit') + run_test_pset_node_execute(fset, 'scipy')