diff --git a/parcels/__init__.py b/parcels/__init__.py index 96e6403b04..ef4acbd509 100644 --- a/parcels/__init__.py +++ b/parcels/__init__.py @@ -3,14 +3,19 @@ from parcels.fieldset import * # noqa from parcels.particle import * # noqa -from parcels.particleset import * # noqa -from parcels.particleset_benchmark import * # noqa +from parcels.particleset_vectorized import * # noqa +from parcels.particleset_vectorized_benchmark import * # noqa +# from parcels.particleset_benchmark import * # noqa +# from parcels.field import * # noqa +from parcels.kernel_vectorized import * # noqa from parcels.field import * # noqa -from parcels.kernel import * # noqa import parcels.rng as random # noqa -from parcels.particlefile import * # noqa +# from parcels import rng as random +# from parcels.particlefile import * # noqa +from parcels.particlefile_vectorized import * # noqa from parcels.kernels import * # noqa from parcels.scripts import * # noqa from parcels.gridset import * # noqa from parcels.grid import * # noqa from parcels.tools import * # noqa +from parcels.wrapping import * # noga diff --git a/parcels/compiler.py b/parcels/compiler.py deleted file mode 100644 index 760f6c7b98..0000000000 --- a/parcels/compiler.py +++ /dev/null @@ -1,86 +0,0 @@ -import subprocess -from os import getenv -from os import path -from struct import calcsize -from tempfile import gettempdir - -try: - from os import getuid -except: - # Windows does not have getuid(), so define to simply return 'tmp' - def getuid(): - return 'tmp' -try: - from mpi4py import MPI -except: - MPI = None -from pathlib import Path - - -def get_package_dir(): - return path.abspath(path.dirname(__file__)) - - -def get_cache_dir(): - directory = path.join(gettempdir(), "parcels-%s" % getuid()) - Path(directory).mkdir(exist_ok=True) - return directory - - -class Compiler(object): - """A compiler object for creating and loading shared libraries. - - :arg cc: C compiler executable (uses environment variable ``CC`` if not provided). - :arg cppargs: A list of arguments to the C compiler (optional). - :arg ldargs: A list of arguments to the linker (optional).""" - - def __init__(self, cc=None, cppargs=None, ldargs=None): - if cppargs is None: - cppargs = [] - if ldargs is None: - ldargs = [] - - self._cc = getenv('CC') if cc is None else cc - self._cppargs = cppargs - self._ldargs = ldargs - - def compile(self, src, obj, log): - cc = [self._cc] + self._cppargs + ['-o', obj, src] + self._ldargs - with open(log, 'w') as logfile: - logfile.write("Compiling: %s\n" % " ".join(cc)) - try: - subprocess.check_call(cc, stdout=logfile, stderr=logfile) - except OSError: - err = """OSError during compilation -Please check if compiler exists: %s""" % self._cc - raise RuntimeError(err) - except subprocess.CalledProcessError: - with open(log, 'r') as logfile2: - err = """Error during compilation: -Compilation command: %s -Source file: %s -Log file: %s - -Log output: %s""" % (" ".join(cc), src, logfile.name, logfile2.read()) - raise RuntimeError(err) - - -class GNUCompiler(Compiler): - """A compiler object for the GNU Linux toolchain. - - :arg cppargs: A list of arguments to pass to the C compiler - (optional). - :arg ldargs: A list of arguments to pass to the linker (optional).""" - def __init__(self, cppargs=None, ldargs=None): - if cppargs is None: - cppargs = [] - if ldargs is None: - ldargs = [] - - opt_flags = ['-g', '-O3'] - arch_flag = ['-m64' if calcsize("P") == 8 else '-m32'] - cppargs = ['-Wall', '-fPIC', '-I%s' % path.join(get_package_dir(), 'include')] + opt_flags + cppargs - cppargs += arch_flag - ldargs = ['-shared'] + ldargs + arch_flag - compiler = "mpicc" if MPI else "gcc" - super(GNUCompiler, self).__init__(compiler, cppargs=cppargs, ldargs=ldargs) diff --git a/parcels/field.py b/parcels/field.py index 3394978338..3b72530f4c 100644 --- a/parcels/field.py +++ b/parcels/field.py @@ -492,12 +492,19 @@ def reshape(self, data, transpose=False): data = self.add_periodic_halo(zonal=self.grid.zonal_halo > 0, meridional=self.grid.meridional_halo > 0, halosize=max(self.grid.meridional_halo, self.grid.zonal_halo), data=data) return data + @property + def scaling_factor(self): + return self._scaling_factor + + @scaling_factor.setter + def scaling_factor(self, factor): + self.set_scaling_factor(factor) + def set_scaling_factor(self, factor): """Scales the field data by some constant factor. :param factor: scaling factor """ - if self._scaling_factor: raise NotImplementedError(('Scaling factor for field %s already defined.' % self.name)) self._scaling_factor = factor @@ -507,9 +514,6 @@ def set_scaling_factor(self, factor): def set_depth_from_field(self, field): self.grid.depth_field = field - def __getitem__(self, key): - return self.eval(*key) - def calc_cell_edge_sizes(self): """Method to calculate cell sizes based on numpy.gradient method Currently only works for Rectilinear Grids""" @@ -931,6 +935,9 @@ def depth_index(self, depth, lat, lon): else: return depth_index.argmin() - 1 if depth_index.any() else 0 + def __getitem__(self, key): + return self.eval(*key) + def eval(self, time, z, y, x, applyConversion=True): """Interpolate field values in space and time. diff --git a/parcels/kernel_node.py b/parcels/kernel_node.py new file mode 100644 index 0000000000..9f7a5ee696 --- /dev/null +++ b/parcels/kernel_node.py @@ -0,0 +1,361 @@ +import inspect +import re +import math # noga +import random # noga +from ast import parse +from copy import deepcopy +from ctypes import byref +from ctypes import c_double +from ctypes import c_int +# from ctypes import c_void_p +from ctypes import pointer +from os import path +from sys import version_info +import numpy as np + +from parcels import Field, NestedField, SummedField, VectorField +from parcels import ErrorCode +from parcels.field import FieldOutOfBoundError, FieldOutOfBoundSurfaceError, TimeExtrapolationError +from parcels import AdvectionRK4_3D, logger +from parcels.kernelbase import BaseKernel +from parcels.tools.global_statics import get_cache_dir, get_package_dir +from parcels.wrapping import KernelGenerator, NodeLoopGenerator +from parcels.tools.error import recovery_map as recovery_base_map + +__all__ = ['Kernel'] +DEBUG_MODE = False + + +re_indent = re.compile(r"^(\s+)") + + +def fix_indentation(string): + """Fix indentation to allow in-lined kernel definitions""" + lines = string.split('\n') + indent = re_indent.match(lines[0]) + if indent: + lines = [l.replace(indent.groups()[0], '', 1) for l in lines] + return "\n".join(lines) + + +class Kernel(BaseKernel): + def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, funccode=None, py_ast=None, funcvars=None, + c_include="", delete_cfiles=True): + super(Kernel, self).__init__(fieldset, ptype, pyfunc=pyfunc, funcname=funcname, funccode=funccode, py_ast=py_ast, funcvars=funcvars, c_include=c_include, delete_cfiles=delete_cfiles) + + # Derive meta information from pyfunc, if not given + if pyfunc is AdvectionRK4_3D: # would be better if the idea of a Kernel being '2D', '3D, '4D' or 'uncertain' is captured as Attribute or as class stucture + warning = False + if isinstance(self.fieldset.W, Field) and self.fieldset.W.creation_log != 'from_nemo' and \ + self.fieldset.W.scaling_factor is not None and self.fieldset.W.scaling_factor > 0: + warning = True + if type(self.fieldset.W) in [SummedField, NestedField]: + for f in self.fieldset.W: + if f.creation_log != 'from_nemo' and f.scaling_factor is not None and f.scaling_factor > 0: + warning = True + if warning: + logger.warning_once('Note that in AdvectionRK4_3D, vertical velocity is assumed positive towards increasing z.\n' + '\tIf z increases downward and w is positive upward you can re-orient it downwards by setting fieldset.W.set_scaling_factor(-1.)') + if funcvars is not None: + self.funcvars = funcvars + elif hasattr(pyfunc, '__code__'): + self.funcvars = list(pyfunc.__code__.co_varnames) + else: + self.funcvars = None + self.funccode = funccode or inspect.getsource(pyfunc.__code__) + # Parse AST if it is not provided explicitly + self.py_ast = py_ast or parse(fix_indentation(self.funccode)).body[0] + if pyfunc is None: + # Extract user context by inspecting the call stack + stack = inspect.stack() + try: + user_ctx = stack[-1][0].f_globals + user_ctx['math'] = globals()['math'] + user_ctx['random'] = globals()['random'] + user_ctx['ErrorCode'] = globals()['ErrorCode'] + except: + logger.warning("Could not access user context when merging kernels") + user_ctx = globals() + finally: + del stack # Remove cyclic references + # Compile and generate Python function from AST + py_mod = parse("") + py_mod.body = [self.py_ast] + exec(compile(py_mod, "", "exec"), user_ctx) + self.pyfunc = user_ctx[self.funcname] + else: + self.pyfunc = pyfunc + + if version_info[0] < 3: + numkernelargs = len(inspect.getargspec(self.pyfunc).args) + else: + numkernelargs = len(inspect.getfullargspec(self.pyfunc).args) + + assert numkernelargs == 3, \ + 'Since Parcels v2.0, kernels do only take 3 arguments: particle, fieldset, time !! AND !! Argument order in field interpolation is time, depth, lat, lon.' + + self.name = "%s%s" % (ptype.name, self.funcname) + + # ======== THIS NEEDS TO BE REFACTORED BASED ON THE TYPE OF PARTICLE BEING USED ======== # + # Generate the kernel function and add the outer loop + if self.ptype.uses_jit: + kernelgen = KernelGenerator(ptype, self.fieldset) + kernel_ccode = kernelgen.generate(deepcopy(self.py_ast), self.funcvars) + self.field_args = kernelgen.field_args + self.vector_field_args = kernelgen.vector_field_args + for f in self.vector_field_args.values(): + Wname = f.W.ccode_name if f.W else 'not_defined' + for sF_name, sF_component in zip([f.U.ccode_name, f.V.ccode_name, Wname], ['U', 'V', 'W']): + if sF_name not in self.field_args: + if sF_name != 'not_defined': + self.field_args[sF_name] = getattr(f, sF_component) + self.const_args = kernelgen.const_args + + # loopgen = VectorizedLoopGenerator(ptype) + loopgen = NodeLoopGenerator(ptype, fieldset=self.fieldset) + + if path.isfile(c_include): + with open(c_include, 'r') as f: + c_include_str = f.read() + else: + c_include_str = c_include + self.ccode = loopgen.generate(self.funcname, self.field_args, self.const_args, + kernel_ccode, c_include_str) + # self.src_file, self.lib_file, self.log_file = self.get_kernel_compile_files() + self.dyn_srcs, self.lib_file, self.log_file = self.get_kernel_compile_files() + static_srcs = [path.join(get_package_dir(), 'nodes', 'node.c'), ] + self.static_srcs = static_srcs + self.src_file = [self.dyn_srcs, ] + self.static_srcs + + def __del__(self): + # Clean-up the in-memory dynamic linked libraries. + # This is not really necessary, as these programs are not that large, but with the new random + # naming scheme which is required on Windows OS'es to deal with updates to a Parcels' kernel. + super(Kernel, self).__del__() + + def __add__(self, kernel): + if not isinstance(kernel, Kernel): + kernel = Kernel(self.fieldset, self.ptype, pyfunc=kernel) + return self.merge(kernel, Kernel) + + def __radd__(self, kernel): + if not isinstance(kernel, Kernel): + kernel = Kernel(self.fieldset, self.ptype, pyfunc=kernel) + return kernel.merge(self, Kernel) + + def execute_jit(self, pset, endtime, dt): + """Invokes JIT engine to perform the core update loop""" + # if len(pset.particles) > 0: + # assert pset.fieldset.gridset.size == len(pset.particles[0].xi), 'FieldSet has different amount of grids than Particle.xi. Have you added Fields after creating the ParticleSet?' + if len(pset) > 0: + assert pset.fieldset.gridset.size == len(pset[0].data.xi), \ + 'FieldSet has different amount of grids than Particle.xi. Have you added Fields after creating the ParticleSet?' + for g in pset.fieldset.gridset.grids: + g.cstruct = None # This force to point newly the grids from Python to C + + # Make a copy of the transposed array to enforce + # C-contiguous memory layout for JIT mode. + for f in pset.fieldset.get_fields(): + if type(f) in [VectorField, NestedField, SummedField]: + continue + if f in self.field_args.values(): + f.chunk_data() + else: + for block_id in range(len(f.data_chunks)): + f.data_chunks[block_id] = None + f.c_data_chunks[block_id] = None + + for g in pset.fieldset.gridset.grids: + g.load_chunk = np.where(g.load_chunk == 1, 2, g.load_chunk) + if len(g.load_chunk) > 0: # not the case if a field in not called in the kernel + if not g.load_chunk.flags.c_contiguous: + g.load_chunk = g.load_chunk.copy() + if not g.depth.flags.c_contiguous: + g.depth = g.depth.copy() + if not g.lon.flags.c_contiguous: + g.lon = g.lon.copy() + if not g.lat.flags.c_contiguous: + g.lat = g.lat.copy() + + fargs = [] + if self.field_args is not None: + fargs += [byref(f.ctypes_struct) for f in self.field_args.values()] + if self.const_args is not None: + fargs += [c_double(f) for f in self.const_args.values()] + + # particle_data = pset._particle_data.ctypes.data_as(c_void_p) + node_data = pset.begin() + if len(fargs) > 0: + self._function(c_int(len(pset)), pointer(node_data), c_double(endtime), c_double(dt), *fargs) + else: + self._function(c_int(len(pset)), pointer(node_data), c_double(endtime), c_double(dt)) + + def execute_python(self, pset, endtime, dt): + """Performs the core update loop via Python""" + sign_dt = np.sign(dt) + + # back up variables in case of ErrorCode.Repeat + p_var_back = {} + + for f in self.fieldset.get_fields(): + if type(f) in [VectorField, NestedField, SummedField]: + continue + f.data = np.array(f.data) + + # ========= OLD ======= # + # for p in pset.particles: + # ===================== # + node = pset.begin() + while node is not None: + p = node.data + ptype = p.getPType() + # Don't execute particles that aren't started yet + sign_end_part = np.sign(endtime - p.time) + dt_pos = min(abs(p.dt), abs(endtime - p.time)) + + # ==== numerically stable; also making sure that continuously-recovered particles do end successfully, + # as they fulfil the condition here on entering at the final calculation here. ==== # + if ((sign_end_part != sign_dt) or np.isclose(dt_pos, 0)) and not np.isclose(dt, 0): + if abs(p.time) >= abs(endtime): + p.state = ErrorCode.Success + node = node.next + continue + + # Compute min/max dt for first timestep + # while dt_pos > 1e-6 or dt == 0: + while p.state in [ErrorCode.Evaluate, ErrorCode.Repeat] or np.isclose(dt, 0): + for var in ptype.variables: + p_var_back[var.name] = getattr(p, var.name) + try: + pdt_prekernels = sign_dt * dt_pos + p.dt = pdt_prekernels + state_prev = p.state + # res = self.pyfunc(p, None, p.time) + res = self.pyfunc(p, pset.fieldset, p.time) + if res is None: + res = ErrorCode.Success + + if res is ErrorCode.Success and p.state != state_prev: + res = p.state + + if res == ErrorCode.Success and not np.isclose(p.dt, pdt_prekernels): + res = ErrorCode.Repeat + + except FieldOutOfBoundError as fse_xy: + res = ErrorCode.ErrorOutOfBounds + p.exception = fse_xy + except FieldOutOfBoundSurfaceError as fse_z: + res = ErrorCode.ErrorThroughSurface + p.exception = fse_z + except TimeExtrapolationError as fse_t: + res = ErrorCode.ErrorTimeExtrapolation + p.exception = fse_t + except Exception as e: + res = ErrorCode.Error + p.exception = e + + # Handle particle time and time loop + if res in [ErrorCode.Success, ErrorCode.Delete]: + # Update time and repeat + p.time += p.dt + p.update_next_dt() + dt_pos = min(abs(p.dt), abs(endtime - p.time)) + + sign_end_part = np.sign(endtime - p.time) + if res != ErrorCode.Delete and not np.isclose(dt_pos, 0) and (sign_end_part == sign_dt): + res = ErrorCode.Evaluate + if sign_end_part != sign_dt: + dt_pos = 0 + + p.state = res + if np.isclose(dt, 0): + break + else: + p.state = res + # Try again without time update + for var in ptype.variables: + if var.name not in ['dt', 'state']: + setattr(p, var.name, p_var_back[var.name]) + dt_pos = min(abs(p.dt), abs(endtime - p.time)) + + sign_end_part = np.sign(endtime - p.time) + if sign_end_part != sign_dt: + dt_pos = 0 + break + node = node.next + + def remove_deleted(self, pset, output_file, endtime): + """Utility to remove all particles that signalled deletion""" + indices = pset.get_deleted_item_indices() + if len(indices) > 0: + pdata = [pset[i].data for i in indices] + if len(pdata) > 0 and output_file is not None: + output_file.write(pdata, endtime, deleted_only=True) + pset.remove_deleted_items_by_indices(indices) + return pset + + def execute(self, pset, endtime, dt, recovery=None, output_file=None, execute_once=False): + """Execute this Kernel over a ParticleSet for several timesteps""" + node = pset.begin() + while node is not None: + node.data.reset_state() + node = node.next + + if abs(dt) < 1e-6 and not execute_once: + logger.warning_once("'dt' is too small, causing numerical accuracy limit problems. Please chose a higher 'dt' and rather scale the 'time' axis of the field accordingly. (related issue #762)") + + if recovery is None: + recovery = {} + elif ErrorCode.ErrorOutOfBounds in recovery and ErrorCode.ErrorThroughSurface not in recovery: + recovery[ErrorCode.ErrorThroughSurface] = recovery[ErrorCode.ErrorOutOfBounds] + recovery_map = recovery_base_map.copy() + recovery_map.update(recovery) + + for g in pset.fieldset.gridset.grids: + if len(g.load_chunk) > 0: # not the case if a field in not called in the kernel + g.load_chunk = np.where(g.load_chunk == 2, 3, g.load_chunk) + + # Execute the kernel over the particle set + if self.ptype.uses_jit: + self.execute_jit(pset, endtime, dt) + else: + self.execute_python(pset, endtime, dt) + + # Remove all particles that signalled deletion + self.remove_deleted(pset=pset, output_file=output_file, endtime=endtime) + + # Identify particles that threw errors + # error_particles = [p for p in pset.particles if p.state not in [ErrorCode.Success, ErrorCode.Evaluate]] + error_particles = [n.data for n in pset.data if n.data.state not in [ErrorCode.Success, ErrorCode.Evaluate]] + + while len(error_particles) > 0: + # Apply recovery kernel + for p in error_particles: + if p.state == ErrorCode.StopExecution: + return + if p.state == ErrorCode.Repeat: + p.reset_state() + elif p.state == ErrorCode.Delete: + pass + elif p.state in recovery_map: + recovery_kernel = recovery_map[p.state] + p.state = ErrorCode.Success + recovery_kernel(p, self.fieldset, p.time) + if(p.isComputed()): + p.reset_state() + else: + logger.warning_once('Deleting particle {} because of non-recoverable error'.format(p.id)) + # logger.warning('Deleting particle because of bug in #749 and #737 - particle state: {}'.format(ErrorCode.toString(p.state))) + p.delete() + + # Remove all particles that signalled deletion + self.remove_deleted(pset=pset, output_file=output_file, endtime=endtime) + + # Execute core loop again to continue interrupted particles + if self.ptype.uses_jit: + self.execute_jit(pset, endtime, dt) + else: + self.execute_python(pset, endtime, dt) + + # error_particles = [p for p in pset.particles if p.state not in [ErrorCode.Success, ErrorCode.Evaluate]] + error_particles = [n.data for n in pset.data if n.data.state not in [ErrorCode.Success, ErrorCode.Evaluate]] diff --git a/parcels/kernel_node_benchmark.py b/parcels/kernel_node_benchmark.py new file mode 100644 index 0000000000..59d517803d --- /dev/null +++ b/parcels/kernel_node_benchmark.py @@ -0,0 +1,240 @@ +import inspect +import re +import math # noga +import random # noga +from ast import parse +from copy import deepcopy +from ctypes import byref +from ctypes import c_double +from ctypes import c_int +# from ctypes import c_void_p +from ctypes import pointer +from os import path +from sys import version_info +import numpy as np + +from parcels import Field, NestedField, SummedField, VectorField +from parcels import ErrorCode +from parcels.field import FieldOutOfBoundError, FieldOutOfBoundSurfaceError, TimeExtrapolationError +from parcels.tools.global_statics import get_cache_dir, get_package_dir +from parcels.wrapping import KernelGenerator, NodeLoopGenerator +from parcels.tools.error import recovery_map as recovery_base_map +from parcels import AdvectionRK4_3D, logger + +from parcels.kernel_node import Kernel +from parcels.tools.performance_logger import TimingLog + +__all__ = ['Kernel_Benchmark'] + + +class Kernel_Benchmark(Kernel): + def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, funccode=None, py_ast=None, funcvars=None, + c_include="", delete_cfiles=True): + super(Kernel_Benchmark, self).__init__(fieldset, ptype, pyfunc=pyfunc, funcname=funcname, funccode=funccode, py_ast=py_ast, funcvars=funcvars, c_include=c_include, delete_cfiles=delete_cfiles) + self._compute_timings = TimingLog() + self._io_timings = TimingLog() + self._mem_io_log = TimingLog() + + def __del__(self): + # Clean-up the in-memory dynamic linked libraries. + # This is not really necessary, as these programs are not that large, but with the new random + # naming scheme which is required on Windows OS'es to deal with updates to a Parcels' kernel. + super(Kernel_Benchmark, self).__del__() + + @property + def io_timings(self): + return self._io_timings + + @property + def mem_io_timings(self): + return self._mem_io_log + + @property + def compute_timings(self): + return self._compute_timings + + def __add__(self, kernel): + if not isinstance(kernel, Kernel): + kernel = Kernel_Benchmark(self.fieldset, self.ptype, pyfunc=kernel) + return self.merge(kernel, Kernel_Benchmark) + + def __radd__(self, kernel): + if not isinstance(kernel, Kernel): + kernel = Kernel_Benchmark(self.fieldset, self.ptype, pyfunc=kernel) + return kernel.merge(self, Kernel_Benchmark) + + def execute_jit(self, pset, endtime, dt): + """Invokes JIT engine to perform the core update loop""" + self._io_timings.start_timing() + if len(pset) > 0: + assert pset.fieldset.gridset.size == len(pset[0].data.xi), \ + 'FieldSet has different amount of grids than Particle.xi. Have you added Fields after creating the ParticleSet?' + for g in pset.fieldset.gridset.grids: + g.cstruct = None # This force to point newly the grids from Python to C + # Make a copy of the transposed array to enforce + # C-contiguous memory layout for JIT mode. + for f in pset.fieldset.get_fields(): + if type(f) in [VectorField, NestedField, SummedField]: + continue + if f in self.field_args.values(): + f.chunk_data() + else: + for block_id in range(len(f.data_chunks)): + f.data_chunks[block_id] = None + f.c_data_chunks[block_id] = None + self._io_timings.stop_timing() + self._io_timings.accumulate_timing() + + self._mem_io_log.start_timing() + for g in pset.fieldset.gridset.grids: + g.load_chunk = np.where(g.load_chunk == 1, 2, g.load_chunk) + if len(g.load_chunk) > 0: # not the case if a field in not called in the kernel + if not g.load_chunk.flags.c_contiguous: + g.load_chunk = g.load_chunk.copy() + if not g.depth.flags.c_contiguous: + g.depth = g.depth.copy() + if not g.lon.flags.c_contiguous: + g.lon = g.lon.copy() + if not g.lat.flags.c_contiguous: + g.lat = g.lat.copy() + + fargs = [] + if self.field_args is not None: + # ==== major runtime expense in this function, for plain advection ==== # + fargs += [byref(f.ctypes_struct) for f in self.field_args.values()] + if self.const_args is not None: + fargs += [c_double(f) for f in self.const_args.values()] + self._mem_io_log.stop_timing() + self._mem_io_log.accumulate_timing() + + self._compute_timings.start_timing() + # particle_data = pset._particle_data.ctypes.data_as(c_void_p) + node_data = pset.begin() + if len(fargs) > 0: + self._function(c_int(len(pset)), pointer(node_data), c_double(endtime), c_double(dt), *fargs) + else: + self._function(c_int(len(pset)), pointer(node_data), c_double(endtime), c_double(dt)) + self._compute_timings.stop_timing() + self._compute_timings.accumulate_timing() + + self._io_timings.advance_iteration() + self._mem_io_log.advance_iteration() + self._compute_timings.advance_iteration() + + def execute_python(self, pset, endtime, dt): + """Performs the core update loop via Python""" + sign_dt = np.sign(dt) + + # back up variables in case of ErrorCode.Repeat + p_var_back = {} + + for f in self.fieldset.get_fields(): + if type(f) in [VectorField, NestedField, SummedField]: + continue + + self._io_timings.start_timing() + loaded_data = f.data + self._io_timings.stop_timing() + self._io_timings.accumulate_timing() + self._mem_io_log.start_timing() + f.data = np.array(loaded_data) + self._mem_io_log.stop_timing() + self._mem_io_log.accumulate_timing() + + self._compute_timings.start_timing() + # ========= OLD ======= # + # for p in pset.particles: + # ===================== # + node = pset.begin() + while node is not None: + p = node.data + ptype = p.getPType() + # Don't execute particles that aren't started yet + sign_end_part = np.sign(endtime - p.time) + dt_pos = min(abs(p.dt), abs(endtime - p.time)) + + # ==== numerically stable; also making sure that continuously-recovered particles do end successfully, + # as they fulfil the condition here on entering at the final calculation here. ==== # + if ((sign_end_part != sign_dt) or np.isclose(dt_pos, 0)) and not np.isclose(dt, 0): + if abs(p.time) >= abs(endtime): + p.state = ErrorCode.Success + node = node.next + continue + + # Compute min/max dt for first timestep + # while dt_pos > 1e-6 or dt == 0: + while p.state in [ErrorCode.Evaluate, ErrorCode.Repeat] or np.isclose(dt, 0): + for var in ptype.variables: + p_var_back[var.name] = getattr(p, var.name) + try: + pdt_prekernels = sign_dt * dt_pos + p.dt = pdt_prekernels + state_prev = p.state + # res = self.pyfunc(p, None, p.time) + res = self.pyfunc(p, pset.fieldset, p.time) + if res is None: + res = ErrorCode.Success + + if res is ErrorCode.Success and p.state != state_prev: + res = p.state + + if res == ErrorCode.Success and not np.isclose(p.dt, pdt_prekernels): + res = ErrorCode.Repeat + + except FieldOutOfBoundError as fse_xy: + res = ErrorCode.ErrorOutOfBounds + p.exception = fse_xy + except FieldOutOfBoundSurfaceError as fse_z: + res = ErrorCode.ErrorThroughSurface + p.exception = fse_z + except TimeExtrapolationError as fse_t: + res = ErrorCode.ErrorTimeExtrapolation + p.exception = fse_t + except Exception as e: + res = ErrorCode.Error + p.exception = e + + # Handle particle time and time loop + if res in [ErrorCode.Success, ErrorCode.Delete]: + # Update time and repeat + p.time += p.dt + p.update_next_dt() + dt_pos = min(abs(p.dt), abs(endtime - p.time)) + + sign_end_part = np.sign(endtime - p.time) + if res != ErrorCode.Delete and not np.isclose(dt_pos, 0) and (sign_end_part == sign_dt): + res = ErrorCode.Evaluate + if sign_end_part != sign_dt: + dt_pos = 0 + + p.state = res + if np.isclose(dt, 0): + break + else: + p.state = res + # Try again without time update + for var in ptype.variables: + if var.name not in ['dt', 'state']: + setattr(p, var.name, p_var_back[var.name]) + dt_pos = min(abs(p.dt), abs(endtime - p.time)) + + sign_end_part = np.sign(endtime - p.time) + if sign_end_part != sign_dt: + dt_pos = 0 + break + node = node.next + self._compute_timings.stop_timing() + self._compute_timings.accumulate_timing() + + self._io_timings.advance_iteration() + self._mem_io_log.advance_iteration() + self._compute_timings.advance_iteration() + + def remove_deleted(self, pset, output_file, endtime): + """Utility to remove all particles that signalled deletion""" + self._mem_io_log.start_timing() + super(Kernel_Benchmark, self).remove_deleted(pset=pset, output_file=output_file, endtime=endtime) + self._mem_io_log.stop_timing() + self._mem_io_log.accumulate_timing() + self._mem_io_log.advance_iteration() + diff --git a/parcels/kernel_benchmark.py b/parcels/kernel_vec_benchmark.py similarity index 78% rename from parcels/kernel_benchmark.py rename to parcels/kernel_vec_benchmark.py index 07c430ab71..5c42edf509 100644 --- a/parcels/kernel_benchmark.py +++ b/parcels/kernel_vec_benchmark.py @@ -1,32 +1,25 @@ -import _ctypes import inspect -import math # noqa -import random # noqa +import math # noga +import random # noga import re import time -from ast import FunctionDef from ast import parse from copy import deepcopy from ctypes import byref from ctypes import c_double from ctypes import c_int from ctypes import c_void_p -from hashlib import md5 from os import path -from os import remove -from sys import platform from sys import version_info import numpy as np -import numpy.ctypeslib as npct try: from mpi4py import MPI except: MPI = None -from parcels.codegenerator import KernelGenerator -from parcels.codegenerator import LoopGenerator -from parcels.compiler import get_cache_dir +from parcels.wrapping.code_generator import KernelGenerator +from parcels.wrapping.code_generator import VectorizedLoopGenerator from parcels.field import Field from parcels.field import FieldOutOfBoundError from parcels.field import FieldOutOfBoundSurfaceError @@ -39,12 +32,16 @@ from parcels.tools.error import recovery_map as recovery_base_map from parcels.tools.loggers import logger -from parcels.kernel import Kernel +from parcels.kernel_vectorized import Kernel from parcels.tools.performance_logger import TimingLog + __all__ = ['Kernel_Benchmark'] +re_indent = re.compile(r"^(\s+)") + + class Kernel_Benchmark(Kernel): """Kernel object that encapsulates auto-generated code. @@ -57,14 +54,18 @@ class Kernel_Benchmark(Kernel): or the necessary information (funcname, funccode, funcvars) is provided. The py_ast argument may be derived from the code string, but for concatenation, the merged AST plus the new header definition is required. + + # == CHANGES THAT REFLECT THE 'None'-Type FieldSet still need to be done == # """ - def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, - funccode=None, py_ast=None, funcvars=None, c_include="", delete_cfiles=True): - super(Kernel_Benchmark, self).__init__(fieldset, ptype, pyfunc, funcname, funccode, py_ast, funcvars, c_include, delete_cfiles) + def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, funccode=None, py_ast=None, funcvars=None, c_include="", delete_cfiles=True): + super(Kernel_Benchmark, self).__init__(fieldset, ptype, pyfunc=pyfunc, funcname=funcname, funccode=funccode, py_ast=py_ast, funcvars=funcvars, c_include=c_include, delete_cfiles=delete_cfiles) self._compute_timings = TimingLog() self._io_timings = TimingLog() - self._mem_io_timings = TimingLog() + self._mem_io_log = TimingLog() + + def __del__(self): + super(Kernel_Benchmark, self).__del__() @property def io_timings(self): @@ -72,14 +73,21 @@ def io_timings(self): @property def mem_io_timings(self): - return self._mem_io_timings + return self._mem_io_log @property def compute_timings(self): return self._compute_timings - def __del__(self): - super(Kernel_Benchmark, self).__del__() + def __add__(self, kernel): + if not isinstance(kernel, Kernel): + kernel = Kernel_Benchmark(self.fieldset, self.ptype, pyfunc=kernel) + return self.merge(kernel, Kernel_Benchmark) + + def __radd__(self, kernel): + if not isinstance(kernel, Kernel): + kernel = Kernel_Benchmark(self.fieldset, self.ptype, pyfunc=kernel) + return kernel.merge(self, Kernel_Benchmark) def execute_jit(self, pset, endtime, dt): """Invokes JIT engine to perform the core update loop""" @@ -103,7 +111,7 @@ def execute_jit(self, pset, endtime, dt): self._io_timings.stop_timing() self._io_timings.accumulate_timing() - self._mem_io_timings.start_timing() + self._mem_io_log.start_timing() for g in pset.fieldset.gridset.grids: g.load_chunk = np.where(g.load_chunk == 1, 2, g.load_chunk) if len(g.load_chunk) > 0: # not the case if a field in not called in the kernel @@ -116,22 +124,28 @@ def execute_jit(self, pset, endtime, dt): if not g.lat.flags.c_contiguous: g.lat = g.lat.copy() - fargs = [byref(f.ctypes_struct) for f in self.field_args.values()] - fargs += [c_double(f) for f in self.const_args.values()] - particle_data = pset._particle_data.ctypes.data_as(c_void_p) - self._mem_io_timings.stop_timing() - self._mem_io_timings.accumulate_timing() + fargs = [] + if self.field_args is not None: + fargs += [byref(f.ctypes_struct) for f in self.field_args.values()] + if self.const_args is not None: + fargs += [c_double(f) for f in self.const_args.values()] + self._mem_io_log.stop_timing() + self._mem_io_log.accumulate_timing() self._compute_timings.start_timing() - self._function(c_int(len(pset)), particle_data, - c_double(endtime), - c_double(dt), - *fargs) + # particle_data = pset._particle_data.ctypes.data_as(c_void_p) + pdata = pset.particle_data.ctypes.data_as(c_void_p) + if len(fargs) > 0: + self._function(c_int(len(pset)), pdata, c_double(endtime), c_double(dt), *fargs) + else: + + self._function(c_int(len(pset)), pdata, c_double(endtime), c_double(dt)) + self._compute_timings.stop_timing() self._compute_timings.accumulate_timing() self._io_timings.advance_iteration() - self._mem_io_timings.advance_iteration() + self._mem_io_log.advance_iteration() self._compute_timings.advance_iteration() def execute_python(self, pset, endtime, dt): @@ -149,10 +163,10 @@ def execute_python(self, pset, endtime, dt): loaded_data = f.data self._io_timings.stop_timing() self._io_timings.accumulate_timing() - self._mem_io_timings.start_timing() + self._mem_io_log.start_timing() f.data = np.array(loaded_data) - self._mem_io_timings.stop_timing() - self._mem_io_timings.accumulate_timing() + self._mem_io_log.stop_timing() + self._mem_io_log.accumulate_timing() self._compute_timings.start_timing() for p in pset.particles: @@ -232,34 +246,14 @@ def execute_python(self, pset, endtime, dt): self._compute_timings.accumulate_timing() self._io_timings.advance_iteration() - self._mem_io_timings.advance_iteration() + self._mem_io_log.advance_iteration() self._compute_timings.advance_iteration() def remove_deleted(self, pset, output_file, endtime): """Utility to remove all particles that signalled deletion""" - self._mem_io_timings.start_timing() + self._mem_io_log.start_timing() super(Kernel_Benchmark, self).remove_deleted(pset=pset, output_file=output_file, endtime=endtime) - self._mem_io_timings.stop_timing() - self._mem_io_timings.accumulate_timing() - self._mem_io_timings.advance_iteration() - - def merge(self, kernel): - funcname = self.funcname + kernel.funcname - func_ast = FunctionDef(name=funcname, args=self.py_ast.args, - body=self.py_ast.body + kernel.py_ast.body, - decorator_list=[], lineno=1, col_offset=0) - delete_cfiles = self.delete_cfiles and kernel.delete_cfiles - return Kernel_Benchmark(self.fieldset, self.ptype, pyfunc=None, - funcname=funcname, funccode=self.funccode + kernel.funccode, - py_ast=func_ast, funcvars=self.funcvars + kernel.funcvars, - delete_cfiles=delete_cfiles) - - def __add__(self, kernel): - if not isinstance(kernel, Kernel): - kernel = Kernel_Benchmark(self.fieldset, self.ptype, pyfunc=kernel) - return self.merge(kernel) + self._mem_io_log.stop_timing() + self._mem_io_log.accumulate_timing() + self._mem_io_log.advance_iteration() - def __radd__(self, kernel): - if not isinstance(kernel, Kernel): - kernel = Kernel_Benchmark(self.fieldset, self.ptype, pyfunc=kernel) - return kernel.merge(self) diff --git a/parcels/kernel.py b/parcels/kernel_vectorized.py similarity index 70% rename from parcels/kernel.py rename to parcels/kernel_vectorized.py index 95a785e4f6..f7de1c56d3 100644 --- a/parcels/kernel.py +++ b/parcels/kernel_vectorized.py @@ -1,32 +1,25 @@ -import _ctypes import inspect -import math # noqa -import random # noqa +import math # noga +import random # noga import re -import time -from ast import FunctionDef +import time # noga from ast import parse from copy import deepcopy from ctypes import byref from ctypes import c_double from ctypes import c_int from ctypes import c_void_p -from hashlib import md5 from os import path -from os import remove -from sys import platform from sys import version_info import numpy as np -import numpy.ctypeslib as npct try: from mpi4py import MPI except: MPI = None -from parcels.codegenerator import KernelGenerator -from parcels.codegenerator import LoopGenerator -from parcels.compiler import get_cache_dir +from parcels.wrapping.code_generator import KernelGenerator +from parcels.wrapping.code_generator import VectorizedLoopGenerator from parcels.field import Field from parcels.field import FieldOutOfBoundError from parcels.field import FieldOutOfBoundSurfaceError @@ -38,6 +31,7 @@ from parcels.tools.error import ErrorCode from parcels.tools.error import recovery_map as recovery_base_map from parcels.tools.loggers import logger +from parcels.kernelbase import BaseKernel __all__ = ['Kernel'] @@ -55,7 +49,7 @@ def fix_indentation(string): return "\n".join(lines) -class Kernel(object): +class Kernel(BaseKernel): """Kernel object that encapsulates auto-generated code. :arg fieldset: FieldSet object providing the field information @@ -66,29 +60,26 @@ class Kernel(object): or the necessary information (funcname, funccode, funcvars) is provided. The py_ast argument may be derived from the code string, but for concatenation, the merged AST plus the new header definition is required. + + # == CHANGES THAT REFLECT THE 'None'-Type FieldSet still need to be done == # """ - def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, - funccode=None, py_ast=None, funcvars=None, c_include="", delete_cfiles=True): - self.fieldset = fieldset - self.ptype = ptype - self._lib = None - self.delete_cfiles = delete_cfiles + def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, funccode=None, py_ast=None, funcvars=None, c_include="", delete_cfiles=True): + super(Kernel, self).__init__(fieldset, ptype, pyfunc=pyfunc, funcname=funcname, funccode=funccode, py_ast=py_ast, funcvars=funcvars, c_include=c_include, delete_cfiles=delete_cfiles) # Derive meta information from pyfunc, if not given - self.funcname = funcname or pyfunc.__name__ - if pyfunc is AdvectionRK4_3D: + if pyfunc is AdvectionRK4_3D: # would be better if the idea of a Kernel being '2D', '3D, '4D' or 'uncertain' is captured as Attribute or as class stucture warning = False - if isinstance(fieldset.W, Field) and fieldset.W.creation_log != 'from_nemo' and \ - fieldset.W._scaling_factor is not None and fieldset.W._scaling_factor > 0: + if isinstance(self.fieldset.W, Field) and self.fieldset.W.creation_log != 'from_nemo' and \ + self.fieldset.W.scaling_factor is not None and self.fieldset.W.scaling_factor > 0: warning = True - if type(fieldset.W) in [SummedField, NestedField]: - for f in fieldset.W: - if f.creation_log != 'from_nemo' and f._scaling_factor is not None and f._scaling_factor > 0: + if type(self.fieldset.W) in [SummedField, NestedField]: + for f in self.fieldset.W: + if f.creation_log != 'from_nemo' and f.scaling_factor is not None and f.scaling_factor > 0: warning = True if warning: logger.warning_once('Note that in AdvectionRK4_3D, vertical velocity is assumed positive towards increasing z.\n' - ' If z increases downward and w is positive upward you can re-orient it downwards by setting fieldset.W.set_scaling_factor(-1.)') + '\tIf z increases downward and w is positive upward you can re-orient it downwards by setting fieldset.W.set_scaling_factor(-1.)') if funcvars is not None: self.funcvars = funcvars elif hasattr(pyfunc, '__code__'): @@ -129,14 +120,13 @@ def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, self.name = "%s%s" % (ptype.name, self.funcname) + # ======== THIS NEEDS TO BE REFACTORED BASED ON THE TYPE OF PARTICLE BEING USED ======== # # Generate the kernel function and add the outer loop if self.ptype.uses_jit: - kernelgen = KernelGenerator(fieldset, ptype) - kernel_ccode = kernelgen.generate(deepcopy(self.py_ast), - self.funcvars) + kernelgen = KernelGenerator(ptype, self.fieldset) + kernel_ccode = kernelgen.generate(deepcopy(self.py_ast), self.funcvars) self.field_args = kernelgen.field_args self.vector_field_args = kernelgen.vector_field_args - fieldset = self.fieldset for f in self.vector_field_args.values(): Wname = f.W.ccode_name if f.W else 'not_defined' for sF_name, sF_component in zip([f.U.ccode_name, f.V.ccode_name, Wname], ['U', 'V', 'W']): @@ -144,7 +134,7 @@ def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, if sF_name != 'not_defined': self.field_args[sF_name] = getattr(f, sF_component) self.const_args = kernelgen.const_args - loopgen = LoopGenerator(fieldset, ptype) + loopgen = VectorizedLoopGenerator(ptype, fieldset) if path.isfile(c_include): with open(c_include, 'r') as f: c_include_str = f.read() @@ -152,70 +142,25 @@ def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, c_include_str = c_include self.ccode = loopgen.generate(self.funcname, self.field_args, self.const_args, kernel_ccode, c_include_str) - if MPI: - mpi_comm = MPI.COMM_WORLD - mpi_rank = mpi_comm.Get_rank() - basename = path.join(get_cache_dir(), self._cache_key) if mpi_rank == 0 else None - basename = mpi_comm.bcast(basename, root=0) - basename = basename + "_%d" % mpi_rank - else: - basename = path.join(get_cache_dir(), "%s_0" % self._cache_key) - - self.src_file = "%s.c" % basename - self.lib_file = "%s.%s" % (basename, 'dll' if platform == 'win32' else 'so') - self.log_file = "%s.log" % basename + # self.src_file, self.lib_file, self.log_file = self.get_kernel_compile_files() + self.dyn_srcs, self.lib_file, self.log_file = self.get_kernel_compile_files() + self.src_file = self.dyn_srcs def __del__(self): # Clean-up the in-memory dynamic linked libraries. # This is not really necessary, as these programs are not that large, but with the new random # naming scheme which is required on Windows OS'es to deal with updates to a Parcels' kernel. - if self._lib is not None: - _ctypes.FreeLibrary(self._lib._handle) if platform == 'win32' else _ctypes.dlclose(self._lib._handle) - del self._lib - self._lib = None - if path.isfile(self.lib_file) and self.delete_cfiles: - [remove(s) for s in [self.src_file, self.lib_file, self.log_file]] - - @property - def _cache_key(self): - field_keys = "-".join(["%s:%s" % (name, field.units.__class__.__name__) - for name, field in self.field_args.items()]) - key = self.name + self.ptype._cache_key + field_keys + ('TIME:%f' % time.time()) - return md5(key.encode('utf-8')).hexdigest() - - def remove_lib(self): - # Unload the currently loaded dynamic linked library to be secure - if self._lib is not None: - _ctypes.FreeLibrary(self._lib._handle) if platform == 'win32' else _ctypes.dlclose(self._lib._handle) - del self._lib - self._lib = None - # If file already exists, pull new names. This is necessary on a Windows machine, because - # Python's ctype does not deal in any sort of manner well with dynamic linked libraries on this OS. - if path.isfile(self.lib_file): - [remove(s) for s in [self.src_file, self.lib_file, self.log_file]] - if MPI: - mpi_comm = MPI.COMM_WORLD - mpi_rank = mpi_comm.Get_rank() - basename = path.join(get_cache_dir(), self._cache_key) if mpi_rank == 0 else None - basename = mpi_comm.bcast(basename, root=0) - basename = basename + "_%d" % mpi_rank - else: - basename = path.join(get_cache_dir(), "%s_0" % self._cache_key) - - self.src_file = "%s.c" % basename - self.lib_file = "%s.%s" % (basename, 'dll' if platform == 'win32' else 'so') - self.log_file = "%s.log" % basename + super(Kernel, self).__del__() - def compile(self, compiler): - """ Writes kernel code to file and compiles it.""" - with open(self.src_file, 'w') as f: - f.write(self.ccode) - compiler.compile(self.src_file, self.lib_file, self.log_file) - logger.info("Compiled %s ==> %s" % (self.name, self.lib_file)) + def __add__(self, kernel): + if not isinstance(kernel, Kernel): + kernel = Kernel(self.fieldset, self.ptype, pyfunc=kernel) + return self.merge(kernel, Kernel) - def load_lib(self): - self._lib = npct.load_library(self.lib_file, '.') - self._function = self._lib.particle_loop + def __radd__(self, kernel): + if not isinstance(kernel, Kernel): + kernel = Kernel(self.fieldset, self.ptype, pyfunc=kernel) + return kernel.merge(self, Kernel) def execute_jit(self, pset, endtime, dt): """Invokes JIT engine to perform the core update loop""" @@ -224,6 +169,7 @@ def execute_jit(self, pset, endtime, dt): 'FieldSet has different amount of grids than Particle.xi. Have you added Fields after creating the ParticleSet?' for g in pset.fieldset.gridset.grids: g.cstruct = None # This force to point newly the grids from Python to C + # Make a copy of the transposed array to enforce # C-contiguous memory layout for JIT mode. for f in pset.fieldset.get_fields(): @@ -248,13 +194,18 @@ def execute_jit(self, pset, endtime, dt): if not g.lat.flags.c_contiguous: g.lat = g.lat.copy() - fargs = [byref(f.ctypes_struct) for f in self.field_args.values()] - fargs += [c_double(f) for f in self.const_args.values()] - particle_data = pset._particle_data.ctypes.data_as(c_void_p) - self._function(c_int(len(pset)), particle_data, - c_double(endtime), - c_double(dt), - *fargs) + fargs = [] + if self.field_args is not None: + fargs += [byref(f.ctypes_struct) for f in self.field_args.values()] + if self.const_args is not None: + fargs += [c_double(f) for f in self.const_args.values()] + + # particle_data = pset._particle_data.ctypes.data_as(c_void_p) + pdata = pset.particle_data.ctypes.data_as(c_void_p) + if len(fargs) > 0: + self._function(c_int(len(pset)), pdata, c_double(endtime), c_double(dt), *fargs) + else: + self._function(c_int(len(pset)), pdata, c_double(endtime), c_double(dt)) def execute_python(self, pset, endtime, dt): """Performs the core update loop via Python""" @@ -358,14 +309,6 @@ def execute(self, pset, endtime, dt, recovery=None, output_file=None, execute_on if abs(dt) < 1e-6 and not execute_once: logger.warning_once("'dt' is too small, causing numerical accuracy limit problems. Please chose a higher 'dt' and rather scale the 'time' axis of the field accordingly. (related issue #762)") - # def remove_deleted(pset): - # """Utility to remove all particles that signalled deletion""" - # indices = [i for i, p in enumerate(pset.particles) - # if p.state in [ErrorCode.Delete]] - # if len(indices) > 0 and output_file is not None: - # output_file.write(pset[indices], endtime, deleted_only=True) - # pset.remove(indices) - if recovery is None: recovery = {} elif ErrorCode.ErrorOutOfBounds in recovery and ErrorCode.ErrorThroughSurface not in recovery: @@ -397,7 +340,6 @@ def execute(self, pset, endtime, dt, recovery=None, output_file=None, execute_on if p.state == ErrorCode.Repeat: p.reset_state() elif p.state == ErrorCode.Delete: - # p.delete() pass elif p.state in recovery_map: recovery_kernel = recovery_map[p.state] @@ -420,24 +362,3 @@ def execute(self, pset, endtime, dt, recovery=None, output_file=None, execute_on self.execute_python(pset, endtime, dt) error_particles = [p for p in pset.particles if p.state not in [ErrorCode.Success, ErrorCode.Evaluate]] - - def merge(self, kernel): - funcname = self.funcname + kernel.funcname - func_ast = FunctionDef(name=funcname, args=self.py_ast.args, - body=self.py_ast.body + kernel.py_ast.body, - decorator_list=[], lineno=1, col_offset=0) - delete_cfiles = self.delete_cfiles and kernel.delete_cfiles - return Kernel(self.fieldset, self.ptype, pyfunc=None, - funcname=funcname, funccode=self.funccode + kernel.funccode, - py_ast=func_ast, funcvars=self.funcvars + kernel.funcvars, - delete_cfiles=delete_cfiles) - - def __add__(self, kernel): - if not isinstance(kernel, Kernel): - kernel = Kernel(self.fieldset, self.ptype, pyfunc=kernel) - return self.merge(kernel) - - def __radd__(self, kernel): - if not isinstance(kernel, Kernel): - kernel = Kernel(self.fieldset, self.ptype, pyfunc=kernel) - return kernel.merge(self) diff --git a/parcels/kernelbase.py b/parcels/kernelbase.py new file mode 100644 index 0000000000..6b260b69c4 --- /dev/null +++ b/parcels/kernelbase.py @@ -0,0 +1,175 @@ +import _ctypes +import numpy.ctypeslib as npct +from time import time as ostime +from os import path +from os import remove +from sys import platform +from ast import FunctionDef +from hashlib import md5 +from parcels.tools.loggers import logger + +try: + from mpi4py import MPI +except: + MPI = None + +from parcels.tools import get_cache_dir + + +__all__ = ['BaseKernel'] + + +class BaseKernel(object): + """Base super class for base Kernel objects that encapsulates auto-generated code. + + :arg fieldset: FieldSet object providing the field information (possibly None) + :arg ptype: PType object for the kernel particle + :arg pyfunc: (aggregated) Kernel function + :arg funcname: function name + :param delete_cfiles: Boolean whether to delete the C-files after compilation in JIT mode (default is True) + + Note: A Kernel is either created from a compiled object + or the necessary information (funcname, funccode, funcvars) is provided. + The py_ast argument may be derived from the code string, but for + concatenation, the merged AST plus the new header definition is required. + """ + + def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, funccode=None, py_ast=None, funcvars=None, + c_include="", delete_cfiles=True): + self.fieldset = fieldset + self.field_args = None + self.const_args = None + self.ptype = ptype + self._lib = None + self.delete_cfiles = delete_cfiles + + # Derive meta information from pyfunc, if not given + self.funcname = funcname or pyfunc.__name__ + self.name = "%s%s" % (ptype.name, self.funcname) + self.ccode = "" + self.funcvars = funcvars + self.funccode = funccode + self.py_ast = py_ast + self.dyn_srcs = [] + self.static_srcs = [] + self.src_file = None + self.lib_file = None + self.log_file = None + + # Generate the kernel function and add the outer loop + if self.ptype.uses_jit: + self.dyn_srcs, self.lib_file, self.log_file = self.get_kernel_compile_files() + self.src_file = self.dyn_srcs + + def __del__(self): + # Clean-up the in-memory dynamic linked libraries. + # This is not really necessary, as these programs are not that large, but with the new random + # naming scheme which is required on Windows OS'es to deal with updates to a Parcels' kernel. + # if self._lib is not None: + # _ctypes.FreeLibrary(self._lib._handle) if platform == 'win32' else _ctypes.dlclose(self._lib._handle) + # del self._lib + # self._lib = None + # if path.isfile(self.lib_file) and self.delete_cfiles: + # [remove(s) for s in [self.src_file, self.lib_file, self.log_file]] + self.remove_lib() + self.fieldset = None + self.field_args = None + self.const_args = None + self.funcvars = None + self.funccode = None + + @property + def _cache_key(self): + field_keys = "" + if self.field_args is not None: + field_keys = "-".join( + ["%s:%s" % (name, field.units.__class__.__name__) for name, field in self.field_args.items()]) + key = self.name + self.ptype._cache_key + field_keys + ('TIME:%f' % ostime()) + return md5(key.encode('utf-8')).hexdigest() + + def remove_lib(self): + # Unload the currently loaded dynamic linked library to be secure + if self._lib is not None: + _ctypes.FreeLibrary(self._lib._handle) if platform == 'win32' else _ctypes.dlclose(self._lib._handle) + del self._lib + self._lib = None + # If file already exists, pull new names. This is necessary on a Windows machine, because + # Python's ctype does not deal in any sort of manner well with dynamic linked libraries on this OS. + if self.lib_file is not None and path.isfile(self.lib_file): + [remove(s) for s in [self.lib_file, ]] + [remove(s) for s in [self.dyn_srcs, self.log_file, ] if self.delete_cfiles] + + def get_kernel_compile_files(self): + """ + Returns the correct src_file, lib_file, log_file for this kernel + """ + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + # cache_name = "lib"+self._cache_key # only required here because loading is done by Kernel class instead of Compiler class + cache_name = self._cache_key # only required here because loading is done by Kernel class instead of Compiler class + dyn_dir = get_cache_dir() if mpi_rank == 0 else None + dyn_dir = mpi_comm.bcast(dyn_dir, root=0) + ## basename = path.join(get_cache_dir(), self._cache_key) if mpi_rank == 0 else None + # basename = path.join(get_cache_dir(), cache_name) if mpi_rank == 0 else None + basename = cache_name if mpi_rank == 0 else None + basename = mpi_comm.bcast(basename, root=0) + basename = basename + "_%d" % mpi_rank + else: + # cache_name = "lib"+self._cache_key # only required here because loading is done by Kernel class instead of Compiler class + cache_name = self._cache_key # only required here because loading is done by Kernel class instead of Compiler class + dyn_dir = get_cache_dir() + # basename = path.join(get_cache_dir(), "%s_0" % self._cache_key) + basename = path.join(get_cache_dir(), "%s_0" % cache_name) + lib_path = "lib" + basename + src_file = "%s.c" % path.join(dyn_dir, basename) + lib_file = "%s.%s" % (path.join(dyn_dir, lib_path), 'dll' if platform == 'win32' else 'so') + log_file = "%s.log" % path.join(dyn_dir, basename) + return src_file, lib_file, log_file + + def compile(self, compiler): + """ Writes kernel code to file and compiles it.""" + # with open(self.src_file, 'w') as f: + with open(self.dyn_srcs, 'w') as f: + f.write(self.ccode) + compiler.compile(self.src_file, self.lib_file, self.log_file) + logger.info("Compiled %s ==> %s" % (self.name, self.lib_file)) + + def load_lib(self): + self._lib = npct.load_library(self.lib_file, '.') + # self._lib = npct.load_library(self.lib_file, get_cache_dir()) + self._function = self._lib.particle_loop + + def merge(self, kernel, kclass): + funcname = self.funcname + kernel.funcname + func_ast = None + if self.py_ast is not None: + func_ast = FunctionDef(name=funcname, args=self.py_ast.args, body=self.py_ast.body + kernel.py_ast.body, + decorator_list=[], lineno=1, col_offset=0) + delete_cfiles = self.delete_cfiles and kernel.delete_cfiles + return kclass(self.fieldset, self.ptype, pyfunc=None, + funcname=funcname, funccode=self.funccode + kernel.funccode, + py_ast=func_ast, funcvars=self.funcvars + kernel.funcvars, + delete_cfiles=delete_cfiles) + + def __add__(self, kernel): + if not isinstance(kernel, BaseKernel): + kernel = BaseKernel(self.fieldset, self.ptype, pyfunc=kernel) + return self.merge(kernel, BaseKernel) + + def __radd__(self, kernel): + if not isinstance(kernel, BaseKernel): + kernel = BaseKernel(self.fieldset, self.ptype, pyfunc=kernel) + return kernel.merge(self, BaseKernel) + + def remove_deleted(self, pset, output_file, endtime): + pass + + def execute_jit(self, pset, endtime, dt): + pass + + def execute_python(self, pset, endtime, dt): + pass + + def execute(self, pset, endtime, dt, recovery=None, output_file=None): + pass diff --git a/parcels/nodes/LinkedList.py b/parcels/nodes/LinkedList.py new file mode 100644 index 0000000000..b72fc06874 --- /dev/null +++ b/parcels/nodes/LinkedList.py @@ -0,0 +1,127 @@ +from parcels.nodes.Node import * +from sortedcontainers import SortedList +from numpy import int32, int64, uint32, uint64 +# from copy import copy +from copy import deepcopy +# import gc + + +# ========================== # +# = Verdict: nice try, but = # +# = overrides to the del() = # +# = function won't work. = # +# ========================== # +class RealList(SortedList): + dtype = None + + def __init__(self, iterable=None, dtype=Node): + super(RealList, self).__init__(iterable) + self.dtype = dtype + + def __del__(self): + self.clear() + + def clear(self): + """Remove all the elements from the list.""" + n = self.__len__() + # print("# remaining items: {}".format(n)) + if n > 0: + # print("Deleting {} elements ...".format(n)) + while (n > 0): + # val = self.pop(); del val + # super().__delitem__(n-1) + # self.pop() + + self.__getitem__(-1).unlink() + self.__delitem__(-1) + n = self.__len__() + # print("Deleting {} elements ...".format(n)) + # gc.collect() + super()._clear() + + def __new__(cls, iterable=None, key=None, load=1000, dtype=Node): + return object.__new__(cls) + + def add(self, val): + assert type(val) == self.dtype + if isinstance(val, Node): + self._add_by_node(val) + elif isinstance(val, int) or type(val) in [int32, uint32, int64, uint64]: + self._add_by_id(val) + else: + self._add_by_pdata(val) + + def _add_by_node(self, val): + n = self.__len__() + index = self.bisect_right(val) + if index < n: + next_node = self.__getitem__(index) + else: + next_node = None + if index > 0: + prev_node = self.__getitem__(index - 1) + else: + prev_node = None + if next_node is not None: + next_node.set_prev(val) + val.set_next(next_node) + if prev_node is not None: + prev_node.set_next(val) + val.set_prev(prev_node) + super().add(val) + + def _add_by_id(self, val): + n = self.__len__() + index = self.bisect_right(val) + if index < n: + next_node = self.__getitem__(index) + else: + next_node = None + if index > 0: + prev_node = self.__getitem__(index - 1) + else: + prev_node = None + new_node = self.dtype(prev=prev_node, next=next_node, id=val) + super().add(new_node) + + def _add_by_pdata(self, val): + new_node = self.dtype(data=val) + super().add(new_node) + n = self.__len__() + index = self.index(new_node) + if index < (n - 2): + next_node = self.__getitem__(index + 1) + else: + next_node = None + if index > 0: + prev_node = self.__getitem__(index - 1) + else: + prev_node = None + if next_node is not None: + next_node.set_prev(new_node) + new_node.set_next(next_node) + if prev_node is not None: + prev_node.set_next(new_node) + new_node.set_prev(prev_node) + + def append(self, val): + self.add(val) + + def pop(self, idx=-1, deepcopy_elem=False): + """ + Because we expect the return node to be of use, + the actual node is NOT physically deleted (by calling + the destructor). pop() only dereferences the object + in the list. The parameter 'deepcopy_elem' can be set so as + to physically delete the list object and return a deep copy (unlisted). + :param idx: + :param deepcopy_elem: + :return: + """ + if deepcopy_elem: + val = super().pop(idx) + result = deepcopy(val) + # result = deepcopy(self.__getitem__(idx)) + del val + return result + return super().pop(idx) diff --git a/parcels/nodes/Node.py b/parcels/nodes/Node.py new file mode 100644 index 0000000000..aacb6afb5c --- /dev/null +++ b/parcels/nodes/Node.py @@ -0,0 +1,395 @@ +import ctypes +import sys +import os +from parcels.tools import idgen +from parcels.wrapping import * +from numpy import int32, int64, uint32, uint64 +import random + + +class Node(object): + # theoretical size: 97 bytes + prev = None + next = None + id = None + data = None + registered = False + + def __init__(self, prev=None, next=None, id=None, data=None): + self.registered = True + if prev is not None: + assert (isinstance(prev, Node)) + self.prev = prev + else: + self.prev = None + if next is not None: + assert (isinstance(next, Node)) + self.next = next + else: + self.next = None + if id is not None and (isinstance(id, int) or type(id) in [int32, uint32, int64, uint64]) and (id >= 0): + self.id = id + elif id is None and data is not None: + try: + self.id = data.id + except (ValueError, AttributeError): + self.id = None + if self.id is None: + try: + self.id = idgen.nextID(data.lon, data.lat, data.depth, data.time) + except (ValueError, AttributeError): + self.id = None + else: + self.id = None + self.data = data + + def __deepcopy__(self, memodict={}): + result = type(self)(prev=None, next=None, id=-1, data=None) + result.registered = True + result.id = self.id + result.next = self.next + result.prev = self.prev + result.data = self.data + return result + + def __del__(self): + # print("Node.del() is called.") + self.unlink() + del self.data + idgen.releaseID(self.id) + + def unlink(self): + # print("Node.unlink() [id={}] is called.".format(self.id)) + if self.registered: + if self.prev is not None: + self.prev.set_next(self.next) + if self.next is not None: + self.next.set_prev(self.prev) + self.registered = False + self.prev = None + self.next = None + + def __iter__(self): + return self + + def __next__(self): + if self.next is None: + raise StopIteration + return self.next + + def __eq__(self, other): + if type(self) is not type(other): + return False + if (self.data is not None) and (other.data is not None): + return self.data == other.data + else: + return self.id == other.id + + def __ne__(self, other): + return not (self == other) + + def __lt__(self, other): + # print("less-than({} vs. {})".format(str(self),str(other))) + if type(self) is not type(other): + err_msg = "This object and the other object (type={}) do note have the same type.".format(str(type(other))) + raise AttributeError(err_msg) + return self.id < other.id + + def __le__(self, other): + if type(self) is not type(other): + err_msg = "This object and the other object (type={}) do note have the same type.".format(str(type(other))) + raise AttributeError(err_msg) + return self.id <= other.id + + def __gt__(self, other): + if type(self) is not type(other): + err_msg = "This object and the other object (type={}) do note have the same type.".format(str(type(other))) + raise AttributeError(err_msg) + return self.id > other.id + + def __ge__(self, other): + if type(self) is not type(other): + err_msg = "This object and the other object (type={}) do note have the same type.".format(str(type(other))) + raise AttributeError(err_msg) + return self.id >= other.id + + def __repr__(self): + return '<%s.%s object at %s>' % ( + self.__class__.__module__, + self.__class__.__name__, + hex(id(self)) + ) + + def __str__(self): + return "Node(p: {}, n: {}, id: {}, d: {})".format(repr(self.prev), repr(self.next), self.id, repr(self.data)) + + def __sizeof__(self): + obj_size = sys.getsizeof(object)+sys.getsizeof(object)+sys.getsizeof(self.id) + if self.data is not None: + obj_size += sys.getsizeof(self.data) + return obj_size + + def set_prev(self, prev): + self.prev = prev + + def set_next(self, next): + self.next = next + + def set_data(self, data): + self.data = data + + def reset_data(self): + self.data = None + + def reset_prev(self): + self.prev = None + + def reset_next(self): + self.prev = None + +# global node_c_interface +node_c_interface = None +# global c_funcs +c_funcs = None + + +class NodeJIT(Node, ctypes.Structure): + # theoretical size: 97 bytes + 260 bytes (ctypes data structure) + 56 bytes (ctypes func-refs) + [616 (one time - having the ctypes function link)] + _fields_ = [('_c_prev_p', ctypes.c_void_p), + ('_c_next_p', ctypes.c_void_p), + ('_c_data_p', ctypes.c_void_p), + ('_c_pu_affinity', ctypes.c_int)] + + init_node_c = None + set_prev_ptr_c = None + set_next_ptr_c = None + set_data_ptr_c = None + reset_prev_ptr_c = None + reset_next_ptr_c = None + reset_data_ptr_c = None + + def __init__(self, prev=None, next=None, id=None, data=None): + super().__init__(prev=prev, next=next, id=id, data=data) + if not c_lib_register.is_created("node") or not c_lib_register.is_compiled("node") or not c_lib_register.is_loaded("node"): + c_lib_register.load("node", src_dir=os.path.dirname(os.path.abspath(__file__))) + c_lib_register.register("node") + self.registered = True + global node_c_interface + if node_c_interface is None: + node_c_interface = c_lib_register.get("node") # ["node"] + + func_params = NodeJIT_func_params() + # func_params = [] + # func_params.append({"name": 'init_node', "return": None, "arguments": [ctypes.POINTER(NodeJIT)]}) + # func_params.append({"name": 'set_prev_ptr', "return": None, "arguments": [ctypes.POINTER(NodeJIT), ctypes.POINTER(NodeJIT)]}) + # func_params.append({"name": 'set_next_ptr', "return": None, "arguments": [ctypes.POINTER(NodeJIT), ctypes.POINTER(NodeJIT)]}) + # func_params.append({"name": 'set_data_ptr', "return": None, "arguments": [ctypes.POINTER(NodeJIT), ctypes.c_void_p]}) + # func_params.append({"name": 'set_pu_affinity', "return": None, "arguments": [ctypes.POINTER(NodeJIT), ctypes.c_int]}) + # func_params.append({"name": 'get_pu_affinity', "return": ctypes.c_int, "arguments": [ctypes.POINTER(NodeJIT)]}) + # func_params.append({"name": 'reset_prev_ptr', "return": None, "arguments": [ctypes.POINTER(NodeJIT)]}) + # func_params.append({"name": 'reset_next_ptr', "return": None, "arguments": [ctypes.POINTER(NodeJIT)]}) + # func_params.append({"name": 'reset_data_ptr', "return": None, "arguments": [ctypes.POINTER(NodeJIT)]}) + # func_params.append({"name": 'reset_pu_affinity', "return": None, "arguments": [ctypes.POINTER(NodeJIT)]}) + + # -- Question: do we REALLY need to look them up all the time ? We count store that library link and just re-use it for each particle -- # + global c_funcs + if c_funcs is None: + c_funcs = node_c_interface.load_functions(func_params) + self.link_c_functions(c_funcs) + # self.init_node_c = c_funcs['init_node'] + # self.set_prev_ptr_c = c_funcs['set_prev_ptr'] + # self.set_next_ptr_c = c_funcs['set_next_ptr'] + # self.set_data_ptr_c = c_funcs['set_data_ptr'] + # self.reset_prev_ptr_c = c_funcs['reset_prev_ptr'] + # self.reset_next_ptr_c = c_funcs['reset_next_ptr'] + # self.reset_data_ptr_c = c_funcs['reset_data_ptr'] + + self.init_node_c(self) + + if self.prev is not None and isinstance(self.prev, NodeJIT): + self.set_prev_ptr_c(self, self.prev) + else: + self.reset_prev_ptr_c(self) + # self._c_self_p = ctypes.cast(self, ctypes.c_void_p) + if self.next is not None and isinstance(self.next, NodeJIT): + self.set_next_ptr_c(self, self.next) + else: + self.reset_next_ptr_c(self) + + if self.data is not None: # and isinstance(ctypes.c_void_p): + # self._c_data_p = ctypes.cast(self.data, ctypes.c_void_p) + try: + self.set_data_ptr_c(self, self.data.cdata()) + except AttributeError: + self.set_data_ptr_c(self, ctypes.cast(self.data, ctypes.c_void_p)) + else: + self.reset_data_ptr_c(self) + + def __deepcopy__(self, memodict={}): + result = type(self)(prev=None, next=None, id=-1, data=None) + result.id = self.id + result.next = self.next + result.prev = self.prev + result.data = self.data + c_lib_register.register("node") # should rather be done by 'result' internally + result.registered = True + result.init_node_c = self.init_node_c + result.set_prev_ptr_c = self.set_prev_ptr_c + result.set_next_ptr_c = self.set_next_ptr_c + result.set_data_ptr_c = self.set_data_ptr_c + result.reset_prev_ptr_c = self.reset_prev_ptr_c + result.reset_next_ptr_c = self.reset_next_ptr_c + result.reset_data_ptr_c = self.reset_data_ptr_c + result.init_node_c(self) + if result.prev is not None and isinstance(result.prev, NodeJIT): + result.set_prev_ptr_c(result, result.prev) + else: + result.reset_prev_ptr_c(result) + if result.next is not None and isinstance(result.next, NodeJIT): + result.set_next_ptr_c(result, result.next) + else: + result.reset_next_ptr_c(result) + + if result.data is not None: + result.set_data_ptr_c(result, ctypes.cast(result.data, ctypes.c_void_p)) + else: + result.reset_data_ptr_c(result) + return result + + def __del__(self): + # print("NodeJIT.del() [id={}] is called.".format(self.id)) + self.unlink() + del self.data + idgen.releaseID(self.id) + + def unlink(self): + # print("NodeJIT.unlink() [id={}] is called.".format(self.id)) + if self.registered: + if self.prev is not None: + if self.next is not None: + self.prev.set_next(self.next) + else: + # self.reset_next_ptr_c(self.prev) + self.prev.reset_next() + if self.next is not None: + if self.prev is not None: + self.next.set_prev(self.prev) + else: + # self.reset_prev_ptr_c(self.next) + self.next.reset_prev() + self.reset_prev_ptr_c(self) + self.reset_next_ptr_c(self) + self.reset_data_ptr_c(self) + c_lib_register.deregister("node") + self.registered = False + self.prev = None + self.next = None + + def __repr__(self): + return super().__repr__() + + def __str__(self): + return super().__str__() + + def __sizeof__(self): + return super().__sizeof__()+sys.getsizeof(self._fields_) + + def __eq__(self, other): + return super().__eq__(other) + + def __ne__(self, other): + return super().__ne__(other) + + def __lt__(self, other): + return super().__lt__(other) + + def __le__(self, other): + return super().__le__(other) + + def __gt__(self, other): + return super().__gt__(other) + + # def __ge__(self, other): + # return super().__ge__(other) + + def link_c_functions(self, c_func_dict): + self.init_node_c = c_func_dict['init_node'] + self.set_prev_ptr_c = c_func_dict['set_prev_ptr'] + self.set_next_ptr_c = c_func_dict['set_next_ptr'] + self.set_data_ptr_c = c_func_dict['set_data_ptr'] + self.reset_prev_ptr_c = c_func_dict['reset_prev_ptr'] + self.reset_next_ptr_c = c_func_dict['reset_next_ptr'] + self.reset_data_ptr_c = c_func_dict['reset_data_ptr'] + + def set_data(self, data): + super().set_data(data) + if self.registered: + self.update_data() + + def set_prev(self, prev): + super().set_prev(prev) + if self.registered: + self.update_prev() + + def set_next(self, next): + super().set_next(next) + if self.registered: + self.update_next() + + def reset_data(self): + super().reset_data() + if self.registered: + self.reset_data_ptr_c(self) + + def reset_prev(self): + super().reset_prev() + if self.registered: + self.reset_prev_ptr_c(self) + + def reset_next(self): + super().reset_next() + if self.registered: + self.reset_next_ptr_c(self) + + def update_prev(self): + if self.prev is not None and isinstance(self.prev, NodeJIT): + # self._c_prev_p = ctypes.cast(self.prev, ctypes.c_void_p) + # self._c_prev_p = self.prev._c_self_p + self.set_prev_ptr_c(self, self.prev) + else: + self.reset_prev_ptr_c(self) + + def update_next(self): + if self.next is not None and isinstance(self.next, NodeJIT): + # self._c_next_p = ctypes.cast(self.next, ctypes.c_void_p) + # self._c_next_p = self.next._c_self_p + self.set_next_ptr_c(self, self.next) + else: + self.reset_next_ptr_c(self) + + def update_data(self): + if self.data is not None: # and isinstance(ctypes.c_void_p): + # self._c_data_p = ctypes.cast(self.data, ctypes.c_void_p) + try: + self.set_data_ptr_c(self, self.data.cdata()) + except AttributeError: + self.set_data_ptr_c(self, ctypes.cast(self.data, ctypes.c_void_p)) + else: + self.reset_data_ptr_c(self) + + +def NodeJIT_func_params(): #node_c_func_params = + return [{"name": 'init_node', "return": None, "arguments": [ctypes.POINTER(NodeJIT)]}, + {"name": 'set_prev_ptr', "return": None, "arguments": [ctypes.POINTER(NodeJIT), ctypes.POINTER(NodeJIT)]}, + {"name": 'set_next_ptr', "return": None, "arguments": [ctypes.POINTER(NodeJIT), ctypes.POINTER(NodeJIT)]}, + {"name": 'set_next_ptr', "return": None, "arguments": [ctypes.POINTER(NodeJIT), ctypes.POINTER(NodeJIT)]}, + {"name": 'set_data_ptr', "return": None, "arguments": [ctypes.POINTER(NodeJIT), ctypes.c_void_p]}, + {"name": 'set_pu_affinity', "return": None, "arguments": [ctypes.POINTER(NodeJIT), ctypes.c_int]}, + {"name": 'get_pu_affinity', "return": ctypes.c_int, "arguments": [ctypes.POINTER(NodeJIT)]}, + {"name": 'reset_prev_ptr', "return": None, "arguments": [ctypes.POINTER(NodeJIT)]}, + {"name": 'reset_next_ptr', "return": None, "arguments": [ctypes.POINTER(NodeJIT)]}, + {"name": 'reset_data_ptr', "return": None, "arguments": [ctypes.POINTER(NodeJIT)]}, + {"name": 'reset_pu_affinity', "return": None, "arguments": [ctypes.POINTER(NodeJIT)]}] + + + diff --git a/parcels/nodes/__init__.py b/parcels/nodes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/parcels/nodes/node.c b/parcels/nodes/node.c new file mode 100644 index 0000000000..2e2967851c --- /dev/null +++ b/parcels/nodes/node.c @@ -0,0 +1,61 @@ +/** + * + **/ +#include "node.h" + + + +void init_node(NodeJIT* self_node) { + (*self_node)._c_prev_p = NULL; + (*self_node)._c_next_p = NULL; + (*self_node)._c_data_p = NULL; + (*self_node)._c_pu_affinity = -1; +} + +void set_prev_ptr(NodeJIT* self_node, NodeJIT* prev_node) { + (*self_node)._c_prev_p = (void*)prev_node; +} + +void set_next_ptr(NodeJIT* self_node, NodeJIT* next_node) { + (*self_node)._c_next_p = (void*)next_node; +} + +void set_data_ptr(NodeJIT* self_node, void* data_ptr) { + (*self_node)._c_data_p = (void*)data_ptr; +} + +void set_pu_affinity(NodeJIT* self_node, int pu_number) { + (*self_node)._c_pu_affinity = pu_number; +} + +int get_pu_affinity(NodeJIT* self_node) { + return (*self_node)._c_pu_affinity; +} + +void set_pu_num(NodeJIT* self_node, int pu_number) { + set_pu_affinity(self_node, pu_number); +} + +int get_pu_num(NodeJIT* self_node) { + return get_pu_affinity(self_node); +} + +void reset_prev_ptr(NodeJIT* self_node) { + (*self_node)._c_prev_p = NULL; +} + +void reset_next_ptr(NodeJIT* self_node) { + (*self_node)._c_next_p = NULL; +} + +void reset_data_ptr(NodeJIT* self_node) { + (*self_node)._c_data_p = NULL; +} + +void reset_pu_affinity(NodeJIT* self_node) { + (*self_node)._c_pu_affinity = -1; +} + +void reset_pu_number(NodeJIT* self_node) { + reset_pu_affinity(self_node); +} \ No newline at end of file diff --git a/parcels/nodes/node.h b/parcels/nodes/node.h new file mode 100644 index 0000000000..d7d89f878a --- /dev/null +++ b/parcels/nodes/node.h @@ -0,0 +1,41 @@ +#ifndef _NODE_H +#define _NODE_H +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include + +typedef struct _NodeJIT { + void* _c_prev_p; + //void* _c_self_p; + void* _c_next_p; + void* _c_data_p; + int _c_pu_affinity; +} NodeJIT; + +void init_node(NodeJIT* self_node); +void set_prev_ptr(NodeJIT* self_node, NodeJIT* prev_node); +void set_next_ptr(NodeJIT* self_node, NodeJIT* next_node); +void set_data_ptr(NodeJIT* self_node, void* data_ptr); +void set_pu_affinity(NodeJIT* self_node, int pu_number); +int get_pu_affinity(NodeJIT* self_node); +void set_pu_num(NodeJIT* self_node, int pu_number); +int get_pu_num(NodeJIT* self_node); +void reset_prev_ptr(NodeJIT* self_node); +void reset_next_ptr(NodeJIT* self_node); +void reset_data_ptr(NodeJIT* self_node); +void reset_pu_affinity(NodeJIT* self_node); +void reset_pu_number(NodeJIT* self_node); + + + + +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/parcels/particle.py b/parcels/particle.py index ff093b13ba..b120ea1987 100644 --- a/parcels/particle.py +++ b/parcels/particle.py @@ -3,13 +3,15 @@ import numpy as np +# from numpy.random import random +# from parcels.rng import random from parcels.field import Field from parcels.tools.error import ErrorCode from parcels.tools.loggers import logger __all__ = ['ScipyParticle', 'JITParticle', 'Variable'] -indicators_64bit = [np.float64, np.int64, c_void_p] +indicators_64bit = [np.float64, np.uint64, np.int64, c_void_p] class Variable(object): @@ -33,7 +35,12 @@ def __get__(self, instance, cls): if instance is None: return self if issubclass(cls, JITParticle): - return instance._cptr.__getitem__(self.name) + # val = instance._cptr.__getitem__(self.name) + # if isinstance(val, np.ndarray): + # val = val[0] + # return val + # return instance._cptr.__getitem__(self.name)[0] + return instance.get_cptr().__getitem__(self.name) else: return getattr(instance, "_%s" % self.name, self.initial) @@ -43,6 +50,12 @@ def __set__(self, instance, value): else: setattr(instance, "_%s" % self.name, value) +# def random(self, pinstance): +# if isinstance(pinstance, JITParticle): +# pinstance._cptr.__setitem__(self.name, random().astype(dtype=self.dtype)) +# else: +# setattr(pinstance, "_%s" % self.name, random().astype(dtype=self.dtype)) + def __repr__(self): return "PVar<%s|%s>" % (self.name, self.dtype) @@ -100,6 +113,11 @@ def size(self): """Size of the underlying particle struct in bytes""" return sum([8 if v.is64bit() else 4 for v in self.variables]) + @property + def bsize(self): + """Size of the underlying particle struct in bytes""" + return sum([v.dtype.itemsize for v in self.variables]) + @property def supported_dtypes(self): """List of all supported numpy dtypes. All others are not supported""" @@ -107,7 +125,7 @@ def supported_dtypes(self): # Developer note: other dtypes (mostly 2-byte ones) are not supported now # because implementing and aligning them in cgen.GenerableStruct is a # major headache. Perhaps in a later stage - return [np.int32, np.int64, np.float32, np.double, np.float64, c_void_p] + return [np.int32, np.uint32, np.int64, np.uint64, np.float32, np.double, np.float64, c_void_p] class _Particle(object): @@ -140,6 +158,9 @@ def __init__(self): # Placeholder for explicit error handling self.exception = None + def __del__(self): + pass + @classmethod def getPType(cls): return ParticleType(cls) @@ -169,12 +190,14 @@ class ScipyParticle(_Particle): lon = Variable('lon', dtype=np.float32) lat = Variable('lat', dtype=np.float32) depth = Variable('depth', dtype=np.float32) - time = Variable('time', dtype=np.float64) - id = Variable('id', dtype=np.int32) + time = Variable('time', dtype=np.float64, initial=np.nan) + # id = Variable('id', dtype=np.int32) + id = Variable('id', dtype=np.uint64) + index = Variable('index', dtype=np.int32) dt = Variable('dt', dtype=np.float64, to_write=False) state = Variable('state', dtype=np.int32, initial=ErrorCode.Evaluate, to_write=False) - def __init__(self, lon, lat, pid, fieldset, depth=0., time=0., cptr=None): + def __init__(self, lon, lat, pid, fieldset, depth=0., time=0., dt=None, cptr=None, index=-1): # Enforce default values through Variable descriptor type(self).lon.initial = lon @@ -182,11 +205,19 @@ def __init__(self, lon, lat, pid, fieldset, depth=0., time=0., cptr=None): type(self).depth.initial = depth type(self).time.initial = time type(self).id.initial = pid - _Particle.lastID = max(_Particle.lastID, pid) - type(self).dt.initial = None + # if isinstance(pid, int): + if type(pid) in [int, np.uint64, np.int64]: + _Particle.lastID = max(_Particle.lastID, pid) + else: + _Particle.lastID = max(_Particle.lastID, index) + type(self).dt.initial = dt + type(self).index.initial = index super(ScipyParticle, self).__init__() self._next_dt = None + def __del__(self): + super(ScipyParticle, self).__del__() + def __repr__(self): time_string = 'not_yet_set' if self.time is None or np.isnan(self.time) else "{:f}".format(self.time) str = "P[%d](lon=%f, lat=%f, depth=%f, " % (self.id, self.lon, self.lat, self.depth) @@ -195,6 +226,13 @@ def __repr__(self): str += "%s=%f, " % (var, getattr(self, var)) return str + "time=%s)" % time_string +# def random(self): +# ptype = self.getPType() +# for var in ptype.variables: +# var_value = getattr(self, var.name) +# var_value.random() +# setattr(self, var.name, var_value) + def delete(self): self.state = ErrorCode.Delete @@ -221,6 +259,49 @@ def update_next_dt(self, next_dt=None): else: self._next_dt = next_dt + def __eq__(self, other): + if type(self) is not type(other): + return False + ids_eq = (self.id == other.id) + attr_eq = True + attr_eq &= (self.lon == other.lon) + attr_eq &= (self.lat == other.lat) + attr_eq &= (self.depth == other.depth) + attr_eq &= (self.time == other.time) + attr_eq &= (self.dt == other.dt) + return ids_eq and attr_eq + + def __ne__(self, other): + return not (self == other) + + def __lt__(self, other): + if type(self) is not type(other): + err_msg = "This object and the other object (type={}) do note have the same type.".format(str(type(other))) + raise AttributeError(err_msg) + return self.id < other.id + + def __le__(self, other): + if type(self) is not type(other): + err_msg = "This object and the other object (type={}) do note have the same type.".format(str(type(other))) + raise AttributeError(err_msg) + return self.id <= other.id + + def __gt__(self, other): + if type(self) is not type(other): + err_msg = "This object and the other object (type={}) do note have the same type.".format(str(type(other))) + raise AttributeError(err_msg) + return self.id > other.id + + def __ge__(self, other): + if type(self) is not type(other): + err_msg = "This object and the other object (type={}) do note have the same type.".format(str(type(other))) + raise AttributeError(err_msg) + return self.id >= other.id + + def __sizeof__(self): + ptype = self.getPType() + return sum([v.size for v in ptype.variables]) + class JITParticle(ScipyParticle): """Particle class for JIT-based (Just-In-Time) Particle objects @@ -247,14 +328,60 @@ def __init__(self, *args, **kwargs): if self._cptr is None: # Allocate data for a single particle ptype = self.getPType() - self._cptr = np.empty(1, dtype=ptype.dtype)[0] + self._cptr = np.empty(1, dtype=ptype.dtype) # [0] super(JITParticle, self).__init__(*args, **kwargs) fieldset = kwargs.get('fieldset') - for index in ['xi', 'yi', 'zi', 'ti']: - if index != 'ti': - setattr(self, index, np.zeros((fieldset.gridset.size), dtype=np.int32)) - else: - setattr(self, index, -1*np.ones((fieldset.gridset.size), dtype=np.int32)) - setattr(self, index+'p', getattr(self, index).ctypes.data_as(c_void_p)) - setattr(self, 'c'+index, getattr(self, index+'p').value) + if fieldset is not None: + for index in ['xi', 'yi', 'zi', 'ti']: + if index != 'ti': + setattr(self, index, np.zeros((fieldset.gridset.size), dtype=np.int32)) + else: + setattr(self, index, -1*np.ones((fieldset.gridset.size), dtype=np.int32)) + setattr(self, index+'p', getattr(self, index).ctypes.data_as(c_void_p)) + setattr(self, 'c'+index, getattr(self, index+'p').value) + + def __del__(self): + super(JITParticle, self).__del__() + + def cdata(self): + if self._cptr is None: + return None + return self._cptr.ctypes.data_as(c_void_p) + + def set_cptr(self, value): + if isinstance(value, np.ndarray): + ptype = self.getPType() + self._cptr = np.array(value, dtype=ptype.dtype) + else: + self._cptr = None + + def get_cptr(self): + if isinstance(self._cptr, np.ndarray): + return self._cptr[0] + return self._cptr + + def reset_cptr(self): + self._cptr = None + + def __eq__(self, other): + return super(JITParticle, self).__eq__(other) + + def __ne__(self, other): + return not (self == other) + + def __lt__(self, other): + return super(JITParticle, self).__lt__(other) + + def __le__(self, other): + return super(JITParticle, self).__le__(other) + + def __gt__(self, other): + return super(JITParticle, self).__gt__(other) + + def __ge__(self, other): + return super(JITParticle, self).__ge__(other) + + def __sizeof__(self): + ptype = self.getPType() + return sum([v.size for v in ptype.variables]) diff --git a/parcels/particlefile_node.py b/parcels/particlefile_node.py new file mode 100644 index 0000000000..aa468dce13 --- /dev/null +++ b/parcels/particlefile_node.py @@ -0,0 +1,564 @@ +"""Module controlling the writing of ParticleSets to NetCDF file""" +import os +import random +import shutil +import string +from datetime import timedelta as delta +from glob import glob + +import netCDF4 +import numpy as np + +from parcels.tools.error import ErrorCode +from parcels.tools.loggers import logger +try: + from mpi4py import MPI +except: + MPI = None +try: + from parcels._version import version as parcels_version +except: + raise EnvironmentError('Parcels version can not be retrieved. Have you run ''python setup.py install''?') +try: + from os import getuid +except: + # Windows does not have getuid(), so define to simply return 'tmp' + def getuid(): + return 'tmp' +# from parcels.particleset_node import ParticleSet +# from parcels.particleset_vectorized import ParticleSet as BadParticleSet + + +__all__ = ['ParticleFile'] + + +def _is_particle_started_yet(particle, time): + """We don't want to write a particle that is not started yet. + Particle will be written if: + * particle.time is equal to time argument of pfile.write() + * particle.time is before time (in case particle was deleted between previous export and current one) + """ + return (particle.dt*particle.time <= particle.dt*time or np.isclose(particle.time, time)) + + +def _set_calendar(origin_calendar): + if origin_calendar == 'np_datetime64': + return 'standard' + else: + return origin_calendar + +# ==== Comment: the way this class is written is really unsustainable / non-maintainable. This is because it holds +# ==== vectorized attribute copies of the particle set als well as their mapped memory areas in the NetCDF file, +# ==== but NONE of the attributes are visible in the constructor. Basically: one wanted to subclass the ParticleSet +# ==== and the NetCDF.Dataset, but instead mapped their attributes manually by lazy binding into this class. +# ==== This obviously circumvents any variable checks and is also prone to leave garbage memory open during the computation. + + +class ParticleFile(object): + """Initialise trajectory output. + :param name: Basename of the output file + :param particleset: ParticleSet to output + :param outputdt: Interval which dictates the update frequency of file output + while ParticleFile is given as an argument of ParticleSet.execute() + It is either a timedelta object or a positive double. + :param write_ondelete: Boolean to write particle data only when they are deleted. Default is False + :param convert_at_end: Boolean to convert npy files to netcdf at end of run. Default is True + :param tempwritedir: directories to write temporary files to during executing. + Default is out-XXXXXX where Xs are random capitals. Files for individual + processors are written to subdirectories 0, 1, 2 etc under tempwritedir + :param pset_info: dictionary of info on the ParticleSet, stored in tempwritedir/XX/pset_info.npy, + used to create NetCDF file from npy-files. + """ + + def __init__(self, name, particleset, outputdt=np.infty, write_ondelete=False, convert_at_end=True, + tempwritedir=None, pset_info=None): + # if particleset is not None and isinstance(particleset, ParticleSet): + # err_msg_addendum = "Classes do not match." + # if isinstance(particleset, BadParticleSet): + # err_msg_addendum = "You tried to initialize a node-based particle file with a vectorized particle set - action forbidden." + # logger.error("'particleset' is not a node-based Particle Set. %s. Exiting" % (err_msg_addendum)) + # exit() + + self.write_ondelete = write_ondelete + self.convert_at_end = convert_at_end + self.outputdt = outputdt + self.lasttime_written = None # variable to check if time has been written already + + self.dataset = None + self.metadata = {} + if pset_info is not None: + for v in pset_info.keys(): + setattr(self, v, pset_info[v]) + else: + self.name = name + self.particleset = particleset + self.parcels_mesh = self.particleset.fieldset.gridset.grids[0].mesh + self.time_origin = self.particleset.time_origin + self.lonlatdepth_dtype = self.particleset.lonlatdepth_dtype + self.var_names = [] + self.var_names_once = [] + for v in self.particleset.ptype.variables: + if v.to_write == 'once': + self.var_names_once += [v.name] + elif v.to_write is True: + self.var_names += [v.name] + if len(self.var_names_once) > 0: + self.written_once = [] + self.file_list_once = [] + + self.file_list = [] + self.time_written = [] + # self.num_ids_written = -1 # => Idea: take IDgenerator.length that always gives you the maximum number of particles! + # self.maxid_written = -1 + self.max_index_written = -1 + + if tempwritedir is None: + tempwritedir = os.path.join(os.path.dirname(str(self.name)), "out-%s" + % ''.join(random.choice(string.ascii_uppercase) for _ in range(8))) + + if MPI: + mpi_rank = MPI.COMM_WORLD.Get_rank() + self.tempwritedir_base = MPI.COMM_WORLD.bcast(tempwritedir, root=0) + else: + self.tempwritedir_base = tempwritedir + mpi_rank = 0 + self.tempwritedir = os.path.join(self.tempwritedir_base, "%d" % mpi_rank) + + if pset_info is None: # otherwise arrive here from convert_npydir_to_netcdf + self.delete_tempwritedir() + + def open_netcdf_file(self, data_shape): + """Initialise NetCDF4.Dataset for trajectory output. + The output follows the format outlined in the Discrete Sampling Geometries + section of the CF-conventions: + http://cfconventions.org/cf-conventions/v1.6.0/cf-conventions.html#discrete-sampling-geometries + The current implementation is based on the NCEI template: + http://www.nodc.noaa.gov/data/formats/netcdf/v2.0/trajectoryIncomplete.cdl + :param data_shape: shape of the variables in the NetCDF4 file + """ + extension = os.path.splitext(str(self.name))[1] + fname = self.name if extension in ['.nc', '.nc4'] else "%s.nc" % self.name + if os.path.exists(str(fname)): + os.remove(str(fname)) + + # ==== coords = self._create_trajectory_file(fname, data_shape) ==== # + self.dataset = netCDF4.Dataset(fname, "w", format="NETCDF4") + self.dataset.createDimension("obs", data_shape[1]) + self.dataset.createDimension("traj", data_shape[0]) + coords = ("traj", "obs") + self.dataset.feature_type = "trajectory" + self.dataset.Conventions = "CF-1.6/CF-1.7" + self.dataset.ncei_template_version = "NCEI_NetCDF_Trajectory_Template_v2.0" + self.dataset.parcels_version = parcels_version + self.dataset.parcels_mesh = self.parcels_mesh + + # ==== self._create_trajectory_records(coords=coords) ==== # + # Create ID variable according to CF conventions + # self.id = self.dataset.createVariable("trajectory", "i4", coords, fill_value=-2**(31)) # maxint32 fill_value + self.id = self.dataset.createVariable("trajectory", "u8", coords, fill_value=np.iinfo(np.uint64).max) + self.id.long_name = "Unique identifier for each particle" + self.id.cf_role = "trajectory_id" + + self.index = self.dataset.createVariable("index", "i4", coords, fill_value=-2**(31)) + self.index.long_name = "running (zero-based continuous) indexing element referring to the LOCAL index of a particle within one timestep" + + # Create time, lat, lon and z variables according to CF conventions: + self.time = self.dataset.createVariable("time", "f8", coords, fill_value=np.nan) + self.time.long_name = "" + self.time.standard_name = "time" + if self.time_origin.calendar is None: + self.time.units = "seconds" + else: + self.time.units = "seconds since " + str(self.time_origin) + self.time.calendar = 'standard' if self.time_origin.calendar == 'np_datetime64' else self.time_origin.calendar + self.time.axis = "T" + + if self.lonlatdepth_dtype is np.float64: + lonlatdepth_precision = "f8" + else: + lonlatdepth_precision = "f4" + + self.lat = self.dataset.createVariable("lat", lonlatdepth_precision, coords, fill_value=np.nan) + self.lat.long_name = "" + self.lat.standard_name = "latitude" + self.lat.units = "degrees_north" + self.lat.axis = "Y" + + self.lon = self.dataset.createVariable("lon", lonlatdepth_precision, coords, fill_value=np.nan) + self.lon.long_name = "" + self.lon.standard_name = "longitude" + self.lon.units = "degrees_east" + self.lon.axis = "X" + + self.z = self.dataset.createVariable("z", lonlatdepth_precision, coords, fill_value=np.nan) + self.z.long_name = "" + self.z.standard_name = "depth" + self.z.units = "m" + self.z.positive = "down" + + for vname in self.var_names: + if vname not in ['time', 'lat', 'lon', 'depth', 'id', 'index']: + # ==== Comment - hm, that looks a bit 'crooked' - certainly from getting the ptype + # ==== you can parse the variables' bytesize - or make a "numpy-size to netcdf-size' conversion table here ... + setattr(self, vname, self.dataset.createVariable(vname, "f4", coords, fill_value=np.nan)) + getattr(self, vname).long_name = "" + getattr(self, vname).standard_name = vname + getattr(self, vname).units = "unknown" + + for vname in self.var_names_once: + setattr(self, vname, self.dataset.createVariable(vname, "f4", "traj", fill_value=np.nan)) + # ==== Comment - hm, that looks a bit 'crooked' - certainly from getting the ptype + # ==== you can parse the variables' bytesize - or make a "numpy-size to netcdf-size' conversion table here ... + getattr(self, vname).long_name = "" + getattr(self, vname).standard_name = vname + getattr(self, vname).units = "unknown" + + # ==== self._create_metadata_records() ==== # + for name, message in self.metadata.items(): + setattr(self.dataset, name, message) + + def __del__(self): + if self.convert_at_end: + self.close() + + def close(self, delete_tempfiles=True): + """Close the ParticleFile object by exporting and then deleting + the temporary npy files""" + self.export() + mpi_rank = MPI.COMM_WORLD.Get_rank() if MPI else 0 + if mpi_rank == 0: + if delete_tempfiles: + self.delete_tempwritedir(tempwritedir=self.tempwritedir_base) + self.convert_at_end = False + + def add_metadata(self, name, message): + """Add metadata to :class:`parcels.particleset.ParticleSet` + :param name: Name of the metadata variabale + :param message: message to be written + """ + if self.dataset is None: + self.metadata[name] = message + else: + setattr(self.dataset, name, message) + + def convert_pset_to_dict(self, pset, time, deleted_only=False): + """Convert all Particle data from one time step to a python dictionary. + :param pset: ParticleSet object to write + :param time: Time at which to write ParticleSet + :param deleted_only: Flag to write only the deleted Particles + returns two dictionaries: one for all variables to be written each outputdt, + and one for all variables to be written once + """ + data_dict = {} + data_dict_once = {} + + time = time.total_seconds() if isinstance(time, delta) else time + + if self.lasttime_written != time and \ + (self.write_ondelete is False or deleted_only is True): + # if len(pset) == 0: + if pset.size == 0: + logger.warning("ParticleSet is empty on writing as array at time %g" % time) + else: + if deleted_only: + pset_towrite = pset + # == commented due to git rebase to master 27 02 2020 == # + # elif pset[0].dt > 0: + # pset_towrite = [p for p in pset if time - p.dt/2 <= p.time < time + p.dt and np.isfinite(p.id)] + # else: + # pset_towrite = [p for p in pset if time + p.dt < p.time <= time - p.dt/2 and np.isfinite(p.id)] + # else: + # pset_towrite = [p for p in pset if time - np.abs(p.dt/2) <= p.time < time + np.abs(p.dt) and np.isfinite(p.id)] + pset_towrite = [] + node = pset.begin() + while node is not None: + if (time - np.abs(node.data.dt/2)) <= node.data.time < (time + np.abs(node.data.dt)) and np.isfinite(node.data.id): + pset_towrite.append(node.data) + node = node.next + if len(pset_towrite) > 0: + for var in self.var_names: + # if type(getattr(pset_towrite[0], var)) in [np.uint64, np.int64, np.uint32]: + # data_dict[var] = np.array([np.int32(getattr(p, var)) for p in pset_towrite]) + # else: + if 1: + data_dict[var] = np.array([getattr(p, var) for p in pset_towrite]) + # ---- this doesn't work anymore cause the number of particles cannot be inferred from the IDs anymore ---- # + # self.maxid_written = np.max([self.maxid_written, np.max(data_dict['id'])]) + # ==== does not work because the index changes depending on the MPI environment - it needs a running overview of how many particles there are ==== # + self.max_index_written = np.max([self.max_index_written, np.max(data_dict['index'])]) + + pset_errs = [p for p in pset_towrite if p.state != ErrorCode.Delete and abs(time-p.time) > 1e-3] + for p in pset_errs: + logger.warning_once('time argument in pfile.write() is %g, but a particle has time % g.' % (time, p.time)) + pset_towrite.clear() + + if time not in self.time_written: + self.time_written.append(time) + + if len(self.var_names_once) > 0: + # ------ ------ ------ ------ ------ ------ # + # first_write = [p for p in pset if (np.int32(p.id) not in self.written_once) and _is_particle_started_yet(p, time)] + # data_dict_once['id'] = np.array([np.int32(p.id) for p in first_write]) + # for var in self.var_names_once: + # data_dict_once[var] = np.array([getattr(p, var) for p in first_write]) + # self.written_once += data_dict_once['id'].tolist() + # ------ ------ ------ ------ ------ ------ # + first_write = [] + node = pset.begin() + while node is not None: + if (node.data.index not in self.written_once) and _is_particle_started_yet(node.data, time): + first_write.append(node.data) + node = node.next + data_dict_once['id'] = np.array([p.id for p in first_write]) + for var in self.var_names_once: + data_dict_once[var] = np.array([getattr(p, var) for p in first_write]) + self.written_once += data_dict_once['index'].tolist() + first_write.clear() + + if not deleted_only: + self.lasttime_written = time + + return data_dict, data_dict_once + + def convert_parray_to_dict(self, parray, time, deleted_only=False): + """Convert all Particle data from one time step to a python dictionary. + :param pset: ParticleSet object to write + :param time: Time at which to write ParticleSet + :param deleted_only: Flag to write only the deleted Particles + returns two dictionaries: one for all variables to be written each outputdt, + and one for all variables to be written once + """ + data_dict = {} + data_dict_once = {} + + time = time.total_seconds() if isinstance(time, delta) else time + + if self.lasttime_written != time and \ + (self.write_ondelete is False or deleted_only is True): + if len(parray) == 0: + logger.warning("ParticleSet is empty on writing as array at time %g" % time) + else: + pset_towrite = None + if deleted_only: + pset_towrite = [i for i in range(len(parray))] + else: + # == commented due to git rebase to master 27 02 2020 == # + # elif pset[0].dt > 0: + # pset_towrite = [p for p in pset if time - p.dt/2 <= p.time < time + p.dt and np.isfinite(p.id)] + # else: + # pset_towrite = [p for p in pset if time + p.dt < p.time <= time - p.dt/2 and np.isfinite(p.id)] + # else: + # pset_towrite = [p for p in pset if time - np.abs(p.dt/2) <= p.time < time + np.abs(p.dt) and np.isfinite(p.id)] + # --- memory-heavy action --- # + # pset_towrite = [p for p in parray if time - np.abs(p.dt/2) <= p.time < time + np.abs(p.dt) and np.isfinite(p.id)] + pset_towrite = [i for i, p in enumerate(parray) if time - np.abs(p.dt / 2) <= p.time < time + np.abs(p.dt) and np.isfinite(p.id)] + if len(pset_towrite) > 0: + for var in self.var_names: + # if type(getattr(pset_towrite[0], var)) in [np.uint64, np.int64, np.uint32]: + # data_dict[var] = np.array([np.int32(getattr(p, var)) for p in pset_towrite]) + # else: + # --- memory-heavy action --- # + # data_dict[var] = np.array([getattr(p, var) for p in pset_towrite]) + data_dict[var] = np.array([getattr(parray[i], var) for i in pset_towrite]) + # ---- this doesn't work anymore cause the number of particles cannot be inferred from the IDs anymore ---- # + # self.maxid_written = np.max([self.maxid_written, np.max(data_dict['id'])]) + # ==== does not work because the index changes depending on the MPI environment - it needs a running overview of how many particles there are ==== # + self.max_index_written = np.max([self.max_index_written, np.max(data_dict['index'])]) + + # pset_errs = [p for p in pset_towrite if p.state != ErrorCode.Delete and abs(time-p.time) > 1e-3] + pset_errs = [parray[i] for i in pset_towrite if parray[i].state != ErrorCode.Delete and abs(time - parray[i].time) > 1e-3] + for p in pset_errs: + logger.warning_once('time argument in pfile.write() is %g, but a particle has time % g.' % (time, p.time)) + # pset_towrite.clear() + del pset_towrite + + if time not in self.time_written: + self.time_written.append(time) + + if len(self.var_names_once) > 0: + # ------ ------ ------ ------ ------ ------ # + # first_write = [p for p in pset if (np.int32(p.id) not in self.written_once) and _is_particle_started_yet(p, time)] + # data_dict_once['id'] = np.array([np.int32(p.id) for p in first_write]) + # for var in self.var_names_once: + # data_dict_once[var] = np.array([getattr(p, var) for p in first_write]) + # self.written_once += data_dict_once['id'].tolist() + # ------ ------ ------ ------ ------ ------ # + # --- memory-heavy action --- # + # first_write = [p for p in parray if (p.index not in self.written_once) and _is_particle_started_yet(p, time)] + first_write = [i for i, p in enumerate(parray) if (p.index not in self.written_once) and _is_particle_started_yet(p, time)] + # data_dict_once['id'] = np.array([p.id for p in first_write]) + data_dict_once['id'] = np.array([parray[i].id for i in first_write]) + for var in self.var_names_once: + # data_dict_once[var] = np.array([getattr(p, var) for p in first_write]) + data_dict_once[var] = np.array([getattr(parray[i], var) for i in first_write]) + self.written_once += data_dict_once['index'].tolist() + # first_write.clear() + del first_write + + if not deleted_only: + self.lasttime_written = time + + return data_dict, data_dict_once + + def dump_dict_to_npy(self, data_dict, data_dict_once): + """Buffer data to set of temporary numpy files, using np.save""" + + if not os.path.exists(self.tempwritedir): + os.makedirs(self.tempwritedir) + + if len(data_dict) > 0: + tmpfilename = os.path.join(self.tempwritedir, str(len(self.file_list)) + ".npy") + with open(tmpfilename, 'wb') as f: + np.save(f, data_dict) + self.file_list.append(tmpfilename) + + if len(data_dict_once) > 0: + tmpfilename = os.path.join(self.tempwritedir, str(len(self.file_list_once)) + '_once.npy') + with open(tmpfilename, 'wb') as f: + np.save(f, data_dict_once) + self.file_list_once.append(tmpfilename) + + def dump_psetinfo_to_npy(self): + pset_info = {} + # attrs_to_dump = ['name', 'var_names', 'var_names_once', 'time_origin', 'lonlatdepth_dtype', + # 'file_list', 'file_list_once', 'maxid_written', 'time_written', 'parcels_mesh', + # 'metadata'] + attrs_to_dump = ['name', 'var_names', 'var_names_once', 'time_origin', 'lonlatdepth_dtype', + 'file_list', 'file_list_once', 'max_index_written', 'time_written', 'parcels_mesh', + 'metadata'] + for a in attrs_to_dump: + if hasattr(self, a): + pset_info[a] = getattr(self, a) + with open(os.path.join(self.tempwritedir, 'pset_info.npy'), 'wb') as f: + np.save(f, pset_info) + + def write(self, pset, time, deleted_only=False): + """Write all data from one time step to a temporary npy-file + using a python dictionary. The data is saved in the folder 'out'. + :param pset: ParticleSet object to write + :param time: Time at which to write ParticleSet + :param deleted_only: Flag to write only the deleted Particles + """ + if isinstance(pset, list) or isinstance(pset, np.ndarray): + self._write_by_array(pset, time, deleted_only) + else: + self._write_by_set_(pset, time, deleted_only) + + def _write_by_set_(self, pset, time, deleted_only=False): + """Write all data from one time step to a temporary npy-file + using a python dictionary. The data is saved in the folder 'out'. + :param pset: ParticleSet object to write + :param time: Time at which to write ParticleSet + :param deleted_only: Flag to write only the deleted Particles + """ + data_dict, data_dict_once = self.convert_pset_to_dict(pset, time, deleted_only=deleted_only) + self.dump_dict_to_npy(data_dict, data_dict_once) + self.dump_psetinfo_to_npy() + + def _write_by_array(self, parray, time, deleted_only=False): + """Write all data from one time step to a temporary npy-file + using a python dictionary. The data is saved in the folder 'out'. + :param pset: array (.i.e. Python list) of particles + :param time: Time at which to write the particles + :param deleted_only: Flag to write only the deleted Particles + """ + data_dict, data_dict_once = self.convert_parray_to_dict(parray, time, deleted_only=deleted_only) + self.dump_dict_to_npy(data_dict, data_dict_once) + self.dump_psetinfo_to_npy() + + def read_from_npy(self, file_list, time_steps, var): + """Read NPY-files for one variable using a loop over all files. + :param file_list: List that contains all file names in the output directory + :param time_steps: Number of time steps that were written in out directory + :param var: name of the variable to read + """ + + # data = np.nan * np.zeros((self.maxid_written+1, time_steps)) + # time_index = np.zeros(self.maxid_written+1, dtype=int) + + data = np.nan * np.zeros((self.max_index_written+1, time_steps)) + time_index = np.zeros(self.max_index_written+1, dtype=np.int32) + t_ind_used = np.zeros(time_steps, dtype=int) + + # loop over all files + for npyfile in file_list: + try: + data_dict = np.load(npyfile, allow_pickle=True).item() + except NameError: + raise RuntimeError('Cannot combine npy files into netcdf file because your ParticleFile is ' + 'still open on interpreter shutdown.\nYou can use ' + '"parcels_convert_npydir_to_netcdf %s" to convert these to ' + 'a NetCDF file yourself.\nTo avoid this error, make sure you ' + 'close() your ParticleFile at the end of your script.' % self.tempwritedir) + # id_ind = np.array(data_dict["id"], dtype=np.int32) + # ------ ------ ------ ------ ------ ------ + # id_ind = np.array(data_dict["id"], dtype=np.uint64) + id_ind = np.array(data_dict['index']) + # ------ ------ ------ ------ ------ ------ + t_ind = time_index[id_ind] if 'once' not in file_list[0] else 0 + t_ind_used[t_ind] = 1 + data[id_ind, t_ind] = data_dict[var] + time_index[id_ind] = time_index[id_ind] + 1 + + # remove rows and columns that are completely filled with nan values + tmp = data[time_index > 0, :] + return tmp[:, t_ind_used == 1] + + def export(self): + """Exports outputs in temporary NPY-files to NetCDF file""" + + if MPI: + # The export can only start when all threads are done. + MPI.COMM_WORLD.Barrier() + if MPI.COMM_WORLD.Get_rank() > 0: + return # export only on threat 0 + + # Retrieve all temporary writing directories and sort them in numerical order + temp_names = sorted(glob(os.path.join("%s" % self.tempwritedir_base, "*")), + key=lambda x: int(os.path.basename(x))) + + if len(temp_names) == 0: + raise RuntimeError("No npy files found in %s" % self.tempwritedir_base) + + global_max_index_written = -1 + # global_maxid_written = -1 + global_time_written = [] + global_file_list = [] + if len(self.var_names_once) > 0: + global_file_list_once = [] + for tempwritedir in temp_names: + if os.path.exists(tempwritedir): + pset_info_local = np.load(os.path.join(tempwritedir, 'pset_info.npy'), allow_pickle=True).item() + # global_maxid_written = np.max([global_maxid_written, pset_info_local['maxid_written']]) + global_max_index_written = np.max([global_max_index_written, pset_info_local['max_index_written']]) + global_time_written += pset_info_local['time_written'] + global_file_list += pset_info_local['file_list'] + if len(self.var_names_once) > 0: + global_file_list_once += pset_info_local['file_list_once'] + # self.maxid_written = global_maxid_written + self.max_index_written = global_max_index_written + self.time_written = np.unique(global_time_written) + + for var in self.var_names: + data = self.read_from_npy(global_file_list, len(self.time_written), var) + if var == self.var_names[0]: + self.open_netcdf_file(data.shape) + varout = 'z' if var == 'depth' else var + getattr(self, varout)[:, :] = data + # idx_data = self.read_from_npy(global_file_list, len(self.time_written), 'index') + # getattr(self, 'index')[:, :] = idx_data + + if len(self.var_names_once) > 0: + for var in self.var_names_once: + getattr(self, var)[:] = self.read_from_npy(global_file_list_once, 1, var) + + self.dataset.close() + + def delete_tempwritedir(self, tempwritedir=None): + """Deleted all temporary npy files + :param tempwritedir Optional path of the directory to delete + """ + if tempwritedir is None: + tempwritedir = self.tempwritedir + if os.path.exists(tempwritedir): + shutil.rmtree(tempwritedir) diff --git a/parcels/particlefile.py b/parcels/particlefile_vectorized.py similarity index 81% rename from parcels/particlefile.py rename to parcels/particlefile_vectorized.py index 9485cee97c..186cdc293f 100644 --- a/parcels/particlefile.py +++ b/parcels/particlefile_vectorized.py @@ -45,6 +45,12 @@ def _set_calendar(origin_calendar): else: return origin_calendar +# ==== Comment: the way this class is written is really unsustainable / non-maintainable. This is because it holds +# ==== vectorized attribute copies of the particle set als well as their mapped memory areas in the NetCDF file, +# ==== but NONE of the attributes are visible in the constructor. Basically: one wanted to subclass the ParticleSet +# ==== and the NetCDF.Dataset, but instead mapped their attributes manually by lazy binding into this class. +# ==== This obviously circumvents any variable checks and is also prone to leave garbage memory open during the computation. + class ParticleFile(object): """Initialise trajectory output. @@ -95,6 +101,7 @@ def __init__(self, name, particleset, outputdt=np.infty, write_ondelete=False, c self.file_list = [] self.time_written = [] self.maxid_written = -1 + # self.max_index_written = -1 if tempwritedir is None: tempwritedir = os.path.join(os.path.dirname(str(self.name)), "out-%s" @@ -135,11 +142,14 @@ def open_netcdf_file(self, data_shape): self.dataset.parcels_mesh = self.parcels_mesh # Create ID variable according to CF conventions - self.id = self.dataset.createVariable("trajectory", "i4", coords, - fill_value=-2**(31)) # maxint32 fill_value + self.id = self.dataset.createVariable("trajectory", "i4", coords, fill_value=-2**(31)) # maxint32 fill_value + # self.id = self.dataset.createVariable("trajectory", "u8", coords, fill_value=(2**(64))-1) self.id.long_name = "Unique identifier for each particle" self.id.cf_role = "trajectory_id" + # self.index = self.dataset.createVariable("index_map", "i4", coords, fill_value=-2**(31)) + # self.index.long_name = "running (zero-based continuous) indexing element referring to the LOCAL index of a particle within one timestep" + # Create time, lat, lon and z variables according to CF conventions: self.time = self.dataset.createVariable("time", "f8", coords, fill_value=np.nan) self.time.long_name = "" @@ -176,6 +186,8 @@ def open_netcdf_file(self, data_shape): for vname in self.var_names: if vname not in ['time', 'lat', 'lon', 'depth', 'id']: + # ==== Comment - hm, that looks a bit 'crooked' - certainly from getting the ptype + # ==== you can parse the variables' bytesize - or make a "numpy-size to netcdf-size' conversion table here ... setattr(self, vname, self.dataset.createVariable(vname, "f4", coords, fill_value=np.nan)) getattr(self, vname).long_name = "" getattr(self, vname).standard_name = vname @@ -183,6 +195,8 @@ def open_netcdf_file(self, data_shape): for vname in self.var_names_once: setattr(self, vname, self.dataset.createVariable(vname, "f4", "traj", fill_value=np.nan)) + # ==== Comment - hm, that looks a bit 'crooked' - certainly from getting the ptype + # ==== you can parse the variables' bytesize - or make a "numpy-size to netcdf-size' conversion table here ... getattr(self, vname).long_name = "" getattr(self, vname).standard_name = vname getattr(self, vname).units = "unknown" @@ -232,6 +246,7 @@ def convert_pset_to_dict(self, pset, time, deleted_only=False): if pset.size == 0: logger.warning("ParticleSet is empty on writing as array at time %g" % time) else: + # self.max_index_written = -1 if deleted_only: pset_towrite = pset # == commented due to git rebase to master 27 02 2020 == # @@ -243,8 +258,15 @@ def convert_pset_to_dict(self, pset, time, deleted_only=False): pset_towrite = [p for p in pset if time - np.abs(p.dt/2) <= p.time < time + np.abs(p.dt) and np.isfinite(p.id)] if len(pset_towrite) > 0: for var in self.var_names: - data_dict[var] = np.array([getattr(p, var) for p in pset_towrite]) + if type(getattr(pset_towrite[0], var)) in [np.uint64, np.int64, np.uint32]: + data_dict[var] = np.array([np.int32(getattr(p, var)) for p in pset_towrite]) + else: + data_dict[var] = np.array([getattr(p, var) for p in pset_towrite]) + # ---- this doesn't work anymore cause the number of particles cannot be inferred from the IDs anymore ---- # self.maxid_written = np.max([self.maxid_written, np.max(data_dict['id'])]) + # ==== does not work because the index changes depending on the MPI environment - it needs a running overview of how many particles there are ==== # + # data_dict['index'] = np.array([i for i, p in enumerate(pset) if p.id in data_dict['id']], dtype=int) # self.max_index_written + + # self.max_index_written = np.max([self.max_index_written, np.max(data_dict['index'])]) pset_errs = [p for p in pset_towrite if p.state != ErrorCode.Delete and abs(time-p.time) > 1e-3] for p in pset_errs: @@ -255,11 +277,12 @@ def convert_pset_to_dict(self, pset, time, deleted_only=False): self.time_written.append(time) if len(self.var_names_once) > 0: - first_write = [p for p in pset if (p.id not in self.written_once) and _is_particle_started_yet(p, time)] - data_dict_once['id'] = np.array([p.id for p in first_write]) + first_write = [p for p in pset if (np.int32(p.id) not in self.written_once) and _is_particle_started_yet(p, time)] + data_dict_once['id'] = np.array([np.int32(p.id) for p in first_write]) + # data_dict_once['index'] = np.array([i for i, p in enumerate(pset) if p.id in data_dict_once['id']], dtype=int) for var in self.var_names_once: data_dict_once[var] = np.array([getattr(p, var) for p in first_write]) - self.written_once += [p.id for p in first_write] + self.written_once += data_dict_once['id'].tolist() if not deleted_only: self.lasttime_written = time @@ -279,7 +302,7 @@ def dump_dict_to_npy(self, data_dict, data_dict_once): self.file_list.append(tmpfilename) if len(data_dict_once) > 0: - tmpfilename = os.path.join(self.tempwritedir, str(len(self.file_list)) + '_once.npy') + tmpfilename = os.path.join(self.tempwritedir, str(len(self.file_list_once)) + '_once.npy') with open(tmpfilename, 'wb') as f: np.save(f, data_dict_once) self.file_list_once.append(tmpfilename) @@ -289,6 +312,9 @@ def dump_psetinfo_to_npy(self): attrs_to_dump = ['name', 'var_names', 'var_names_once', 'time_origin', 'lonlatdepth_dtype', 'file_list', 'file_list_once', 'maxid_written', 'time_written', 'parcels_mesh', 'metadata'] + # attrs_to_dump = ['name', 'var_names', 'var_names_once', 'time_origin', 'lonlatdepth_dtype', + # 'file_list', 'file_list_once', 'max_index_written', 'time_written', 'parcels_mesh', + # 'metadata'] for a in attrs_to_dump: if hasattr(self, a): pset_info[a] = getattr(self, a) @@ -315,8 +341,11 @@ def read_from_npy(self, file_list, time_steps, var): """ data = np.nan * np.zeros((self.maxid_written+1, time_steps)) - time_index = np.zeros(self.maxid_written+1, dtype=int) - t_ind_used = np.zeros(time_steps, dtype=int) + time_index = np.zeros(self.maxid_written+1, dtype=np.int32) + + # data = np.nan * np.zeros((self.max_index_written+1, time_steps)) + # time_index = np.zeros(self.max_index_written+1, dtype=int) + t_ind_used = np.zeros(time_steps, dtype=np.int32) # loop over all files for npyfile in file_list: @@ -328,7 +357,9 @@ def read_from_npy(self, file_list, time_steps, var): '"parcels_convert_npydir_to_netcdf %s" to convert these to ' 'a NetCDF file yourself.\nTo avoid this error, make sure you ' 'close() your ParticleFile at the end of your script.' % self.tempwritedir) - id_ind = np.array(data_dict["id"], dtype=int) + id_ind = np.array(data_dict["id"], dtype=np.int32) + # id_ind = np.array(data_dict["id"], dtype=np.uint64) + # id_ind = np.array(data_dict['index']) t_ind = time_index[id_ind] if 'once' not in file_list[0] else 0 t_ind_used[t_ind] = 1 data[id_ind, t_ind] = data_dict[var] @@ -354,6 +385,7 @@ def export(self): if len(temp_names) == 0: raise RuntimeError("No npy files found in %s" % self.tempwritedir_base) + # global_max_index_written = -1 global_maxid_written = -1 global_time_written = [] global_file_list = [] @@ -363,11 +395,13 @@ def export(self): if os.path.exists(tempwritedir): pset_info_local = np.load(os.path.join(tempwritedir, 'pset_info.npy'), allow_pickle=True).item() global_maxid_written = np.max([global_maxid_written, pset_info_local['maxid_written']]) + # global_max_index_written = np.max([global_max_index_written, pset_info_local['max_index_written']]) global_time_written += pset_info_local['time_written'] global_file_list += pset_info_local['file_list'] if len(self.var_names_once) > 0: global_file_list_once += pset_info_local['file_list_once'] self.maxid_written = global_maxid_written + # self.max_index_written = global_max_index_written self.time_written = np.unique(global_time_written) for var in self.var_names: @@ -376,6 +410,8 @@ def export(self): self.open_netcdf_file(data.shape) varout = 'z' if var == 'depth' else var getattr(self, varout)[:, :] = data + # idx_data = self.read_from_npy(global_file_list, len(self.time_written), 'index') + # getattr(self, 'index')[:, :] = idx_data if len(self.var_names_once) > 0: for var in self.var_names_once: diff --git a/parcels/particleset_node.py b/parcels/particleset_node.py new file mode 100644 index 0000000000..908e067f8f --- /dev/null +++ b/parcels/particleset_node.py @@ -0,0 +1,1132 @@ +import time as time_module +from datetime import date +from datetime import datetime as dtime +from datetime import timedelta as delta + +import os +import numpy as np +from scipy.spatial import distance +import itertools +import xarray as xr +import progressbar +# import math # noga +# import random # noga + +from parcels.nodes.LinkedList import * +from parcels.nodes.Node import Node, NodeJIT +from parcels.tools import idgen + +from parcels.tools import get_cache_dir, get_package_dir +# from parcels.tools import cleanup_remove_files, cleanup_unload_lib, get_cache_dir, get_package_dir +# from parcels.wrapping.code_compiler import GNUCompiler +# from parcels.wrapping.code_compiler import GNUCompiler_MS +from parcels import ScipyParticle, JITParticle +from parcels.particlefile_node import ParticleFile +# from parcels import Grid, Field, GridSet, FieldSet +from parcels.grid import GridCode +from parcels.field import NestedField +from parcels.field import SummedField +from parcels.kernelbase import BaseKernel +from parcels.kernel_node import Kernel +from parcels import ErrorCode +from parcels.kernels.advection import AdvectionRK4 +from parcels.tools.loggers import logger + + +try: + from mpi4py import MPI +except: + MPI = None +if MPI: + try: + from sklearn.cluster import KMeans + except: + raise EnvironmentError('sklearn needs to be available if MPI is installed. ' + 'See http://oceanparcels.org/#parallel_install for more information') + +__all__ = ['ParticleSet', 'RepeatParameters'] + + +class RepeatParameters(object): + _n_pts = 0 + _lon = [] + _lat = [] + _depth = [] + _maxID = None + _pclass = ScipyParticle + _partitions = None + kwargs = None + + def __init__(self, pclass=JITParticle, lon=None, lat=None, depth=None, partitions=None, pid_orig=None, **kwargs): + if lon is None: + lon = [] + self._lon = lon + if lat is None: + lat = [] + self._lat = lat + if depth is None: + depth = [] + self._depth = depth + self._maxID = pid_orig # pid - pclass.lastID + # assert type(self._lon)==type(self._lat)==type(self._depth) + if isinstance(self._lon, list): + self._n_pts = len(self._lon) + elif isinstance(self._lon, np.ndarray): + self._n_pts = self._lon.shape[0] + self._pclass = pclass + self._partitions = partitions + self.kwargs = kwargs + + @property + def num_pts(self): + return self._n_pts + + @property + def lon(self): + return self._lon + + def get_longitude(self, index): + return self._lon[index] + + @property + def lat(self): + return self._lat + + def get_latitude(self, index): + return self._lat[index] + + @property + def depth(self): + return self._depth + + def get_depth_value(self, index): + if len(self._depth) == 0: + return 0 + return self._depth[index] + + @property + def maxID(self): + return self._maxID + + def get_particle_id(self, index): + if self._maxID is None: + return None + return self._maxID+index + + @property + def pclass(self): + return self._pclass + + @property + def partitions(self): + return self._partitions + + +class ParticleSet(object): + _nodes = None + _pclass = ScipyParticle + _nclass = Node + _kclass = BaseKernel + _ptype = None + _fieldset = None + _kernel = None + _pu_centers = None + _lonlatdepth_dtype = None + + @staticmethod + def _convert_to_array_(var): + # Convert lists and single integers/floats to one-dimensional numpy arrays + if isinstance(var, np.ndarray): + return var.flatten() + elif isinstance(var, (int, float, np.float32, np.int32)): + return np.array([var]) + else: + return np.array(var) + + @staticmethod + def lonlatdepth_dtype_from_field_interp_method(field): + if type(field) in [SummedField, NestedField]: + for f in field: + if f.interp_method == 'cgrid_velocity': + return np.float64 + else: + if field.interp_method == 'cgrid_velocity': + return np.float64 + return np.float32 + + def __init__(self, fieldset=None, pclass=JITParticle, lon=None, lat=None, depth=None, time=None, + repeatdt=None, lonlatdepth_dtype=None, pid_orig=None, **kwargs): + + self._fieldset = fieldset + if self._fieldset is not None: + self._fieldset.check_complete() + + if lonlatdepth_dtype is not None: + self._lonlatdepth_dtype = lonlatdepth_dtype + else: + self._lonlatdepth_dtype = self.lonlatdepth_dtype_from_field_interp_method(self._fieldset.U) + assert self._lonlatdepth_dtype in [np.float32, np.float64], \ + 'lon lat depth precision should be set to either np.float32 or np.float64' + JITParticle.set_lonlatdepth_dtype(self._lonlatdepth_dtype) + # pid = None if pid_orig is None else pid_orig if isinstance(pid_orig, list) or isinstance(pid_orig, np.ndarray) else pid_orig + pclass.lastID + pid = None if pid_orig is None else pid_orig if isinstance(pid_orig, list) or isinstance(pid_orig, np.ndarray) else pid_orig + idgen.total_length + + self._pclass = pclass + self._kclass = Kernel + self._kernel = None + self._ptype = self._pclass.getPType() + self._pu_centers = None # can be given by parameter + if self._ptype.uses_jit: + self._nclass = NodeJIT + else: + self._nclass = Node + self._nodes = RealList(dtype=self._nclass) + + # ---- init common parameters to ParticleSets ---- # + lon = np.empty(shape=0) if lon is None else self._convert_to_array_(lon) + lat = np.empty(shape=0) if lat is None else self._convert_to_array_(lat) + # ==== pid is determined from the ID generator itself, not user-generated data, so to guarantee ID uniqueness ==== # + # if pid_orig is None: + # pid_orig = np.arange(lon.size) + # pid = pid_orig + pclass.lastID + + if depth is None: + mindepth, _ = self.fieldset.gridset.dimrange('depth') + depth = np.ones(lon.size, dtype=self._lonlatdepth_dtype) * mindepth + else: + depth = self._convert_to_array_(depth) + assert lon.size == lat.size and lon.size == depth.size, ( + 'lon, lat, depth don''t all have the same lenghts') + + time = self._convert_to_array_(time) + time = np.repeat(time, lon.size) if time.size == 1 else time + if time.size > 0 and type(time[0]) in [dtime, date]: + time = np.array([np.datetime64(t) for t in time]) + self.time_origin = fieldset.time_origin + if time.size > 0 and isinstance(time[0], np.timedelta64) and not self.time_origin: + raise NotImplementedError('If fieldset.time_origin is not a date, time of a particle must be a double') + time = np.array([self.time_origin.reltime(t) if isinstance(t, np.datetime64) else t for t in time]) + assert lon.size == time.size, ( + 'time and positions (lon, lat, depth) don''t have the same lengths.') + + # ---- init MPI functionality ---- # + _partitions = kwargs.pop('partitions', None) + if _partitions is not None and _partitions is not False: + _partitions = self._convert_to_array_(_partitions) + + offset = np.max(pid) if (pid is not None) and (len(pid) > 0) else -1 + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + mpi_size = mpi_comm.Get_size() + + if lon.size < mpi_size and mpi_size > 1: + raise RuntimeError('Cannot initialise with fewer particles than MPI processors') + + if mpi_size > 1: + if _partitions is not False: + if _partitions is None or self._pu_centers is None: + _partitions = None + _pu_centers = None + if mpi_rank == 0: + coords = np.vstack((lon, lat)).transpose() + kmeans = KMeans(n_clusters=mpi_size, random_state=0).fit(coords) + _partitions = kmeans.labels_ + _pu_centers = kmeans.cluster_centers_ + #mpi_comm.Barrier() + _partitions = mpi_comm.bcast(_partitions, root=0) + _pu_centers = mpi_comm.bcast(_pu_centers, root=0) + self._pu_centers = _pu_centers + # elif np.max(_partitions) >= mpi_size or self._pu_centers.shape[0] >= mpi_size: + elif np.max(_partitions >= mpi_rank) or self._pu_centers.shape[0] >= mpi_size: + raise RuntimeError('Particle partitions must vary between 0 and the number of mpi procs') + lon = lon[_partitions == mpi_rank] + lat = lat[_partitions == mpi_rank] + time = time[_partitions == mpi_rank] + depth = depth[_partitions == mpi_rank] + if pid is not None and (isinstance(pid, list) or isinstance(pid, np.ndarray)): + pid = pid[_partitions == mpi_rank] + for kwvar in kwargs: + kwargs[kwvar] = kwargs[kwvar][_partitions == mpi_rank] + offset = mpi_comm.allreduce(offset, op=MPI.MAX) + pclass.setLastID(offset+1) + + # ---- particle data parameter length assertions ---- # + for kwvar in kwargs: + kwargs[kwvar] = self._convert_to_array_(kwargs[kwvar]) + assert lon.size == kwargs[kwvar].size, ( + '%s and positions (lon, lat, depth) don''t have the same lengths.' % kwargs[kwvar]) + + self.repeatdt = repeatdt.total_seconds() if isinstance(repeatdt, delta) else repeatdt + rdata_available = True + rdata_available &= (lon is not None) and (isinstance(lon, list) or isinstance(lon, np.ndarray)) + rdata_available &= (lat is not None) and (isinstance(lat, list) or isinstance(lat, np.ndarray)) + rdata_available &= (depth is not None) and (isinstance(depth, list) or isinstance(depth, np.ndarray)) + rdata_available &= (time is not None) and (isinstance(time, list) or isinstance(time, np.ndarray)) + if self.repeatdt and rdata_available: + self.repeat_starttime = self._fieldset.gridset.dimrange('full_time')[0] if time is None else time[0] + self.rparam = RepeatParameters(self._pclass, lon, lat, depth, None, + None if pid is None else (pid - pclass.lastID), **kwargs) + + # fill / initialize / populate the list + if lon is not None and lat is not None: + for i in range(lon.size): + pdata_id = None + index = -1 + if pid is not None and (isinstance(pid, list) or isinstance(pid, np.ndarray)): + index = pid[i] + pdata_id = pid[i] + else: + index = idgen.total_length + pdata_id = idgen.nextID(lon[i], lat[i], depth[i], abs(time[i])) + pdata = self._pclass(lon[i], lat[i], pid=pdata_id, fieldset=self._fieldset, depth=depth[i], time=time[i], index=index) + # Set other Variables if provided + for kwvar in kwargs: + if not hasattr(pdata, kwvar): + raise RuntimeError('Particle class does not have Variable %s' % kwvar) + setattr(pdata, kwvar, kwargs[kwvar][i]) + ndata = self._nclass(id=pdata_id, data=pdata) + self._nodes.add(ndata) + + @classmethod + def from_list(cls, fieldset, pclass, lon, lat, depth=None, time=None, repeatdt=None, lonlatdepth_dtype=None, **kwargs): + """Initialise the ParticleSet from lists of lon and lat + + :param fieldset: :mod:`parcels.fieldset.FieldSet` object from which to sample velocity + :param pclass: mod:`parcels.particle.JITParticle` or :mod:`parcels.particle.ScipyParticle` + object that defines custom particle + :param lon: List of initial longitude values for particles + :param lat: List of initial latitude values for particles + :param depth: Optional list of initial depth values for particles. Default is 0m + :param time: Optional list of start time values for particles. Default is fieldset.U.time[0] + :param repeatdt: Optional interval (in seconds) on which to repeat the release of the ParticleSet + :param lonlatdepth_dtype: Floating precision for lon, lat, depth particle coordinates. + It is either np.float32 or np.float64. Default is np.float32 if fieldset.U.interp_method is 'linear' + and np.float64 if the interpolation method is 'cgrid_velocity' + Other Variables can be initialised using further arguments (e.g. v=... for a Variable named 'v') + """ + return cls(fieldset=fieldset, pclass=pclass, lon=lon, lat=lat, depth=depth, time=time, repeatdt=repeatdt, lonlatdepth_dtype=lonlatdepth_dtype, **kwargs) + + @classmethod + def from_line(cls, fieldset, pclass, start, finish, size, depth=None, time=None, repeatdt=None, lonlatdepth_dtype=None): + """Initialise the ParticleSet from start/finish coordinates with equidistant spacing + Note that this method uses simple numpy.linspace calls and does not take into account + great circles, so may not be a exact on a globe + + :param fieldset: :mod:`parcels.fieldset.FieldSet` object from which to sample velocity + :param pclass: mod:`parcels.particle.JITParticle` or :mod:`parcels.particle.ScipyParticle` + object that defines custom particle + :param start: Starting point for initialisation of particles on a straight line. + :param finish: End point for initialisation of particles on a straight line. + :param size: Initial size of particle set + :param depth: Optional list of initial depth values for particles. Default is 0m + :param time: Optional start time value for particles. Default is fieldset.U.time[0] + :param repeatdt: Optional interval (in seconds) on which to repeat the release of the ParticleSet + :param lonlatdepth_dtype: Floating precision for lon, lat, depth particle coordinates. + It is either np.float32 or np.float64. Default is np.float32 if fieldset.U.interp_method is 'linear' + and np.float64 if the interpolation method is 'cgrid_velocity' + """ + + lon = np.linspace(start[0], finish[0], size) + lat = np.linspace(start[1], finish[1], size) + if type(depth) in [int, float]: + depth = [depth] * size + return cls(fieldset=fieldset, pclass=pclass, lon=lon, lat=lat, depth=depth, time=time, repeatdt=repeatdt, lonlatdepth_dtype=lonlatdepth_dtype) + + @classmethod + def from_field(cls, fieldset, pclass, start_field, size, mode='monte_carlo', depth=None, time=None, repeatdt=None, lonlatdepth_dtype=None): + """Initialise the ParticleSet randomly drawn according to distribution from a field + + :param fieldset: :mod:`parcels.fieldset.FieldSet` object from which to sample velocity + :param pclass: mod:`parcels.particle.JITParticle` or :mod:`parcels.particle.ScipyParticle` + object that defines custom particle + :param start_field: Field for initialising particles stochastically (horizontally) according to the presented density field. + :param size: Initial size of particle set + :param mode: Type of random sampling. Currently only 'monte_carlo' is implemented + :param depth: Optional list of initial depth values for particles. Default is 0m + :param time: Optional start time value for particles. Default is fieldset.U.time[0] + :param repeatdt: Optional interval (in seconds) on which to repeat the release of the ParticleSet + :param lonlatdepth_dtype: Floating precision for lon, lat, depth particle coordinates. + It is either np.float32 or np.float64. Default is np.float32 if fieldset.U.interp_method is 'linear' + and np.float64 if the interpolation method is 'cgrid_velocity' + """ + + if mode == 'monte_carlo': + data = start_field.data if isinstance(start_field.data, np.ndarray) else np.array(start_field.data) + if start_field.interp_method == 'cgrid_tracer': + p_interior = np.squeeze(data[0, 1:, 1:]) + else: # if A-grid + d = data + p_interior = (d[0, :-1, :-1] + d[0, 1:, :-1] + d[0, :-1, 1:] + d[0, 1:, 1:])/4. + p_interior = np.where(d[0, :-1, :-1] == 0, 0, p_interior) + p_interior = np.where(d[0, 1:, :-1] == 0, 0, p_interior) + p_interior = np.where(d[0, 1:, 1:] == 0, 0, p_interior) + p_interior = np.where(d[0, :-1, 1:] == 0, 0, p_interior) + p = np.reshape(p_interior, (1, p_interior.size)) + inds = np.random.choice(p_interior.size, size, replace=True, p=p[0] / np.sum(p)) + xsi = np.random.uniform(size=len(inds)) + eta = np.random.uniform(size=len(inds)) + j, i = np.unravel_index(inds, p_interior.shape) + grid = start_field.grid + if grid.gtype in [GridCode.RectilinearZGrid, GridCode.RectilinearSGrid]: + lon = grid.lon[i] + xsi * (grid.lon[i + 1] - grid.lon[i]) + lat = grid.lat[j] + eta * (grid.lat[j + 1] - grid.lat[j]) + else: + lons = np.array([grid.lon[j, i], grid.lon[j, i+1], grid.lon[j+1, i+1], grid.lon[j+1, i]]) + if grid.mesh == 'spherical': + lons[1:] = np.where(lons[1:] - lons[0] > 180, lons[1:]-360, lons[1:]) + lons[1:] = np.where(-lons[1:] + lons[0] > 180, lons[1:]+360, lons[1:]) + lon = (1-xsi)*(1-eta) * lons[0] +\ + xsi*(1-eta) * lons[1] +\ + xsi*eta * lons[2] +\ + (1-xsi)*eta * lons[3] + lat = (1-xsi)*(1-eta) * grid.lat[j, i] +\ + xsi*(1-eta) * grid.lat[j, i+1] +\ + xsi*eta * grid.lat[j+1, i+1] +\ + (1-xsi)*eta * grid.lat[j+1, i] + else: + raise NotImplementedError('Mode %s not implemented. Please use "monte carlo" algorithm instead.' % mode) + + return cls(fieldset=fieldset, pclass=pclass, lon=lon, lat=lat, depth=depth, time=time, lonlatdepth_dtype=lonlatdepth_dtype, repeatdt=repeatdt) + + @classmethod + def from_particlefile(cls, fieldset, pclass, filename, restart=True, repeatdt=None, lonlatdepth_dtype=None): + """Initialise the ParticleSet from a netcdf ParticleFile. + This creates a new ParticleSet based on the last locations and time of all particles + in the netcdf ParticleFile. Particle IDs are preserved if restart=True + + :param fieldset: :mod:`parcels.fieldset.FieldSet` object from which to sample velocity + :param pclass: mod:`parcels.particle.JITParticle` or :mod:`parcels.particle.ScipyParticle` + object that defines custom particle + :param filename: Name of the particlefile from which to read initial conditions + :param restart: Boolean to signal if pset is used for a restart (default is True). + In that case, Particle IDs are preserved. + :param repeatdt: Optional interval (in seconds) on which to repeat the release of the ParticleSet + :param lonlatdepth_dtype: Floating precision for lon, lat, depth particle coordinates. + It is either np.float32 or np.float64. Default is np.float32 if fieldset.U.interp_method is 'linear' + and np.float64 if the interpolation method is 'cgrid_velocity' + """ + + pfile = xr.open_dataset(str(filename), decode_cf=True) + + lon = np.ma.filled(pfile.variables['lon'][:, -1], np.nan) + lat = np.ma.filled(pfile.variables['lat'][:, -1], np.nan) + depth = np.ma.filled(pfile.variables['z'][:, -1], np.nan) + time = np.ma.filled(pfile.variables['time'][:, -1], np.nan) + pid = np.ma.filled(pfile.variables['trajectory'][:, -1], np.nan) + if isinstance(time[0], np.timedelta64): + time = np.array([t/np.timedelta64(1, 's') for t in time]) + + inds = np.where(np.isfinite(lon))[0] + lon = lon[inds] + lat = lat[inds] + depth = depth[inds] + time = time[inds] + pid = pid[inds] if restart else None + + return cls(fieldset=fieldset, pclass=pclass, lon=lon, lat=lat, depth=depth, time=time, + pid_orig=pid, lonlatdepth_dtype=lonlatdepth_dtype, repeatdt=repeatdt) + + def cptr(self, index): + if self._ptype.uses_jit: + node = self._nodes[index] + return node.data.get_cptr() + else: + return None + + def empty(self): + return self.size <= 0 + + def begin(self): + """ + Returns the begin of the linked particle list (like C++ STL begin() function) + :return: begin Node (Node whose prev element is None); returns None if ParticleSet is empty + """ + if not self.empty(): + node = self._nodes[0] + while node.prev is not None: + node = node.prev + return node + return None + + def end(self): + """ + Returns the end of the linked partile list. UNLIKE in C++ STL, it returns the last element (valid element), + not the element past the last element (invalid element). (see http://www.cplusplus.com/reference/list/list/end/) + :return: end Node (Node whose next element is None); returns None if ParticleSet is empty + """ + if not self.empty(): + node = self._nodes[self.size - 1] + while node.next is not None: + node = node.next + return node + return None + + # def set_kernel_class(self, kclass): + # self._kclass = kclass + + @property + def lonlatdepth_dtype(self): + return self._lonlatdepth_dtype + + @property + def ptype(self): + return self._ptype + + @property + def kernel_class(self): + return self._kclass + + @kernel_class.setter + def kernel_class(self, value): + self._kclass = value + + @property + def size(self): + return len(self._nodes) + + @property + def data(self): + return self._nodes + + @property + def particles(self): + return self._nodes + + @property + def particle_data(self): + return self._nodes + + @property + def fieldset(self): + return self._fieldset + + def __len__(self): + return len(self._nodes) + + def __repr__(self): + result = "\n" + node = self._nodes[0] + while node.prev is not None: + node = node.prev + while node.next is not None: + result += str(node) + "\n" + node = node.next + result += str(node) + "\n" + return result + # return "\n".join([str(p) for p in self]) + + def get_index(self, ndata): + return self._nodes.index(ndata) + + def get(self, index): + return self.get_by_index(index) + + def get_by_index(self, index): + return self.__getitem__(index) + + def get_by_id(self, id): + """ + divide-and-conquer search of SORTED list - needed because the node list internally + can only be scanned for (a) its list index (non-coherent) or (b) a node itself, but not for a specific + Node property alone. That is why using the 'bisect' module alone won't work. + :param id: search Node ID + :return: Node attached to ID - if node not in list: return None + """ + lower = 0 + upper = len(self._nodes) - 1 + pos = lower + int((upper - lower) / 2.0) + current_node = self._nodes[pos] + _found = False + _search_done = False + while current_node.id != id and not _search_done: + prev_upper = upper + prev_lower = lower + if id < current_node.id: + lower = lower + upper = pos - 1 + pos = lower + int((upper - lower) / 2.0) + else: + lower = pos + upper = upper + pos = lower + int((upper - lower) / 2.0) + 1 + if (prev_upper == upper and prev_lower == lower): + _search_done = True + current_node = self._nodes[pos] + if current_node.id == id: + _found = True + if _found: + return current_node + else: + return None + + def get_particle(self, index): + return self.get(index).data + + # def retrieve_item(self, key): + # return self.get(key) + + def __getitem__(self, key): + if key >= 0 and key < len(self._nodes): + return self._nodes[key] + return None + + def __setitem__(self, key, value): + """ + Sets the 'data' portion of the Node list. Replacing the Node itself is error-prone, + but it is possible to replace the data container (i.e. the particle data) or a specific + Node. + :param key: index (int; np.int32) or Node + :param value: particle data of the particle class + :return: modified Node + """ + try: + assert (isinstance(value, self._pclass)) + except AssertionError: + print("setting value not of type '{}'".format(str(self._pclass))) + exit() + if isinstance(key, int) or isinstance(key, np.int32): + search_node = self._nodes[key] + search_node.set_data(value) + elif isinstance(key, self._nclass): + assert (key in self._nodes) + key.set_data(value) + + def __iadd__(self, pdata): + self.add(pdata) + return self + + def add(self, pdata): + if pdata is None: + return + if isinstance(pdata, (list, np.ndarray, ParticleSet)): + self.add_entities(pdata) + else: + self.add_entity(pdata) + + def add_entities(self, data_array): + if len(data_array) <= 0: + return + if isinstance(data_array, list) or isinstance(data_array, tuple): + for item in data_array: + self.add_entity(item) + elif isinstance(data_array, ParticleSet): + for i in range(len(data_array)): + # self.add_entity(data_array[i]) + self.add_entity(data_array.pop(i)) + elif isinstance(data_array, np.ndarray): + if data_array.dtype == self._ptype: + for i in itertools.islice(itertools.count(), 0, data_array.shape[0]): # BAD MISTAKE - ALSO NEEDS TO BE CHECKED AND DISTRIBUTED! + # ndata = self._nclass(id=data_array[i].id, data=data_array[i]) + # self._nodes.add(ndata) + # results.append(self._nodes.bisect_right(ndata)) + pdata = data_array[i] + self.add_entity(pdata) + else: + # expect this to be a nD (2 <= n <= 5) array with [lon, lat, [depth, [time, [dt]]]] + pu_data = None + if MPI and MPI.COMM_WORLD.Get_size() > 1: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + spdata = data_array[:, 0:2] + dists = distance.cdist(spdata, self._pu_centers) + min_pu = np.argmax(dists, axis=1) + pu_data = data_array[ min_pu == mpi_rank ] + else: + pu_data = data_array + for i in itertools.islice(itertools.count(), 0, pu_data.shape[0]): + pdata = self._pclass(lon=pu_data[i, 0], lat=pu_data[i, 1], pid=np.iinfo(np.uint64).max, fieldset=self._fieldset) + if pu_data.shape[1]>2: + pdata.depth = pu_data[i, 2] + if pu_data.shape[1]>3: + pdata.time = pu_data[i, 3] + if pu_data.shape[1]>4: + pdata.dt = pu_data[i, 4] + self.add_entity(pdata, pu_checked=True) + else: + return + + def add_entity(self, pdata, pu_checked=False): + """ + Adds the new data in the list - position is auto-determined (because of sorted-list nature) + :param pdata: new Node or pdata + :return: index of inserted node + """ + # Comment: by current workflow, pset modification is only done on the front node, thus + # the distance determination and assigment is also done on the front node + _add_to_pu = True + # if MPI: + if MPI and MPI.COMM_WORLD.Get_size() > 1 and not pu_checked: + if self._pu_centers is not None and isinstance(self._pu_centers, np.ndarray): + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + mpi_size = mpi_comm.Get_size() + min_dist = np.finfo(self._lonlatdepth_dtype).max + min_pu = 0 + if mpi_size > 1 and mpi_rank == 0: + ppos = pdata + if isinstance(pdata, self._nclass): + ppos = pdata.data + spdata = np.array([ppos.lat, ppos.lon], dtype=self._lonlatdepth_dtype) + n_clusters = self._pu_centers.shape[0] + for i in range(n_clusters): + diff = self._pu_centers[i, :] - spdata + dist = np.dot(diff, diff) + if dist < min_dist: + min_dist = dist + min_pu = i + # NOW: move the related center by: (center-spdata) * 1/(cluster_size+1) + min_pu = mpi_comm.bcast(min_pu, root=0) + if mpi_rank == min_pu: + _add_to_pu = True + else: + _add_to_pu = False + # ==== old non-numpy code ==== # + # ppos = pdata + # if isinstance(pdata, self._nclass): + # ppos = pdata.data + # spdata = np.array([ppos.lat, ppos.lon], dtype=self._lonlatdepth_dtype) + # n_clusters = self._pu_centers.shape[0] + # for i in range(n_clusters): + # diff = self._pu_centers[i, :] - spdata + # dist = np.dot(diff, diff) + # if dist < min_dist: + # min_dist = dist + # min_pu = i + # if mpi_rank == min_pu: + # _add_to_pu = True + # else: + # _add_to_pu = False + # # TODO: still need to propagate the new information + if _add_to_pu: + index = -1 + pid = np.iinfo(np.uint64).max + if isinstance(pdata, self._nclass): + self._nodes.add(pdata) + index = self._nodes.bisect_right(pdata) + else: + if pdata.id == pid: + index = idgen.total_length + pid = idgen.nextID(pdata.lon, pdata.lat, pdata.depth, pdata.time) + pdata.id = pid + pdata.index = index + else: + pid = pdata.id + index = pdata.index + node = self._nclass(id=pid, data=pdata) + self._nodes.add(node) + index = self._nodes.bisect_right(node) + if index >= 0: + # return self._nodes[index] + return index + return None + + def __isub__(self, ndata): + self.remove(ndata) + return self + + def remove(self, ndata): + """ + Removes a specific Node from the list. The Node can either be given directly or determined via it's index + or it's data package (i.e. particle data). When using the index, note though that Nodes are shifting + (non-coherent indices), so the reliable method is to provide the Node to-be-removed directly + (similar to an iterator in C++). + :param ndata: Node object, Particle object or int index to the Node to-be-removed + """ + if ndata is None: + pass + elif isinstance(ndata, list) or isinstance(ndata, np.ndarray): + self.remove_entities(ndata) # remove multiple instances + elif isinstance(ndata, self._nclass): + self.remove_entity(ndata) + else: + pass + + def remove_entity(self, ndata): + if isinstance(ndata, int) or isinstance(ndata, np.int32): + del self._nodes[ndata] + # search_node = self._nodes[ndata] + # self._nodes.remove(search_node) + elif isinstance(ndata, self._nclass): + try: + # self._nodes.remove(ndata) + nindex = self.get_index(ndata) + del self._nodes[nindex] + except ValueError: + logger("Node {} not found.".format(ndata)) + elif isinstance(ndata, self._pclass): + try: + node = self.get_by_id(ndata.id) + nindex = self.get_index(node) + # self._nodes.remove(node) + del self._nodes[nindex] + except ValueError: + logger("Particle data {} not found.".format(ndata)) + + def remove_entities(self, ndata_array): + rm_list = ndata_array + if len(ndata_array) <= 0: + return + if isinstance(rm_list[0], int) or isinstance(rm_list[0], np.int32) or isinstance(rm_list[0], np.int64): + rm_list = [] + for index in ndata_array: + rm_list.append(self.get_by_index(index)) + for ndata in rm_list: + self.remove_entity(ndata) + + def merge(self, key1, key2): + # TODO + pass + + def split(self, key): + """ + splits a node, returning the result 2 new nodes + :param key: index (int; np.int32), Node + :return: 'node1, node2' or 'index1, index2' + """ + # TODO + + def pop(self, idx=-1, deepcopy_elem=False): + try: + return self._nodes.pop(idx, deepcopy_elem) + except IndexError: + return None + + def insert(self, node_or_pdata): + """ + Inserts new data in the list - position is auto-determined (semantically equal to 'add') + :param node_or_pdata: new Node or pdata + :return: index of inserted node + """ + return self.add(node_or_pdata) + + # ==== high-level functions to execute operations (Add, Delete, Merge, Split) requested by the ==== # + # ==== internal :variables Particle.state of each Node. ==== # + + def get_deleted_item_indices(self): + indices = [i for i, n in enumerate(self._nodes) if n.data.state == ErrorCode.Delete] + return indices + + def remove_deleted_items_by_indices(self, indices): + if len(indices) > 0: + indices.sort(reverse=True) + for index in indices: + del self._nodes[index] + + def remove_deleted_items(self): + node = self.begin() + while node is not None: + next_node = node.next + if node.data.state == ErrorCode.Delete: + self._nodes.remove(node) + node = next_node + + def execute(self, pyfunc=AdvectionRK4, endtime=None, runtime=None, dt=1., + moviedt=None, recovery=None, output_file=None, movie_background_field=None, + verbose_progress=None, postIterationCallbacks=None, callbackdt=None): + """Execute a given kernel function over the particle set for + multiple timesteps. Optionally also provide sub-timestepping + for particle output. + + :param pyfunc: Kernel function to execute. This can be the name of a + defined Python function or a :class:`parcels.kernel.Kernel` object. + Kernels can be concatenated using the + operator + :param endtime: End time for the timestepping loop. + It is either a datetime object or a positive double. + :param runtime: Length of the timestepping loop. Use instead of endtime. + It is either a timedelta object or a positive double. [DURATION] + :param dt: Timestep interval to be passed to the kernel. + It is either a timedelta object or a double. + Use a negative value for a backward-in-time simulation. + :param recovery: Dictionary with additional `:mod:parcels.tools.error` + recovery kernels to allow custom recovery behaviour in case of + kernel errors. + :param output_file: :mod:`parcels.particlefile.ParticleFile` object for particle output + :param verbose_progress: Boolean for providing a progress bar for the kernel execution loop. + :param postIterationCallbacks: (Optional) Array of functions that are to be called after each iteration (post-process, non-Kernel) + :param callbackdt: (Optional, in conjecture with 'postIterationCallbacks) timestep inverval to (latestly) interrupt the running kernel and invoke post-iteration callbacks from 'postIterationCallbacks' + """ + + # check if pyfunc has changed since last compile. If so, recompile + if self._kernel is None or (self._kernel.pyfunc is not pyfunc and self._kernel is not pyfunc): + # Generate and store Kernel + if isinstance(pyfunc, self._kclass): + self._kernel = pyfunc + else: + self._kernel = self.Kernel(pyfunc) + # Prepare JIT kernel execution + if self._ptype.uses_jit: + self._kernel.remove_lib() + cppargs = ['-DDOUBLE_COORD_VARIABLES'] if self.lonlatdepth_dtype == np.float64 else None + # self._kernel.compile(compiler=GNUCompiler(cppargs=cppargs)) + # self._kernel.compile(compiler=GNUCompiler(cppargs=cppargs, incdirs=[os.path.join(get_package_dir(), 'include'), os.path.join(get_package_dir(), 'nodes'), "."], libdirs=[".", get_cache_dir()], libs=["node"])) + self._kernel.compile(compiler=GNUCompiler_MS(cppargs=cppargs, incdirs=[os.path.join(get_package_dir(), 'include'), os.path.join(get_package_dir(), 'nodes'), "."], tmp_dir=get_cache_dir())) + self._kernel.load_lib() + + # Convert all time variables to seconds + if isinstance(endtime, delta): + raise RuntimeError('endtime must be either a datetime or a double') + if isinstance(endtime, dtime): + endtime = np.datetime64(endtime) + + if isinstance(endtime, np.datetime64): + if self.time_origin.calendar is None: + raise NotImplementedError('If fieldset.time_origin is not a date, execution endtime must be a double') + endtime = self.time_origin.reltime(endtime) + + if isinstance(runtime, delta): + runtime = runtime.total_seconds() + if isinstance(dt, delta): + dt = dt.total_seconds() + outputdt = output_file.outputdt if output_file else np.infty + if isinstance(outputdt, delta): + outputdt = outputdt.total_seconds() + + if isinstance(moviedt, delta): + moviedt = moviedt.total_seconds() + if isinstance(callbackdt, delta): + callbackdt = callbackdt.total_seconds() + + assert runtime is None or runtime >= 0, 'runtime must be positive' + assert outputdt is None or outputdt >= 0, 'outputdt must be positive' + assert moviedt is None or moviedt >= 0, 'moviedt must be positive' + + # ==== Set particle.time defaults based on sign of dt, if not set at ParticleSet construction => moved below (l. xyz) + # piter = 0 + # while piter < len(self._nodes): + # pdata = self._nodes[piter].data + # #node = self.begin() + # #while node is not None: + # # pdata = node.data + # if np.isnan(pdata.time): + # mintime, maxtime = self._fieldset.gridset.dimrange('time_full') + # pdata.time = mintime if dt >= 0 else maxtime + # # node.set_data(pdata) + # self._nodes[piter].set_data(pdata) + # piter += 1 + + # Derive _starttime and endtime from arguments or fieldset defaults + if runtime is not None and endtime is not None: + raise RuntimeError('Only one of (endtime, runtime) can be specified') + + mintime, maxtime = self._fieldset.gridset.dimrange('time_full') + _starttime = min([n.data.time for n in self._nodes if not np.isnan(n.data.time)] + [mintime, ]) if dt >= 0 else max([n.data.time for n in self._nodes if not np.isnan(n.data.time)] + [maxtime, ]) + if self.repeatdt is not None and self.repeat_starttime is None: + self.repeat_starttime = _starttime + if runtime is not None: + endtime = _starttime + runtime * np.sign(dt) + elif endtime is None: + endtime = maxtime if dt >= 0 else mintime + + # print("Fieldset min-max: {} to {}".format(mintime, maxtime)) + # print("starttime={} to endtime={} (runtime={})".format(_starttime, endtime, runtime)) + + execute_once = False + if abs(endtime-_starttime) < 1e-5 or dt == 0 or runtime == 0: + dt = 0 + runtime = 0 + endtime = _starttime + + logger.warning_once("dt or runtime are zero, or endtime is equal to Particle.time. " + "The kernels will be executed once, without incrementing time") + execute_once = True + + # ==== Initialise particle timestepping + # for p in self: + # p.dt = dt + piter = 0 + while piter < len(self._nodes): + pdata = self._nodes[piter].data + pdata.dt = dt + if np.isnan(pdata.time): + pdata.time = _starttime + self._nodes[piter].set_data(pdata) + piter += 1 + + # First write output_file, because particles could have been added + if output_file is not None: + output_file.write(self, _starttime) + + if moviedt: + self.show(field=movie_background_field, show_time=_starttime, animation=True) + if moviedt is None: + moviedt = np.infty + if callbackdt is None: + interupt_dts = [np.infty, moviedt, outputdt] + if self.repeatdt is not None: + interupt_dts.append(self.repeatdt) + callbackdt = np.min(np.array(interupt_dts)) + + time = _starttime + if self.repeatdt and self.rparam is not None: + next_prelease = self.repeat_starttime + (abs(time - self.repeat_starttime) // self.repeatdt + 1) * self.repeatdt * np.sign(dt) + else: + next_prelease = np.infty if dt > 0 else - np.infty + next_output = time + outputdt if dt > 0 else time - outputdt + + next_movie = time + moviedt if dt > 0 else time - moviedt + next_callback = time + callbackdt if dt > 0 else time - callbackdt + + next_input = self._fieldset.computeTimeChunk(time, np.sign(dt)) + + tol = 1e-12 + + if verbose_progress is None: + walltime_start = time_module.time() + if verbose_progress: + pbar = self._create_progressbar_(_starttime, endtime) + + while (time < endtime and dt > 0) or (time > endtime and dt < 0) or dt == 0: + + if verbose_progress is None and time_module.time() - walltime_start > 10: + # Showing progressbar if runtime > 10 seconds + if output_file: + logger.info('Temporary output files are stored in %s.' % output_file.tempwritedir_base) + logger.info('You can use "parcels_convert_npydir_to_netcdf %s" to convert these ' + 'to a NetCDF file during the run.' % output_file.tempwritedir_base) + pbar = self._create_progressbar_(_starttime, endtime) + verbose_progress = True + + if dt > 0: + time = min(next_prelease, next_input, next_output, next_movie, next_callback, endtime) + else: + time = max(next_prelease, next_input, next_output, next_movie, next_callback, endtime) + self._kernel.execute(self, endtime=time, dt=dt, recovery=recovery, output_file=output_file, execute_once=execute_once) + if abs(time-next_prelease) < tol: # if that is true, 'rparam' is + if self.rparam.get_particle_id(0) is None: + pdata = np.array(self.rparam.lon, dtype=self._lonlatdepth_dtype).reshape((self.rparam.num_pts, 1)) + pdata = np.concatenate((pdata, np.array(self.rparam.lat, dtype=self.lonlatdepth_dtype).reshape((self.rparam.num_pts, 1))), axis=1) + if len(self.rparam.depth) > 0: + pdata = np.concatenate((pdata, np.array(self.rparam.depth, dtype=self.lonlatdepth_dtype).reshape((self.rparam.num_pts, 1))), axis=1) + else: + pdata = np.concatenate((pdata, np.zeros(pdata.shape[0], dtype=self._lonlatdepth_dtype).reshape((self.rparam.num_pts, 1))), axis=1) + pdata = np.concatenate((pdata, (np.ones(pdata.shape[0], dtype=np.float64) * time).reshape((self.rparam.num_pts, 1))), axis=1) + pdata = np.concatenate((pdata, (np.ones(pdata.shape[0], dtype=np.float64) * dt).reshape((self.rparam.num_pts, 1))), axis=1) + self.add(pdata) + else: + add_iter = 0 + while add_iter < self.rparam.num_pts: + gen_id = self.rparam.get_particle_id(add_iter) + lon = self.rparam.get_longitude(add_iter) + lat = self.rparam.get_latitude(add_iter) + pdepth = self.rparam.get_depth_value(add_iter) + ptime = time + # pindex = idgen.total_length + pid = idgen.nextID(lon, lat, pdepth, ptime) if gen_id is None else gen_id + # pid = np.iinfo(np.uint64).max if gen_id is None else gen_id + # pdata = self._pclass(lon=lon, lat=lat, pid=pid, fieldset=self._fieldset, depth=pdepth, time=ptime, index=pindex) + pdata = self._pclass(lon=lon, lat=lat, pid=pid, fieldset=self._fieldset, depth=pdepth, time=ptime) + pdata.dt = dt + self.add(self._nclass(id=pid, data=pdata)) + add_iter += 1 + next_prelease += self.repeatdt * np.sign(dt) + if abs(time-next_output) < tol: + if output_file is not None: + output_file.write(self, time) + next_output += outputdt * np.sign(dt) + + if abs(time-next_movie) < tol: + self.show(field=movie_background_field, show_time=time, animation=True) + next_movie += moviedt * np.sign(dt) + # ==== insert post-process here to also allow for memory clean-up via external func ==== # + if abs(time-next_callback) < tol: + if postIterationCallbacks is not None: + for extFunc in postIterationCallbacks: + extFunc() + next_callback += callbackdt * np.sign(dt) + + if time != endtime: + next_input = self._fieldset.computeTimeChunk(time, dt) + if dt == 0: + break + + if verbose_progress: + pbar.update(abs(time - _starttime)) + + if output_file is not None: + output_file.write(self, time) + + if verbose_progress: + pbar.finish() + + def Kernel(self, pyfunc, c_include="", delete_cfiles=True): + """Wrapper method to convert a `pyfunc` into a :class:`parcels.kernel.Kernel` object + based on `fieldset` and `ptype` of the ParticleSet + :param delete_cfiles: Boolean whether to delete the C-files after compilation in JIT mode (default is True) + """ + return self._kclass(self._fieldset, self._ptype, pyfunc=pyfunc, c_include=c_include, delete_cfiles=delete_cfiles) + + def ParticleFile(self, *args, **kwargs): + """Wrapper method to initialise a :class:`parcels.particlefile.ParticleFile` + object from the ParticleSet""" + return ParticleFile(*args, particleset=self, **kwargs) + + def _create_progressbar_(self, starttime, endtime): + pbar = None + try: + pbar = progressbar.ProgressBar(max_value=abs(endtime - starttime)).start() + except: # for old versions of progressbar + try: + pbar = progressbar.ProgressBar(maxvalue=abs(endtime - starttime)).start() + except: # for even older OR newer versions + pbar = progressbar.ProgressBar(maxval=abs(endtime - starttime)).start() + return pbar + + def show(self, with_particles=True, show_time=None, field=None, domain=None, projection=None, + land=True, vmin=None, vmax=None, savefile=None, animation=False, **kwargs): + """Method to 'show' a Parcels ParticleSet + + :param with_particles: Boolean whether to show particles + :param show_time: Time at which to show the ParticleSet + :param field: Field to plot under particles (either None, a Field object, or 'vector') + :param domain: dictionary (with keys 'N', 'S', 'E', 'W') defining domain to show + :param projection: type of cartopy projection to use (default PlateCarree) + :param land: Boolean whether to show land. This is ignored for flat meshes + :param vmin: minimum colour scale (only in single-plot mode) + :param vmax: maximum colour scale (only in single-plot mode) + :param savefile: Name of a file to save the plot to + :param animation: Boolean whether result is a single plot, or an animation + """ + from parcels.plotting import plotparticles + plotparticles(particles=self, with_particles=with_particles, show_time=show_time, field=field, domain=domain, + projection=projection, land=land, vmin=vmin, vmax=vmax, savefile=savefile, animation=animation, **kwargs) + + def density(self, field=None, particle_val=None, relative=False, area_scale=False): + """Method to calculate the density of particles in a ParticleSet from their locations, + through a 2D histogram. + + :param field: Optional :mod:`parcels.field.Field` object to calculate the histogram + on. Default is `fieldset.U` + :param particle_val: Optional numpy-array of values to weigh each particle with, + or string name of particle variable to use weigh particles with. + Default is None, resulting in a value of 1 for each particle + :param relative: Boolean to control whether the density is scaled by the total + weight of all particles. Default is False + :param area_scale: Boolean to control whether the density is scaled by the area + (in m^2) of each grid cell. Default is False + """ + + field = field if field else self.fieldset.U + if isinstance(particle_val, str): + particle_val = [getattr(p, particle_val) for p in self.particles] + else: + particle_val = particle_val if particle_val else np.ones(len(self.particles)) + density = np.zeros((field.grid.lat.size, field.grid.lon.size), dtype=np.float32) + + for pi, p in enumerate(self.particles): + try: # breaks if either p.xi, p.yi, p.zi, p.ti do not exist (in scipy) or field not in fieldset + if p.ti[field.igrid] < 0: # xi, yi, zi, ti, not initialised + raise('error') + xi = p.xi[field.igrid] + yi = p.yi[field.igrid] + except: + _, _, _, xi, yi, _ = field.search_indices(p.lon, p.lat, p.depth, 0, 0, search2D=True) + density[yi, xi] += particle_val[pi] + + if relative: + density /= np.sum(particle_val) + + if area_scale: + density /= field.cell_areas() + + return density diff --git a/parcels/particleset_node_benchmark.py b/parcels/particleset_node_benchmark.py new file mode 100644 index 0000000000..8ac0c163b8 --- /dev/null +++ b/parcels/particleset_node_benchmark.py @@ -0,0 +1,530 @@ +import time as time_module +from datetime import datetime as dtime +from datetime import timedelta as delta +import psutil +import os +from platform import system as system_name +import matplotlib.pyplot as plt +import sys + + +import numpy as np + +try: + from mpi4py import MPI +except: + MPI = None + +# from parcels.compiler import GNUCompiler +from parcels.wrapping.code_compiler import GNUCompiler_MS +from parcels.particleset_node import ParticleSet +# from parcels.kernel_vectorized import Kernel +# from parcels.kernel_node_benchmark import Kernel_Benchmark +# from parcels.kernel_benchmark import Kernel_Benchmark +from parcels.kernels.advection import AdvectionRK4 +from parcels.particle import JITParticle +from parcels.tools.loggers import logger +from parcels.tools import get_cache_dir, get_package_dir +from parcels.tools import idgen +from parcels.kernel_node_benchmark import Kernel_Benchmark +# from parcels.kernel_node_benchmark import Kernel +from parcels.tools.performance_logger import TimingLog, ParamLogging, Asynchronous_ParamLogging + +from resource import getrusage, RUSAGE_SELF + +from resource import getrusage, RUSAGE_SELF + +__all__ = ['ParticleSet_Benchmark'] + +def measure_mem(): + process = psutil.Process(os.getpid()) + pmem = process.memory_info() + pmem_total = pmem.shared + pmem.text + pmem.data + pmem.lib + # print("psutil - res-set: {}; res-shr: {} res-text: {}, res-data: {}, res-lib: {}; res-total: {}".format(pmem.rss, pmem.shared, pmem.text, pmem.data, pmem.lib, pmem_total)) + return pmem_total + +def measure_mem_rss(): + process = psutil.Process(os.getpid()) + pmem = process.memory_info() + pmem_total = pmem.shared + pmem.text + pmem.data + pmem.lib + # print("psutil - res-set: {}; res-shr: {} res-text: {}, res-data: {}, res-lib: {}; res-total: {}".format(pmem.rss, pmem.shared, pmem.text, pmem.data, pmem.lib, pmem_total)) + return pmem.rss + +def measure_mem_usage(): + rsc = getrusage(RUSAGE_SELF) + print("RUSAGE - Max. RES set-size: {}; shr. mem size: {}; ushr. mem size: {}".format(rsc.ru_maxrss, rsc.ru_ixrss, rsc.ru_idrss)) + if system_name() == "Linux": + return rsc.ru_maxrss*1024 + return rsc.ru_maxrss + +USE_ASYNC_MEMLOG = False +USE_RUSE_SYNC_MEMLOG = False # can be faulty + +class ParticleSet_Benchmark(ParticleSet): + + def __init__(self, fieldset, pclass=JITParticle, lon=None, lat=None, depth=None, time=None, repeatdt=None, + lonlatdepth_dtype=None, pid_orig=None, **kwargs): + super(ParticleSet_Benchmark, self).__init__(fieldset, pclass, lon, lat, depth, time, repeatdt, lonlatdepth_dtype, pid_orig, **kwargs) + self.total_log = TimingLog() + self.compute_log = TimingLog() + self.io_log = TimingLog() + self.mem_io_log = TimingLog() + self.plot_log = TimingLog() + self.nparticle_log = ParamLogging() + self.mem_log = ParamLogging() + self.async_mem_log = Asynchronous_ParamLogging() + self.process = psutil.Process(os.getpid()) + + def set_async_memlog_interval(self, interval): + self.async_mem_log.measure_interval = interval + + def execute(self, pyfunc=AdvectionRK4, endtime=None, runtime=None, dt=1., + moviedt=None, recovery=None, output_file=None, movie_background_field=None, + verbose_progress=None, postIterationCallbacks=None, callbackdt=None): + """Execute a given kernel function over the particle set for + multiple timesteps. Optionally also provide sub-timestepping + for particle output. + + :param pyfunc: Kernel function to execute. This can be the name of a + defined Python function or a :class:`parcels.kernel.Kernel` object. + Kernels can be concatenated using the + operator + :param endtime: End time for the timestepping loop. + It is either a datetime object or a positive double. + :param runtime: Length of the timestepping loop. Use instead of endtime. + It is either a timedelta object or a positive double. + :param dt: Timestep interval to be passed to the kernel. + It is either a timedelta object or a double. + Use a negative value for a backward-in-time simulation. + :param moviedt: Interval for inner sub-timestepping (leap), which dictates + the update frequency of animation. + It is either a timedelta object or a positive double. + None value means no animation. + :param output_file: :mod:`parcels.particlefile.ParticleFile` object for particle output + :param recovery: Dictionary with additional `:mod:parcels.tools.error` + recovery kernels to allow custom recovery behaviour in case of + kernel errors. + :param movie_background_field: field plotted as background in the movie if moviedt is set. + 'vector' shows the velocity as a vector field. + :param verbose_progress: Boolean for providing a progress bar for the kernel execution loop. + :param postIterationCallbacks: (Optional) Array of functions that are to be called after each iteration (post-process, non-Kernel) + :param callbackdt: (Optional, in conjecture with 'postIterationCallbacks) timestep inverval to (latestly) interrupt the running kernel and invoke post-iteration callbacks from 'postIterationCallbacks' + """ + + # check if pyfunc has changed since last compile. If so, recompile + if self._kernel is None or (self._kernel.pyfunc is not pyfunc and self._kernel is not pyfunc): + # Generate and store Kernel + if isinstance(pyfunc, self._kclass): + self._kernel = pyfunc + else: + self._kernel = self.Kernel(pyfunc) + # Prepare JIT kernel execution + if self._ptype.uses_jit: + self._kernel.remove_lib() + cppargs = ['-DDOUBLE_COORD_VARIABLES'] if self.lonlatdepth_dtype == np.float64 else None + self._kernel.compile(compiler=GNUCompiler_MS(cppargs=cppargs, incdirs=[os.path.join(get_package_dir(), 'include'), os.path.join(get_package_dir(), 'nodes'), "."], tmp_dir=get_cache_dir())) + self._kernel.load_lib() + + # Convert all time variables to seconds + if isinstance(endtime, delta): + raise RuntimeError('endtime must be either a datetime or a double') + if isinstance(endtime, dtime): + endtime = np.datetime64(endtime) + + if isinstance(endtime, np.datetime64): + if self.time_origin.calendar is None: + raise NotImplementedError('If fieldset.time_origin is not a date, execution endtime must be a double') + endtime = self.time_origin.reltime(endtime) + + if isinstance(runtime, delta): + runtime = runtime.total_seconds() + if isinstance(dt, delta): + dt = dt.total_seconds() + outputdt = output_file.outputdt if output_file else np.infty + if isinstance(outputdt, delta): + outputdt = outputdt.total_seconds() + + if isinstance(moviedt, delta): + moviedt = moviedt.total_seconds() + if isinstance(callbackdt, delta): + callbackdt = callbackdt.total_seconds() + + assert runtime is None or runtime >= 0, 'runtime must be positive' + assert outputdt is None or outputdt >= 0, 'outputdt must be positive' + assert moviedt is None or moviedt >= 0, 'moviedt must be positive' + + # ==== Set particle.time defaults based on sign of dt, if not set at ParticleSet construction => moved below (l. xyz) + # piter = 0 + # while piter < len(self._nodes): + # pdata = self._nodes[piter].data + # #node = self.begin() + # #while node is not None: + # # pdata = node.data + # if np.isnan(pdata.time): + # mintime, maxtime = self._fieldset.gridset.dimrange('time_full') + # pdata.time = mintime if dt >= 0 else maxtime + # # node.set_data(pdata) + # self._nodes[piter].set_data(pdata) + # piter += 1 + + # Derive _starttime and endtime from arguments or fieldset defaults + if runtime is not None and endtime is not None: + raise RuntimeError('Only one of (endtime, runtime) can be specified') + + mintime, maxtime = self._fieldset.gridset.dimrange('time_full') + _starttime = min([n.data.time for n in self._nodes if not np.isnan(n.data.time)] + [mintime, ]) if dt >= 0 else max([n.data.time for n in self._nodes if not np.isnan(n.data.time)] + [maxtime, ]) + if self.repeatdt is not None and self.repeat_starttime is None: + self.repeat_starttime = _starttime + if runtime is not None: + endtime = _starttime + runtime * np.sign(dt) + elif endtime is None: + endtime = maxtime if dt >= 0 else mintime + + # print("Fieldset min-max: {} to {}".format(mintime, maxtime)) + # print("starttime={} to endtime={} (runtime={})".format(_starttime, endtime, runtime)) + + execute_once = False + if abs(endtime-_starttime) < 1e-5 or dt == 0 or runtime == 0: + dt = 0 + runtime = 0 + endtime = _starttime + + logger.warning_once("dt or runtime are zero, or endtime is equal to Particle.time. " + "The kernels will be executed once, without incrementing time") + execute_once = True + + # ==== Initialise particle timestepping + # for p in self: + # p.dt = dt + piter = 0 + while piter < len(self._nodes): + pdata = self._nodes[piter].data + pdata.dt = dt + if np.isnan(pdata.time): + pdata.time = _starttime + self._nodes[piter].set_data(pdata) + piter += 1 + + # First write output_file, because particles could have been added + if output_file is not None: + output_file.write(self, _starttime) + + if moviedt: + self.show(field=movie_background_field, show_time=_starttime, animation=True) + if moviedt is None: + moviedt = np.infty + if callbackdt is None: + interupt_dts = [np.infty, moviedt, outputdt] + if self.repeatdt is not None: + interupt_dts.append(self.repeatdt) + callbackdt = np.min(np.array(interupt_dts)) + + time = _starttime + if self.repeatdt and self.rparam is not None: + next_prelease = self.repeat_starttime + (abs(time - self.repeat_starttime) // self.repeatdt + 1) * self.repeatdt * np.sign(dt) + else: + next_prelease = np.infty if dt > 0 else - np.infty + next_output = time + outputdt if dt > 0 else time - outputdt + + next_movie = time + moviedt if dt > 0 else time - moviedt + next_callback = time + callbackdt if dt > 0 else time - callbackdt + + next_input = self._fieldset.computeTimeChunk(time, np.sign(dt)) + + tol = 1e-12 + walltime_start = None + if verbose_progress is None: + walltime_start = time_module.time() + pbar = None + if verbose_progress: + pbar = self._create_progressbar_(_starttime, endtime) + + mem_used_start = 0 + if USE_ASYNC_MEMLOG: + self.async_mem_log.measure_func = measure_mem + mem_used_start = measure_mem() + + while (time < endtime and dt > 0) or (time > endtime and dt < 0) or dt == 0: + self.total_log.start_timing() + if USE_ASYNC_MEMLOG: + self.async_mem_log.measure_start_value = mem_used_start + self.async_mem_log.start_partial_measurement() + + if verbose_progress is None and time_module.time() - walltime_start > 10: + # Showing progressbar if runtime > 10 seconds + if output_file: + logger.info('Temporary output files are stored in %s.' % output_file.tempwritedir_base) + logger.info('You can use "parcels_convert_npydir_to_netcdf %s" to convert these ' + 'to a NetCDF file during the run.' % output_file.tempwritedir_base) + pbar = self._create_progressbar_(_starttime, endtime) + verbose_progress = True + + if dt > 0: + time = min(next_prelease, next_input, next_output, next_movie, next_callback, endtime) + else: + time = max(next_prelease, next_input, next_output, next_movie, next_callback, endtime) + # ==== compute ==== # + if not isinstance(self._kernel, Kernel_Benchmark): + self.compute_log.start_timing() + self._kernel.execute(self, endtime=time, dt=dt, recovery=recovery, output_file=output_file, execute_once=execute_once) + if abs(time-next_prelease) < tol: + # creating new particles equals a memory-io operation + if not isinstance(self._kernel, Kernel_Benchmark): + self.compute_log.stop_timing() + self.compute_log.accumulate_timing() + + self.mem_io_log.start_timing() + if self.rparam.get_particle_id(0) is None: + pdata = np.array(self.rparam.lon, dtype=self._lonlatdepth_dtype).reshape((self.rparam.num_pts, 1)) + pdata = np.concatenate((pdata, np.array(self.rparam.lat, dtype=self.lonlatdepth_dtype).reshape((self.rparam.num_pts, 1))), axis=1) + if len(self.rparam.depth) > 0: + pdata = np.concatenate((pdata, np.array(self.rparam.depth, dtype=self.lonlatdepth_dtype).reshape((self.rparam.num_pts, 1))), axis=1) + else: + pdata = np.concatenate((pdata, np.zeros(pdata.shape[0], dtype=self._lonlatdepth_dtype).reshape((self.rparam.num_pts, 1))), axis=1) + pdata = np.concatenate((pdata, (np.ones(pdata.shape[0], dtype=np.float64) * time).reshape((self.rparam.num_pts, 1))), axis=1) + pdata = np.concatenate((pdata, (np.ones(pdata.shape[0], dtype=np.float64) * dt).reshape((self.rparam.num_pts, 1))), axis=1) + self.add(pdata) + else: + add_iter = 0 + while add_iter < self.rparam.num_pts: + gen_id = self.rparam.get_particle_id(add_iter) + lon = self.rparam.get_longitude(add_iter) + lat = self.rparam.get_latitude(add_iter) + pdepth = self.rparam.get_depth_value(add_iter) + ptime = time + # pindex = idgen.total_length + pid = idgen.nextID(lon, lat, pdepth, ptime) if gen_id is None else gen_id + # pid = np.iinfo(np.uint64).max if gen_id is None else gen_id + # pdata = self._pclass(lon=lon, lat=lat, pid=pid, fieldset=self._fieldset, depth=pdepth, time=ptime, index=pindex) + pdata = self._pclass(lon=lon, lat=lat, pid=pid, fieldset=self._fieldset, depth=pdepth, time=ptime) + pdata.dt = dt + self.add(self._nclass(id=pid, data=pdata)) + add_iter += 1 + self.mem_io_log.stop_timing() + self.mem_io_log.accumulate_timing() + + next_prelease += self.repeatdt * np.sign(dt) + else: + if not isinstance(self._kernel, Kernel_Benchmark): + self.compute_log.stop_timing() + else: + pass + if isinstance(self._kernel, Kernel_Benchmark): + self.compute_log.add_aux_measure(self._kernel.compute_timings.sum()) + self._kernel.compute_timings.reset() + self.io_log.add_aux_measure(self._kernel.io_timings.sum()) + self._kernel.io_timings.reset() + self.mem_io_log.add_aux_measure(self._kernel.mem_io_timings.sum()) + self._kernel.mem_io_timings.reset() + self.compute_log.accumulate_timing() + # logger.info("Pset length: {}".format(len(self))) + self.nparticle_log.advance_iteration(len(self)) + # ==== end compute ==== # + if abs(time-next_output) < tol: # ==== IO ==== # + if output_file is not None: + self.io_log.start_timing() + output_file.write(self, time) + self.io_log.stop_timing() + self.io_log.accumulate_timing() + next_output += outputdt * np.sign(dt) + if abs(time-next_movie) < tol: # ==== Plotting ==== # + self.plot_log.start_timing() + self.show(field=movie_background_field, show_time=time, animation=True) + self.plot_log.stop_timing() + self.plot_log.accumulate_timing() + next_movie += moviedt * np.sign(dt) + # ==== insert post-process here to also allow for memory clean-up via external func ==== # + if abs(time-next_callback) < tol: + # ==== assuming post-processing functions largely use memory than hard computation ... ==== # + self.mem_io_log.start_timing() + if postIterationCallbacks is not None: + for extFunc in postIterationCallbacks: + extFunc() + self.mem_io_log.stop_timing() + self.mem_io_log.accumulate_timing() + next_callback += callbackdt * np.sign(dt) + if time != endtime: # ==== IO ==== # + self.io_log.start_timing() + next_input = self.fieldset.computeTimeChunk(time, dt) + self.io_log.stop_timing() + self.io_log.accumulate_timing() + if dt == 0: + break + if verbose_progress: # ==== Plotting ==== # + self.plot_log.start_timing() + pbar.update(abs(time - _starttime)) + self.plot_log.stop_timing() + self.plot_log.accumulate_timing() + self.total_log.stop_timing() + self.total_log.accumulate_timing() + mem_B_used_total = 0 + if USE_RUSE_SYNC_MEMLOG: + mem_B_used_total = measure_mem_usage() + else: + mem_B_used_total = measure_mem_rss() + self.mem_log.advance_iteration(mem_B_used_total) + if USE_ASYNC_MEMLOG: + self.async_mem_log.stop_partial_measurement() # does 'advance_iteration' internally + + self.compute_log.advance_iteration() + self.io_log.advance_iteration() + self.mem_io_log.advance_iteration() + self.plot_log.advance_iteration() + self.total_log.advance_iteration() + + if output_file is not None: + self.io_log.start_timing() + output_file.write(self, time) + self.io_log.stop_timing() + self.io_log.accumulate_timing() + if verbose_progress: + self.plot_log.start_timing() + pbar.finish() + self.plot_log.stop_timing() + self.plot_log.accumulate_timing() + + # ==== Those lines include the timing for file I/O of the set. ==== # + # ==== Disabled as it doesn't have anything to do with advection. ==== # + # self.nparticle_log.advance_iteration(self.size) + # self.compute_log.advance_iteration() + # self.io_log.advance_iteration() + # self.mem_log.advance_iteration(self.process.memory_info().rss) + # self.mem_io_log.advance_iteration() + # self.plot_log.advance_iteration() + # self.total_log.advance_iteration() + + def Kernel(self, pyfunc, c_include="", delete_cfiles=True): + """Wrapper method to convert a `pyfunc` into a :class:`parcels.kernel_benchmark.Kernel` object + based on `fieldset` and `ptype` of the ParticleSet + :param delete_cfiles: Boolean whether to delete the C-files after compilation in JIT mode (default is True) + """ + return Kernel_Benchmark(self.fieldset, self.ptype, pyfunc=pyfunc, c_include=c_include, + delete_cfiles=delete_cfiles) + + def plot_and_log(self, total_times = None, compute_times = None, io_times = None, plot_times = None, memory_used = None, nparticles = None, target_N = 1, imageFilePath = "", odir = os.getcwd(), xlim_range=None, ylim_range=None): + # == do something with the log-arrays == # + if total_times is None or type(total_times) not in [list, dict, np.ndarray]: + total_times = self.total_log.get_values() + if not isinstance(total_times, np.ndarray): + total_times = np.array(total_times) + if compute_times is None or type(compute_times) not in [list, dict, np.ndarray]: + compute_times = self.compute_log.get_values() + if not isinstance(compute_times, np.ndarray): + compute_times = np.array(compute_times) + mem_io_times = None + if io_times is None or type(io_times) not in [list, dict, np.ndarray]: + io_times = self.io_log.get_values() + mem_io_times = self.mem_io_log.get_values() + if not isinstance(io_times, np.ndarray): + io_times = np.array(io_times) + if mem_io_times is not None: + mem_io_times = np.array(mem_io_times) + io_times += mem_io_times + if plot_times is None or type(plot_times) not in [list, dict, np.ndarray]: + plot_times = self.plot_log.get_values() + if not isinstance(plot_times, np.ndarray): + plot_times = np.array(plot_times) + if memory_used is None or type(memory_used) not in [list, dict, np.ndarray]: + memory_used = self.mem_log.get_params() + if not isinstance(memory_used, np.ndarray): + memory_used = np.array(memory_used) + if nparticles is None or type(nparticles) not in [list, dict, np.ndarray]: + nparticles = [] + if not isinstance(nparticles, np.ndarray): + nparticles = np.array(nparticles, dtype=np.int32) + + memory_used_async = None + if USE_ASYNC_MEMLOG: + memory_used_async = np.array(self.async_mem_log.get_params(), dtype=np.int64) + + t_scaler = 1. * 10./1.0 + npart_scaler = 1.0 / 1000.0 + mem_scaler = 1.0 / (1024 * 1024 * 1024) + plot_t = (total_times * t_scaler).tolist() + plot_ct = (compute_times * t_scaler).tolist() + plot_iot = (io_times * t_scaler).tolist() + plot_drawt = (plot_times * t_scaler).tolist() + plot_npart = (nparticles * npart_scaler).tolist() + plot_mem = [] + if memory_used is not None and len(memory_used) > 1: + plot_mem = (memory_used * mem_scaler).tolist() + + plot_mem_async = None + if USE_ASYNC_MEMLOG: + plot_mem_async = (memory_used_async * mem_scaler).tolist() + + do_iot_plot = True + do_drawt_plot = False + do_mem_plot = True + do_mem_plot_async = True + do_npart_plot = True + assert (len(plot_t) == len(plot_ct)) + if len(plot_t) != len(plot_iot): + print("plot_t and plot_iot have different lengths ({} vs {})".format(len(plot_t), len(plot_iot))) + do_iot_plot = False + if len(plot_t) != len(plot_drawt): + print("plot_t and plot_drawt have different lengths ({} vs {})".format(len(plot_t), len(plot_iot))) + do_drawt_plot = False + if len(plot_t) != len(plot_mem): + print("plot_t and plot_mem have different lengths ({} vs {})".format(len(plot_t), len(plot_mem))) + do_mem_plot = False + if len(plot_t) != len(plot_npart): + print("plot_t and plot_npart have different lengths ({} vs {})".format(len(plot_t), len(plot_npart))) + do_npart_plot = False + x = np.arange(start=0, stop=len(plot_t)) + + fig, ax = plt.subplots(1, 1, figsize=(21, 12)) + ax.plot(x, plot_t, 's-', label="total time_spent [100ms]") + ax.plot(x, plot_ct, 'o-', label="compute-time spent [100ms]") + if do_iot_plot: + ax.plot(x, plot_iot, 'o-', label="io-time spent [100ms]") + if do_drawt_plot: + ax.plot(x, plot_drawt, 'o-', label="draw-time spent [100ms]") + if (memory_used is not None) and do_mem_plot: + ax.plot(x, plot_mem, '--', label="memory_used (cumulative) [1 GB]") + if USE_ASYNC_MEMLOG: + if (memory_used_async is not None) and do_mem_plot_async: + ax.plot(x, plot_mem_async, ':', label="memory_used [async] (cum.) [1GB]") + if do_npart_plot: + ax.plot(x, plot_npart, '-', label="sim. particles [# 1000]") + if xlim_range is not None: + plt.xlim(list(xlim_range)) # [0, 730] + if ylim_range is not None: + plt.ylim(list(ylim_range)) # [0, 120] + plt.legend() + ax.set_xlabel('iteration') + plt.savefig(os.path.join(odir, imageFilePath), dpi=600, format='png') + + sys.stdout.write("cumulative total runtime: {}\n".format(total_times.sum())) + sys.stdout.write("cumulative compute time: {}\n".format(compute_times.sum())) + sys.stdout.write("cumulative I/O time: {}\n".format(io_times.sum())) + sys.stdout.write("cumulative plot time: {}\n".format(plot_times.sum())) + + csv_file = os.path.splitext(imageFilePath)[0]+".csv" + with open(os.path.join(odir, csv_file), 'w') as f: + nparticles_t0 = 0 + nparticles_tN = 0 + if nparticles is not None: + nparticles_t0 = nparticles[0] + nparticles_tN = nparticles[-1] + ncores = 1 + if MPI: + mpi_comm = MPI.COMM_WORLD + ncores = mpi_comm.Get_size() + header_string = "target_N, start_N, final_N, avg_N, ncores, avg_kt_total[s], avg_kt_compute[s], avg_kt_io[s], avg_kt_plot[s], cum_t_total[s], cum_t_compute[s], com_t_io[s], cum_t_plot[s], max_mem[MB]\n" + f.write(header_string) + data_string = "{}, {}, {}, {}, {}, ".format(target_N, nparticles_t0, nparticles_tN, nparticles.mean(), ncores) + data_string+= "{:2.10f}, {:2.10f}, {:2.10f}, {:2.10f}, ".format(total_times.mean(), compute_times.mean(), io_times.mean(), plot_times.mean()) + max_mem_sync = 0 + if memory_used is not None and len(memory_used) > 1: + memory_used = np.floor(memory_used / (1024*1024)) + memory_used = memory_used.astype(dtype=np.uint32) + max_mem_sync = memory_used.max() + max_mem_async = 0 + if USE_ASYNC_MEMLOG: + if memory_used_async is not None and len(memory_used_async) > 1: + memory_used_async = np.floor(memory_used_async / (1024*1024)) + memory_used_async = memory_used_async.astype(dtype=np.int64) + max_mem_async = memory_used_async.max() + max_mem = max(max_mem_sync, max_mem_async) + data_string += "{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {}".format(total_times.sum(), compute_times.sum(), io_times.sum(), plot_times.sum(), max_mem) + f.write(data_string) diff --git a/parcels/particleset.py b/parcels/particleset_vectorized.py similarity index 94% rename from parcels/particleset.py rename to parcels/particleset_vectorized.py index 419666c97e..f448c6f3c9 100644 --- a/parcels/particleset.py +++ b/parcels/particleset_vectorized.py @@ -1,22 +1,26 @@ import collections import time as time_module from datetime import date -from datetime import datetime +from datetime import datetime as dtime from datetime import timedelta as delta +import os import numpy as np import xarray as xr import progressbar -from parcels.compiler import GNUCompiler +# from parcels.tools import cleanup_remove_files, cleanup_unload_lib, get_cache_dir, get_package_dir +from parcels.tools import get_package_dir +# from parcels.compiler import GNUCompiler +from parcels.wrapping.code_compiler import GNUCompiler from parcels.field import NestedField from parcels.field import SummedField from parcels.grid import GridCode -from parcels.kernel import Kernel +from parcels.kernel_vectorized import Kernel from parcels.kernels.advection import AdvectionRK4 from parcels.particle import JITParticle -from parcels.particlefile import ParticleFile -from parcels.tools.error import ErrorCode +from parcels.particlefile_vectorized import ParticleFile +# from parcels.tools.error import ErrorCode from parcels.tools.loggers import logger try: from mpi4py import MPI @@ -52,23 +56,23 @@ class ParticleSet(object): are distributed automatically on the processors Other Variables can be initialised using further arguments (e.g. v=... for a Variable named 'v') """ + @staticmethod + def _convert_to_array_(var): + # Convert lists and single integers/floats to one-dimensional numpy arrays + if isinstance(var, np.ndarray): + return var.flatten() + elif isinstance(var, (int, float, np.float32, np.int32)): + return np.array([var]) + else: + return np.array(var) def __init__(self, fieldset, pclass=JITParticle, lon=None, lat=None, depth=None, time=None, repeatdt=None, lonlatdepth_dtype=None, pid_orig=None, **kwargs): self.fieldset = fieldset self.fieldset.check_complete() partitions = kwargs.pop('partitions', None) - def convert_to_array(var): - # Convert lists and single integers/floats to one-dimensional numpy arrays - if isinstance(var, np.ndarray): - return var.flatten() - elif isinstance(var, (int, float, np.float32, np.int32)): - return np.array([var]) - else: - return np.array(var) - - lon = np.empty(shape=0) if lon is None else convert_to_array(lon) - lat = np.empty(shape=0) if lat is None else convert_to_array(lat) + lon = np.empty(shape=0) if lon is None else self._convert_to_array_(lon) + lat = np.empty(shape=0) if lat is None else self._convert_to_array_(lat) if pid_orig is None: pid_orig = np.arange(lon.size) pid = pid_orig + pclass.lastID @@ -77,13 +81,13 @@ def convert_to_array(var): mindepth, _ = self.fieldset.gridset.dimrange('depth') depth = np.ones(lon.size) * mindepth else: - depth = convert_to_array(depth) + depth = self._convert_to_array_(depth) assert lon.size == lat.size and lon.size == depth.size, ( 'lon, lat, depth don''t all have the same lenghts') - time = convert_to_array(time) + time = self._convert_to_array_(time) time = np.repeat(time, lon.size) if time.size == 1 else time - if time.size > 0 and type(time[0]) in [datetime, date]: + if time.size > 0 and type(time[0]) in [dtime, date]: time = np.array([np.datetime64(t) for t in time]) self.time_origin = fieldset.time_origin if time.size > 0 and isinstance(time[0], np.timedelta64) and not self.time_origin: @@ -93,14 +97,17 @@ def convert_to_array(var): 'time and positions (lon, lat, depth) don''t have the same lengths.') if partitions is not None and partitions is not False: - partitions = convert_to_array(partitions) + partitions = self._convert_to_array_(partitions) for kwvar in kwargs: - kwargs[kwvar] = convert_to_array(kwargs[kwvar]) + kwargs[kwvar] = self._convert_to_array_(kwargs[kwvar]) assert lon.size == kwargs[kwvar].size, ( '%s and positions (lon, lat, depth) don''t have the same lengths.' % kwargs[kwvar]) - offset = np.max(pid) if len(pid) > 0 else -1 + # this is not all to clever because all particles of a particle set are only initialized ONCE, + # thus once determined on setup their MPI distribution stays fixed. That means that after several iterations + # particles within one cluster are not necessarily co-located as they moved ... + offset = np.max(pid) if (pid is not None) and (len(pid) > 0) else -1 if MPI: mpi_comm = MPI.COMM_WORLD mpi_rank = mpi_comm.Get_rank() @@ -184,6 +191,10 @@ def cptr(i): else: raise ValueError("Latitude and longitude required for generating ParticleSet") + @property + def particle_data(self): + return self._particle_data + @classmethod def from_list(cls, fieldset, pclass, lon, lat, depth=None, time=None, repeatdt=None, lonlatdepth_dtype=None, **kwargs): """Initialise the ParticleSet from lists of lon and lat @@ -309,7 +320,7 @@ def from_particlefile(cls, fieldset, pclass, filename, restart=True, repeatdt=No lat = np.ma.filled(pfile.variables['lat'][:, -1], np.nan) depth = np.ma.filled(pfile.variables['z'][:, -1], np.nan) time = np.ma.filled(pfile.variables['time'][:, -1], np.nan) - pid = np.ma.filled(pfile.variables['trajectory'][:, -1], np.nan) + pid = np.ma.filled(pfile.variables['trajectory'][:, -1], np.nan).astype(dtype=np.uint64) if isinstance(time[0], np.timedelta64): time = np.array([t/np.timedelta64(1, 's') for t in time]) @@ -373,10 +384,12 @@ def add(self, particles): raise NotImplementedError('Only ParticleSets can be added to a ParticleSet') self.particles = np.append(self.particles, particles) if self.ptype.uses_jit: + # particles_data = [p.get_cptr() for p in particles] particles_data = [p._cptr for p in particles] self._particle_data = np.append(self._particle_data, particles_data) # Update C-pointer on particles for p, pdata in zip(self.particles, self._particle_data): + # p.set_cptr(pdata) p._cptr = pdata def remove(self, indices): @@ -390,6 +403,7 @@ def remove(self, indices): self._particle_data = np.delete(self._particle_data, indices) # Update C-pointer on particles for p, pdata in zip(self.particles, self._particle_data): + # p.set_cptr(pdata) p._cptr = pdata return particles @@ -436,13 +450,13 @@ def execute(self, pyfunc=AdvectionRK4, endtime=None, runtime=None, dt=1., if self.ptype.uses_jit: self.kernel.remove_lib() cppargs = ['-DDOUBLE_COORD_VARIABLES'] if self.lonlatdepth_dtype == np.float64 else None - self.kernel.compile(compiler=GNUCompiler(cppargs=cppargs)) + self.kernel.compile(compiler=GNUCompiler(cppargs=cppargs, incdirs=[os.path.join(get_package_dir(), 'include'), "."])) self.kernel.load_lib() # Convert all time variables to seconds if isinstance(endtime, delta): raise RuntimeError('endtime must be either a datetime or a double') - if isinstance(endtime, datetime): + if isinstance(endtime, dtime): endtime = np.datetime64(endtime) if isinstance(endtime, np.datetime64): if self.time_origin.calendar is None: diff --git a/parcels/particleset_benchmark.py b/parcels/particleset_vectorized_benchmark.py similarity index 97% rename from parcels/particleset_benchmark.py rename to parcels/particleset_vectorized_benchmark.py index 5bc903acbf..0e0c030ef4 100644 --- a/parcels/particleset_benchmark.py +++ b/parcels/particleset_vectorized_benchmark.py @@ -15,12 +15,16 @@ except: MPI = None -from parcels.compiler import GNUCompiler +# from parcels.compiler import GNUCompiler +from parcels.wrapping.code_compiler import GNUCompiler +from parcels.particleset_vectorized import ParticleSet +# from parcels.kernel_vectorized import Kernel +from parcels.kernel_vec_benchmark import Kernel_Benchmark +# from parcels.kernel_benchmark import Kernel_Benchmark from parcels.kernels.advection import AdvectionRK4 -from parcels.particleset import ParticleSet from parcels.particle import JITParticle -from parcels.kernel_benchmark import Kernel_Benchmark, Kernel from parcels.tools.loggers import logger +from parcels.tools import get_cache_dir, get_package_dir from parcels.tools.performance_logger import TimingLog, ParamLogging, Asynchronous_ParamLogging from resource import getrusage, RUSAGE_SELF @@ -69,7 +73,6 @@ def __init__(self, fieldset, pclass=JITParticle, lon=None, lat=None, depth=None, def set_async_memlog_interval(self, interval): self.async_mem_log.measure_interval = interval - # @profile def execute(self, pyfunc=AdvectionRK4, endtime=None, runtime=None, dt=1., moviedt=None, recovery=None, output_file=None, movie_background_field=None, verbose_progress=None, postIterationCallbacks=None, callbackdt=None): @@ -105,7 +108,7 @@ def execute(self, pyfunc=AdvectionRK4, endtime=None, runtime=None, dt=1., # check if pyfunc has changed since last compile. If so, recompile if self.kernel is None or (self.kernel.pyfunc is not pyfunc and self.kernel is not pyfunc): # Generate and store Kernel - if isinstance(pyfunc, Kernel): + if isinstance(pyfunc, Kernel_Benchmark): self.kernel = pyfunc else: self.kernel = self.Kernel(pyfunc) @@ -113,7 +116,8 @@ def execute(self, pyfunc=AdvectionRK4, endtime=None, runtime=None, dt=1., if self.ptype.uses_jit: self.kernel.remove_lib() cppargs = ['-DDOUBLE_COORD_VARIABLES'] if self.lonlatdepth_dtype == np.float64 else None - self.kernel.compile(compiler=GNUCompiler(cppargs=cppargs)) + # self.kernel.compile(compiler=GNUCompiler(cppargs=cppargs)) + self.kernel.compile(compiler=GNUCompiler(cppargs=cppargs, incdirs=[os.path.join(get_package_dir(), 'include'), os.path.join(get_package_dir()), "."], tmp_dir=get_cache_dir())) self.kernel.load_lib() # Convert all time variables to seconds @@ -324,7 +328,7 @@ def execute(self, pyfunc=AdvectionRK4, endtime=None, runtime=None, dt=1., self.plot_log.advance_iteration() self.total_log.advance_iteration() - if output_file: + if output_file is not None: self.io_log.start_timing() output_file.write(self, time) self.io_log.stop_timing() diff --git a/parcels/rng.py b/parcels/rng.py index aac3a3851e..78aacd6ef4 100644 --- a/parcels/rng.py +++ b/parcels/rng.py @@ -1,15 +1,17 @@ import uuid +import _ctypes from ctypes import c_float from ctypes import c_int from os import path +from os import remove +from sys import platform import numpy.ctypeslib as npct -from parcels.compiler import get_cache_dir -from parcels.compiler import GNUCompiler +from parcels.tools import get_cache_dir, get_package_dir +from parcels.wrapping import GNUCompiler from parcels.tools.loggers import logger - __all__ = ['seed', 'random', 'uniform', 'randint', 'normalvariate', 'expovariate', 'vonmisesvariate'] @@ -50,24 +52,77 @@ class Random(object): return parcels_vonmisesvariate(mu, kappa); } """ - ccode = stmt_import + fnct_seed - ccode += fnct_random + fnct_uniform + fnct_randint + fnct_normalvariate + fnct_expovariate + fnct_vonmisesvariate - basename = path.join(get_cache_dir(), 'parcels_random_%s' % uuid.uuid4()) - src_file = "%s.c" % basename - lib_file = "%s.so" % basename - log_file = "%s.log" % basename + _lib = None + ccode = None + src_file = None + lib_file = None + log_file = None def __init__(self): self._lib = None - - @property - def lib(self, compiler=GNUCompiler()): + self.ccode = "" + self.ccode += self.stmt_import + self.ccode += self.fnct_seed + self.ccode += self.fnct_random + self.ccode += self.fnct_uniform + self.ccode += self.fnct_randint + self.ccode += self.fnct_normalvariate + self.ccode += self.fnct_expovariate + self.ccode += self.fnct_vonmisesvariate + self._loaded = False + self.compile() + self.load_lib() + + def __del__(self): + self.unload_lib() + self.remove_lib() + + def unload_lib(self): + # Unload the currently loaded dynamic linked library to be secure + if self._lib is not None and self._loaded: + _ctypes.FreeLibrary(self._lib._handle) if platform == 'win32' else _ctypes.dlclose(self._lib._handle) + del self._lib + self._lib = None + self._loaded = False + + def load_lib(self): + self._lib = npct.load_library(self.lib_file, '.') + self._loaded = True + + def remove_lib(self): + # If file already exists, pull new names. This is necessary on a Windows machine, because + # Python's ctype does not deal in any sort of manner well with dynamic linked libraries on this OS. + if path.isfile(self.lib_file): + [remove(s) for s in [self.src_file, self.lib_file, self.log_file]] + + def compile(self, compiler=None): + if self.src_file is None or self.lib_file is None or self.log_file is None: + basename = 'parcels_random_%s' % uuid.uuid4() + lib_filename = "lib" + basename + basepath = path.join(get_cache_dir(), "%s" % basename) + libpath = path.join(get_cache_dir(), "%s" % lib_filename) + # basename = path.join(get_cache_dir(), 'parcels_random_%s' % uuid.uuid4()) + self.src_file = "%s.c" % basepath + self.lib_file = "%s.so" % libpath + self.log_file = "%s.log" % basepath + ccompiler = compiler + if ccompiler is None: + cppargs = [] + incdirs = [path.join(get_package_dir(), 'include'), ] + ccompiler = GNUCompiler(cppargs=cppargs, incdirs=incdirs) if self._lib is None: - with open(self.src_file, 'w') as f: + with open(self.src_file, 'w+') as f: f.write(self.ccode) - compiler.compile(self.src_file, self.lib_file, self.log_file) + ccompiler.compile(self.src_file, self.lib_file, self.log_file) logger.info("Compiled %s ==> %s" % ("random", self.lib_file)) - self._lib = npct.load_library(self.lib_file, '.') + + @property + def lib(self): + if self.src_file is None or self.lib_file is None or self.log_file is None: + self.compile() + if self._lib is None or not self._loaded: + self.load_lib() + # self._lib = npct.load_library(self.lib_file, '.') return self._lib diff --git a/parcels/tools/__init__.py b/parcels/tools/__init__.py index 26f2f8f792..a5fad05496 100644 --- a/parcels/tools/__init__.py +++ b/parcels/tools/__init__.py @@ -3,4 +3,13 @@ from .interpolation_utils import * # noqa from .loggers import * # noqa from .timer import * # noqa -from .performance_logger import * # noga \ No newline at end of file +from .global_statics import * # noga +from .id_generators import * # noga +from .performance_logger import * # noga + +# global idgen +# idgen = SpatioTemporalIdGenerator() +idgen = GenerateID_Service(SpatioTemporalIdGenerator) +# idgen = GenerateID_Service(SequentialIdGenerator) +idgen.setDepthLimits(0, 100.0) +idgen.setTimeLine(0, 240.0) diff --git a/parcels/tools/global_statics.py b/parcels/tools/global_statics.py new file mode 100644 index 0000000000..96b5edd869 --- /dev/null +++ b/parcels/tools/global_statics.py @@ -0,0 +1,36 @@ +import os +import sys +import _ctypes +from tempfile import gettempdir +from pathlib import Path + +try: + from os import getuid +except: + # Windows does not have getuid(), so define to simply return 'tmp' + def getuid(): + return 'tmp' + + +def cleanup_remove_files(lib_file, log_file): + if os.path.isfile(lib_file): + [os.remove(s) for s in [lib_file, log_file]] + + +def cleanup_unload_lib(lib): + # Clean-up the in-memory dynamic linked libraries. + # This is not really necessary, as these programs are not that large, but with the new random + # naming scheme which is required on Windows OS'es to deal with updates to a Parcels' kernel. + if lib is not None: + _ctypes.FreeLibrary(lib._handle) if sys.platform == 'win32' else _ctypes.dlclose(lib._handle) + + +def get_package_dir(): + fpath = Path(__file__) + return fpath.parent.parent + + +def get_cache_dir(): + directory = os.path.join(gettempdir(), "parcels-%s" % getuid()) + Path(directory).mkdir(exist_ok=True) + return directory diff --git a/parcels/tools/id_generators.py b/parcels/tools/id_generators.py new file mode 100644 index 0000000000..ed65568dc2 --- /dev/null +++ b/parcels/tools/id_generators.py @@ -0,0 +1,685 @@ +import random # could be python's random if parcels not active; can be parcel's random; can be numpy's random +from abc import ABC, abstractmethod +# from numpy import random as nprandom +# from multiprocessing import Process +from threading import Thread +from .message_service import mpi_execute_requested_messages as executor +# from os import getpid +import numpy as np + +try: + from mpi4py import MPI +except: + MPI = None + + +class BaseIdGenerator(ABC): + _total_ids = 0 + _recover_ids = False + + def __init__(self): + self._total_ids = 0 + + def setTimeLine(self, min_time, max_time): + pass + + def setDepthLimits(self, min_depth, max_depth): + pass + + def preGenerateIDs(self, high_value): + pass + + def permuteIDs(self): + pass + + def close(self): + pass + + @abstractmethod + def __len__(self): + pass + + @property + def total_length(self): + return self._total_ids + + @property + def recover_ids(self): + return self._recover_ids + + @recover_ids.setter + def recover_ids(self, bool_param): + self._recover_ids = bool_param + + def enable_ID_recovery(self): + self._recover_ids = True + + def disable_ID_recovery(self): + self._recover_ids = False + + @abstractmethod + def getID(self, lon, lat, depth, time): + pass + + def nextID(self, lon, lat, depth, time): + return self.getID(lon, lat, depth, time) + + @abstractmethod + def releaseID(self, id): + pass + + @abstractmethod + def get_length(self): + return self.__len__() + + @abstractmethod + def get_total_length(self): + return self._total_ids + + +class SequentialIdGenerator(BaseIdGenerator): + released_ids = [] + next_id = 0 + + def __init__(self): + super(SequentialIdGenerator, self).__init__() + self.released_ids = [] + self.next_id = np.uint64(0) + self._recover_ids = False + + def __del__(self): + if len(self.released_ids) > 0: + del self.released_ids + + def getID(self, lon, lat, depth, time): + n = len(self.released_ids) + if n == 0: + result = self.next_id + self.next_id += 1 + self._total_ids += 1 + return np.uint64(result) + else: + result = self.released_ids.pop(n-1) + return np.uint64(result) + + def releaseID(self, id): + if not self._recover_ids: + return + self.released_ids.append(id) + + def preGenerateIDs(self, high_value): + if len(self.released_ids) > 0: + self.released_ids.clear() + self.released_ids = [i for i in range(0, high_value)] + self.next_id = high_value + + def permuteIDs(self): + n = len(self.released_ids) + indices = random.randint(0, n, 2*n) + for index in indices: + id = self.released_ids.pop(index) + self.released_ids.append(id) + + def __len__(self): + return self.next_id + + def get_length(self): + return self.__len__() + + def get_total_length(self): + return self._total_ids + + +class SpatioTemporalIdGenerator(BaseIdGenerator): + """Generates 64-bit IDs""" + timebounds = np.zeros(2, dtype=np.float64) + depthbounds = np.zeros(2, dtype=np.float32) + local_ids = None + released_ids = {} + _total_ids = 0 + + def __init__(self): + super(SpatioTemporalIdGenerator, self).__init__() + self.timebounds = np.zeros(2, dtype=np.float64) + self.depthbounds = np.zeros(2, dtype=np.float32) + self.local_ids = np.zeros((360, 180, 128, 256), dtype=np.uint32) + self.released_ids = {} # 32-bit spatio-temporal index => [] + self._total_ids = 0 + self._recover_ids = False + + def __del__(self): + # occupancy_file = path.join( get_cache_dir(), 'id_occupancy.npy') + # idreleases_file = path.join( get_cache_dir(), 'id_releases.pkl' ) + # if path.exists(occupancy_file): + # remove(occupancy_file) + # if path.exists(idreleases_file): + # remove(idreleases_file) + if self.local_ids is not None: + del self.local_ids + if len(self.released_ids) > 0: + del self.released_ids + + def setTimeLine(self, min_time=0.0, max_time=1.0): + self.timebounds = np.array([min_time, max_time], dtype=np.float64) + + def setDepthLimits(self, min_depth=0.0, max_depth=1.0): + self.depthbounds = np.array([min_depth, max_depth], dtype=np.float32) + + def getID(self, lon, lat, depth, time): + if lon < -180.0 or lon > 180.0: + vsgn = np.sign(lon) + lon = np.fmod(np.fabs(lon), 180.0) * vsgn + if lat < -90.0 or lat > 90.0: + vsgn = np.sign(lat) + lat = np.fmod(np.fabs(lat), 90.0) * vsgn + if depth is None: + depth = self.depthbounds[0] + if time is None: + time = self.timebounds[0] + lon_discrete = np.int32(lon) + lat_discrete = np.int32(lat) + depth_discrete = (depth-self.depthbounds[0])/(self.depthbounds[1]-self.depthbounds[0]) + depth_discrete = np.int32(127.0 * depth_discrete) + time_discrete = (time-self.timebounds[0])/(self.timebounds[1]-self.timebounds[0]) + time_discrete = np.int32(255.0 * time_discrete) + lon_index = np.uint32(np.int32(lon_discrete)+180) + lat_index = np.uint32(np.int32(lat_discrete)+90) + depth_index = np.uint32(np.int32(depth_discrete)) + time_index = np.uint32(np.int32(time_discrete)) + # if MPI: + # id = self._distribute_next_id_by_file(lon_index, lat_index, depth_index, time_index) + # else: + # id = self._get_next_id(lon_index, lat_index, depth_index, time_index) + id = self._get_next_id(lon_index, lat_index, depth_index, time_index) + return id + + def nextID(self, lon, lat, depth, time): + return self.getID(lon, lat, depth, time) + + def releaseID(self, id): + full_bits = np.uint32(4294967295) + nil_bits = np.int32(0) + spatiotemporal_id = np.bitwise_and(np.bitwise_or(np.left_shift(np.int64(full_bits), 32), np.int64(nil_bits)), np.int64(id)) + spatiotemporal_id = np.uint32(np.right_shift(spatiotemporal_id, 32)) + local_id = np.bitwise_and(np.bitwise_or(np.left_shift(np.int64(nil_bits), 32), np.int64(full_bits)), np.int64(id)) + local_id = np.uint32(local_id) + # if MPI: + # self._gather_released_ids_by_file(spatiotemporal_id, local_id) + # else: + # self._release_id(spatiotemporal_id, local_id) + self._release_id(spatiotemporal_id, local_id) + + def __len__(self): + # if MPI: + # # return self._length_() + # return np.sum(self.local_ids) + sum([len(entity) for entity in self.released_ids]) + # else: + # return np.sum(self.local_ids) + sum([len(entity) for entity in self.released_ids]) + return np.sum(self.local_ids) + sum([len(entity) for entity in self.released_ids]) + + @property + def total_length(self): + return self._total_ids + + def get_length(self): + return self.__len__() + + def get_total_length(self): + # return self._total_ids + return self.total_length + + # def _distribute_next_id(self, lon_index, lat_index, depth_index, time_index): + # mpi_comm = MPI.COMM_WORLD + # mpi_rank = mpi_comm.Get_rank() + # mpi_size = mpi_comm.Get_size() + # snd_requested_id = np.zeros((mpi_size, 1), dtype=np.byte) + # rcv_requested_id = np.zeros((mpi_size, 1), dtype=np.byte) + # snd_requested_add = np.zeros((mpi_size, 4), dtype=np.uint32) + # rcv_requested_add = np.zeros((mpi_size, 4), dtype=np.uint32) + # return_id = np.zeros(mpi_size, dtype=np.uint64) + # return_id.fill(np.iinfo(np.uint64).max) + # snd_requested_id[mpi_rank] = 1 + # snd_requested_add[mpi_rank, :] = np.array([lon_index, lat_index, depth_index, time_index], dtype=np.uint32) + # mpi_comm.Reduce(snd_requested_id, rcv_requested_id, op=MPI.MAX, root=0) + # # rcv_requested_id = mpi_comm.reduce(snd_requested_id, op=MPI.MAX, root=0) + # mpi_comm.Reduce(snd_requested_add, rcv_requested_add, op=MPI.MAX, root=0) + # if mpi_rank == 0: + # for i in range(mpi_size): + # if rcv_requested_id[i] > 0: + # return_id[i] = self._get_next_id(rcv_requested_add[i, 0], rcv_requested_add[i, 1], rcv_requested_add[i, 2], rcv_requested_add[i, 3]) + # # mpi_comm.Bcast(return_id, root=0) + # return_id = mpi_comm.bcast(return_id, root=0) + # return return_id[mpi_rank] + + # def _distribute_next_id_by_file(self, lon_index, lat_index, depth_index, time_index): + # # mpi_comm = MPI.COMM_WORLD + # # mpi_rank = mpi_comm.Get_rank() + # # mpi_size = mpi_comm.Get_size() + + # return_id = None + # access_flag_file = path.join( get_cache_dir(), 'id_access' ) + # occupancy_file = path.join( get_cache_dir(), 'id_occupancy.npy') + # idreleases_file = path.join( get_cache_dir(), 'id_releases.pkl' ) + # while path.exists(access_flag_file): + # sleep(0.1) + # with open(access_flag_file, 'wb') as f_access: + # f_access.write(bytearray([True,])) + # # self.local_ids = np.fromfile(occupancy_file, dtype=np.uint32) + # self.local_ids = np.load(occupancy_file) + # with open(idreleases_file, 'rb') as f_idrel: + # self.released_ids = pickle.load( f_idrel ) + # return_id = self._get_next_id(lon_index, lat_index, depth_index, time_index) + # with open(idreleases_file, 'wb') as f_idrel: + # pickle.dump(self.released_ids, f_idrel) + # # self.local_ids.tofile(occupancy_file) + # np.save(occupancy_file, self.local_ids) + # remove(access_flag_file) + # + # return return_id + + # def _gather_released_ids(self, spatiotemporal_id, local_id): + # mpi_comm = MPI.COMM_WORLD + # mpi_rank = mpi_comm.Get_rank() + # mpi_size = mpi_comm.Get_size() + # snd_release_id = np.zeros(mpi_size, dtype=np.byte) + # rcv_release_id = np.zeros(mpi_size, dtype=np.byte) + # snd_release_add = np.zeros((mpi_size, 2), dtype=np.uint32) + # rcv_release_add = np.zeros((mpi_size, 2), dtype=np.uint32) + # snd_release_id[mpi_rank] = 1 + # snd_release_add[mpi_rank, :] = np.array([spatiotemporal_id, local_id], dtype=np.uint32) + # mpi_comm.Reduce(snd_release_id, rcv_release_id, op=MPI.MAX, root=0) + # mpi_comm.Reduce(snd_release_add, rcv_release_add, op=MPI.MAX, root=0) + # if mpi_rank == 0: + # for i in range(mpi_size): + # if rcv_release_id[i] > 0: + # self._release_id(rcv_release_add[i, 0], rcv_release_add[i, 1]) + + # def _gather_released_ids_by_file(self, spatiotemporal_id, local_id): + # + # return_id = None + # access_flag_file = path.join( get_cache_dir(), 'id_access' ) + # occupancy_file = path.join( get_cache_dir(), 'id_occupancy.npy') + # idreleases_file = path.join( get_cache_dir(), 'id_releases.pkl' ) + # while path.exists(access_flag_file): + # sleep(0.1) + # with open(access_flag_file, 'wb') as f_access: + # f_access.write(bytearray([True,])) + # # self.local_ids = np.fromfile(occupancy_file, dtype=np.uint32) + # self.local_ids = np.load(occupancy_file) + # with open(idreleases_file, 'rb') as f_idrel: + # self.released_ids = pickle.load( f_idrel ) + # self._release_id(spatiotemporal_id, local_id) + # with open(idreleases_file, 'wb') as f_idrel: + # pickle.dump(self.released_ids, f_idrel) + # # self.local_ids.tofile(occupancy_file) + # np.save(occupancy_file, self.local_ids) + # remove(access_flag_file) + # + # return return_id + + def _get_next_id(self, lon_index, lat_index, depth_index, time_index): + local_index = -1 + id = np.left_shift(lon_index, 23) + np.left_shift(lat_index, 15) + np.left_shift(depth_index, 8) + time_index + # id = np.left_shift(lon_index, 23) + np.left_shift(lat_index, 15) + np.left_shift(depth_index, 8) + time + # == print("requtested indices: ({}, {}, {}, {})".format(lon_index, lat_index, depth_index, time_index)) == # + # == print("spatial id: {}".format(id)) == # + if len(self.released_ids) > 0 and (id in self.released_ids.keys()) and len(self.released_ids[id]) > 0: + # mlist = self.released_ids[id] + local_index = np.uint32(self.released_ids[id].pop()) + if len(self.released_ids[id]) <= 0: + del self.released_ids[id] + else: + local_index = self.local_ids[lon_index, lat_index, depth_index, time_index] + self.local_ids[lon_index, lat_index, depth_index, time_index] += 1 + id = np.int64(id) + id = np.bitwise_or(np.left_shift(id, 32), np.int64(local_index)) + id = np.uint64(id) + self._total_ids += 1 + return id + + def _release_id(self, spatiotemporal_id, local_id): + if not self._recover_ids: + return + if spatiotemporal_id not in self.released_ids.keys(): + self.released_ids[spatiotemporal_id] = [] + self.released_ids[spatiotemporal_id].append(local_id) + + +# ====== ====== ====== ====== # +# == Imports and codes for == # +# == asyncronous ID gen. == # +# == via Threads/Processes == # +# ====== ====== ====== ====== # +# #from multiprocessing import Process, Pipe +# from threading import Thread +# from multiprocessing import Process +# # from multiprocessing.connection import Connection +# from .message_service import mpi_execute_requested_messages as executor +# from sys import stdout +# from os import getpid +# from parcels.tools import logger +# +# class GenerateID_Service(object): +# _request_tag = 5 +# _response_tag = 6 +# +# def __init__(self, base_generator_obj): +# super(GenerateID_Service, self).__init__() +# self._service_process = None +# self._serverrank = 0 +# self._request_tag = 5 +# self._response_tag = 6 +# self._use_subprocess = True +# +# if MPI: +# mpi_comm = MPI.COMM_WORLD +# mpi_rank = mpi_comm.Get_rank() +# mpi_size = mpi_comm.Get_size() +# if mpi_size <= 1: +# self._use_subprocess = False +# else: +# self._serverrank = mpi_size-1 +# if mpi_rank == self._serverrank: +# self._service_process = Thread(target=executor, name="IdService", args=(base_generator_obj, self._request_tag, self._response_tag), daemon=True) +# self._service_process.start() +# self._subscribe_() +# else: +# self._use_subprocess = False +# +# if not self._use_subprocess: +# self._service_process = base_generator_obj() +# +# def __del__(self): +# self._abort_() +# +# def _subscribe_(self): +# if MPI and self._use_subprocess: +# mpi_comm = MPI.COMM_WORLD +# mpi_rank = mpi_comm.Get_rank() +# data_package = {} +# data_package["func_name"] = "thread_subscribe" +# data_package["args"] = 0 +# data_package["src_rank"] = mpi_rank +# mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) +# +# def _abort_(self): +# if MPI and self._use_subprocess: +# mpi_comm = MPI.COMM_WORLD +# mpi_rank = mpi_comm.Get_rank() +# data_package = {} +# data_package["func_name"] = "thread_abort" +# data_package["args"] = 0 +# data_package["src_rank"] = mpi_rank +# mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) +# +# def close(self): +# self._abort_() +# +# def setTimeLine(self, min_time=0.0, max_time=1.0): +# if MPI and self._use_subprocess: +# mpi_comm = MPI.COMM_WORLD +# mpi_rank = mpi_comm.Get_rank() +# if mpi_rank == 0: +# data_package = {} +# data_package["func_name"] = "setTimeLine" +# data_package["args"] = 2 +# data_package["argv"] = [min_time, max_time] +# data_package["src_rank"] = mpi_rank +# mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) +# else: +# self._service_process.setTimeLine(min_time, max_time) +# +# def setDepthLimits(self, min_depth=0.0, max_depth=1.0): +# if MPI and self._use_subprocess: +# mpi_comm = MPI.COMM_WORLD +# mpi_rank = mpi_comm.Get_rank() +# if mpi_rank == 0: +# data_package = {} +# data_package["func_name"] = "setDepthLimits" +# data_package["args"] = 2 +# data_package["argv"] = [min_depth, max_depth] +# data_package["src_rank"] = mpi_rank +# mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) +# else: +# self._service_process.setDepthLimits(min_depth, max_depth) +# +# def getID(self, lon, lat, depth, time): +# if MPI and self._use_subprocess: +# mpi_comm = MPI.COMM_WORLD +# mpi_rank = mpi_comm.Get_rank() +# +# data_package = {} +# data_package["func_name"] = "getID" +# data_package["args"] = 4 +# data_package["argv"] = [lon, lat, depth, time] +# data_package["src_rank"] = mpi_rank +# mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) +# data = mpi_comm.recv(source=self._serverrank, tag=self._response_tag) +# return int(data["result"]) +# else: +# return self._service_process.getID(lon, lat, depth, time) +# +# def nextID(self, lon, lat, depth, time): +# return self.getID(lon, lat, depth, time) +# +# def releaseID(self, id): +# if MPI and self._use_subprocess: +# mpi_comm = MPI.COMM_WORLD +# mpi_rank = mpi_comm.Get_rank() +# +# data_package = {} +# data_package["func_name"] = "releaseID" +# data_package["args"] = 1 +# data_package["argv"] = [id, ] +# data_package["src_rank"] = mpi_rank +# mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) +# else: +# self._service_process.releaseID(id) +# +# def get_length(self): +# if MPI and self._use_subprocess: +# mpi_comm = MPI.COMM_WORLD +# mpi_rank = mpi_comm.Get_rank() +# +# data_package = {} +# data_package["func_name"] = "get_length" +# data_package["args"] = 0 +# data_package["src_rank"] = mpi_rank +# mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) +# data = mpi_comm.recv(source=self._serverrank, tag=self._response_tag) +# +# return int(data["result"]) +# else: +# return self._service_process.__len__() +# +# def get_total_length(self): +# if MPI and self._use_subprocess: +# mpi_comm = MPI.COMM_WORLD +# mpi_rank = mpi_comm.Get_rank() +# +# data_package = {} +# data_package["func_name"] = "get_total_length" +# data_package["args"] = 0 +# data_package["src_rank"] = mpi_rank +# mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) +# data = mpi_comm.recv(source=self._serverrank, tag=self._response_tag) +# +# return int(data["result"]) +# else: +# return self._service_process.total_length +# +# def __len__(self): +# return self.get_length() +# +# @property +# def total_length(self): +# return self.get_total_length() + + + +class GenerateID_Service(BaseIdGenerator): + _request_tag = 5 + _response_tag = 6 + + def __init__(self, base_generator_obj): + super(GenerateID_Service, self).__init__() + self._service_process = None + self._serverrank = 0 + self._request_tag = 5 + self._response_tag = 6 + self._recover_ids = False + self._use_subprocess = True + + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + mpi_size = mpi_comm.Get_size() + if mpi_size <= 1: + self._use_subprocess = False + else: + self._serverrank = mpi_size-1 + if mpi_rank == self._serverrank: + # self._service_process = Process(target=executor, name="IdService", args=(service_bundle, base_generator_obj), daemon=True) + # self._service_process.start() + # print("Starting ID service process") + # logger.info("Starting ID service process") + self._service_process = Thread(target=executor, name="IdService", args=(base_generator_obj, self._request_tag, self._response_tag), daemon=True) + # self._service_process.daemon = True + self._service_process.start() + # executor(base_generator_obj, self._request_tag, self._response_tag) + # mpi_comm.Barrier() + # logger.info("worker - MPI rank: {} pid: {}".format(mpi_rank, getpid())) + self._subscribe_() + else: + self._use_subprocess = False + + if not self._use_subprocess: + self._service_process = base_generator_obj() + + def __del__(self): + self._abort_() + + def _subscribe_(self): + if MPI and self._use_subprocess: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + data_package = {} + data_package["func_name"] = "thread_subscribe" + data_package["args"] = 0 + data_package["src_rank"] = mpi_rank + mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) + + def _abort_(self): + if MPI and self._use_subprocess: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + data_package = {} + data_package["func_name"] = "thread_abort" + data_package["args"] = 0 + data_package["src_rank"] = mpi_rank + mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) + + def close(self): + self._abort_() + + def setTimeLine(self, min_time=0.0, max_time=1.0): + if MPI and self._use_subprocess: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank == 0: + data_package = {} + data_package["func_name"] = "setTimeLine" + data_package["args"] = 2 + data_package["argv"] = [min_time, max_time] + data_package["src_rank"] = mpi_rank + mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) + else: + self._service_process.setTimeLine(min_time, max_time) + + def setDepthLimits(self, min_depth=0.0, max_depth=1.0): + if MPI and self._use_subprocess: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + if mpi_rank == 0: + data_package = {} + data_package["func_name"] = "setDepthLimits" + data_package["args"] = 2 + data_package["argv"] = [min_depth, max_depth] + data_package["src_rank"] = mpi_rank + mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) + else: + self._service_process.setDepthLimits(min_depth, max_depth) + + def getID(self, lon, lat, depth, time): + if MPI and self._use_subprocess: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + + data_package = {} + data_package["func_name"] = "getID" + data_package["args"] = 4 + data_package["argv"] = [lon, lat, depth, time] + data_package["src_rank"] = mpi_rank + mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) + data = mpi_comm.recv(source=self._serverrank, tag=self._response_tag) + return int(data["result"]) + else: + return self._service_process.getID(lon, lat, depth, time) + + def nextID(self, lon, lat, depth, time): + return self.getID(lon, lat, depth, time) + + def releaseID(self, id): + if not self._recover_ids: + return + if MPI and self._use_subprocess: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + + data_package = {} + data_package["func_name"] = "releaseID" + data_package["args"] = 1 + data_package["argv"] = [id, ] + data_package["src_rank"] = mpi_rank + mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) + else: + self._service_process.releaseID(id) + + def get_length(self): + if MPI and self._use_subprocess: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + + data_package = {} + data_package["func_name"] = "get_length" + data_package["args"] = 0 + data_package["src_rank"] = mpi_rank + mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) + data = mpi_comm.recv(source=self._serverrank, tag=self._response_tag) + + return int(data["result"]) + else: + return self._service_process.__len__() + + def get_total_length(self): + if MPI and self._use_subprocess: + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + + data_package = {} + data_package["func_name"] = "get_total_length" + data_package["args"] = 0 + data_package["src_rank"] = mpi_rank + mpi_comm.send(data_package, dest=self._serverrank, tag=self._request_tag) + data = mpi_comm.recv(source=self._serverrank, tag=self._response_tag) + + return int(data["result"]) + else: + return self._service_process.total_length + + def __len__(self): + return self.get_length() + + @property + def total_length(self): + return self.get_total_length() diff --git a/parcels/tools/message_service.py b/parcels/tools/message_service.py new file mode 100644 index 0000000000..22eb89cee9 --- /dev/null +++ b/parcels/tools/message_service.py @@ -0,0 +1,75 @@ +# from os import getpid +# from parcels.tools import logger + + +try: + from mpi4py import MPI +except: + MPI = None + + +def mpi_execute_requested_messages(exec_class, request_tag=0, response_tag=1): + """ + A sub-thread/sub-process main function that manages a central (i.e. global) object. + The process is that MPI processes can subscribe (via 'thread_subscribe' as function name) to the message queue. Then, + those processes can send point-to-point message requests for functions of the central object to be executed. + Potential results values are returned via message. + IMPORTANT: upon MPI execution end, each process needs to send an 'abort' signal ('thread_abort' as function name) to + unsubscribe from the message queue so to leave the message queue in an orderly fashion. + """ + requester_obj = exec_class() + mpi_comm = MPI.COMM_WORLD + mpi_rank = mpi_comm.Get_rank() + # logger.info("service - MPI rank: {} pid: {}".format(mpi_rank, getpid())) + _subscribed = {} + _running = True + while _running: + msg_status = MPI.Status() + msg = mpi_comm.irecv(source=MPI.ANY_SOURCE, tag=request_tag) + test_result = msg.test(status=msg_status) + # while not (isinstance(test_result, tuple) or isinstance(test_result, list)) or ((test_result[0] == False) or (test_result[0] == True and not isinstance(test_result[1], dict))): + # while (test_result[0] == False) or (test_result[0] == True and not isinstance(test_result[1], dict)): + while (not test_result[0]) or (test_result[0] and not isinstance(test_result[1], dict)): + test_result = msg.test(status=msg_status) + + request_package = test_result[1] + # logger.info("ID serv. - recv.: {} - (srv. rank: {}; snd. rank: {}; pkg. rank: {}".format(request_package["func_name"], mpi_rank, msg_status.Get_source(), request_package["src_rank"])) + # logger.info("package received: {}".format(request_package)) + assert isinstance(request_package, dict) + # dst = int(request_package["src_rank"]) + + # logger.info("Package: {}".format(request_package)) + func_name = request_package["func_name"] + args = int(request_package["args"]) + argv = [] + if args > 0: + argv = request_package["argv"] + + if func_name == "thread_subscribe": + # logger.info("'subscribe' message received.") + _subscribed[msg_status.Get_source()] = True + elif func_name == "thread_abort": + # logger.info("'abort' message received (src: {}).".format(msg_status.Get_source())) + _subscribed[msg_status.Get_source()] = False + # logger.info("Subscribers: {}".format( _subscribed )) + _running = False + for flag in _subscribed: + _running |= flag + # logger.warn("Subscribed: {}".format(_subscribed)) + if not _running: + break + else: + call_func = getattr(requester_obj, func_name) + res = None + if call_func is not None: + if args > 0: + res = call_func(*argv) + else: + res = call_func() + if res is not None: + response_package = {"result": res, "src_rank": mpi_rank} + # logger.info("sending message: {}".format(response_package)) + # msg = mpi_comm.isend(response_package, dest=dst, tag=response_tag) + # msg.wait() + mpi_comm.send(response_package, dest=msg_status.Get_source(), tag=response_tag) + # logger.info("ABORTED ID Service") diff --git a/parcels/tools/perlin2d.py b/parcels/tools/perlin2d.py index dc7afa5fa7..4b39b4cf94 100644 --- a/parcels/tools/perlin2d.py +++ b/parcels/tools/perlin2d.py @@ -4,33 +4,35 @@ from scipy import ndimage from time import time + def generate_perlin_noise_2d(shape, res): def f(t): return 6*t**5 - 15*t**4 + 10*t**3 - + delta = (res[0] / shape[0], res[1] / shape[1]) d = (shape[0] // res[0], shape[1] // res[1]) - grid = np.mgrid[0:res[0]:delta[0],0:res[1]:delta[1]].transpose(1, 2, 0) % 1 + grid = np.mgrid[0:res[0]:delta[0], 0:res[1]:delta[1]].transpose(1, 2, 0) % 1 # Gradients angles = 2*np.pi*np.random.rand(res[0]+1, res[1]+1) gradients = np.dstack((np.cos(angles), np.sin(angles))) - g00 = gradients[0:-1,0:-1].repeat(d[0], 0).repeat(d[1], 1) - g10 = gradients[1: ,0:-1].repeat(d[0], 0).repeat(d[1], 1) - g01 = gradients[0:-1,1: ].repeat(d[0], 0).repeat(d[1], 1) - g11 = gradients[1: ,1: ].repeat(d[0], 0).repeat(d[1], 1) + g00 = gradients[0:-1, 0:-1].repeat(d[0], 0).repeat(d[1], 1) + g10 = gradients[1:, 0:-1].repeat(d[0], 0).repeat(d[1], 1) + g01 = gradients[0:-1, 1:].repeat(d[0], 0).repeat(d[1], 1) + g11 = gradients[1:, 1:].repeat(d[0], 0).repeat(d[1], 1) # Ramps - n00 = np.sum(np.dstack((grid[:,:,0] , grid[:,:,1] )) * g00, 2) - n10 = np.sum(np.dstack((grid[:,:,0]-1, grid[:,:,1] )) * g10, 2) - n01 = np.sum(np.dstack((grid[:,:,0] , grid[:,:,1]-1)) * g01, 2) - n11 = np.sum(np.dstack((grid[:,:,0]-1, grid[:,:,1]-1)) * g11, 2) + n00 = np.sum(np.dstack((grid[:, :, 0], grid[:, :, 1])) * g00, 2) + n10 = np.sum(np.dstack((grid[:, :, 0]-1, grid[:, :, 1])) * g10, 2) + n01 = np.sum(np.dstack((grid[:, :, 0], grid[:, :, 1]-1)) * g01, 2) + n11 = np.sum(np.dstack((grid[:, :, 0]-1, grid[:, :, 1]-1)) * g11, 2) # Interpolation t = f(grid) - n0 = n00*(1-t[:,:,0]) + t[:,:,0]*n10 - n1 = n01*(1-t[:,:,0]) + t[:,:,0]*n11 - return np.sqrt(2)*((1-t[:,:,1])*n0 + t[:,:,1]*n1) - + n0 = n00*(1-t[:, :, 0]) + t[:, :, 0]*n10 + n1 = n01*(1-t[:, :, 0]) + t[:, :, 0]*n11 + return np.sqrt(2)*((1-t[:, :, 1])*n0 + t[:, :, 1]*n1) + + def generate_fractal_noise_2d(shape, res, octaves=1, persistence=0.5): - rs = RandomState( MT19937(SeedSequence(int(round(time() * 1000)))) ) + RandomState(MT19937(SeedSequence(int(round(time() * 1000))))) # rs = noise = np.zeros(shape) frequency = 1 amplitude = 1 @@ -40,8 +42,9 @@ def generate_fractal_noise_2d(shape, res, octaves=1, persistence=0.5): amplitude *= persistence return noise + def generate_fractal_noise_temporal2d(shape, tsteps, res, octaves=1, persistence=0.5, max_shift=((-1, 1), (-1, 1))): - rs = RandomState( MT19937(SeedSequence(int(round(time() * 1000)))) ) + RandomState(MT19937(SeedSequence(int(round(time() * 1000))))) # rs = noise = np.zeros(shape) frequency = 1 amplitude = 1 @@ -54,21 +57,22 @@ def generate_fractal_noise_temporal2d(shape, tsteps, res, octaves=1, persistence result = np.zeros(ishape) timage = np.zeros(noise.shape) for i in range(0, tsteps): - result[i,:,:] = noise + result[i, :, :] = noise sx = np.random.randint(max_shift[0][0], max_shift[0][1], dtype=np.int32) sy = np.random.randint(max_shift[1][0], max_shift[1][1], dtype=np.int32) ndimage.shift(noise, (sx, sy), timage, order=3, mode='mirror') noise = timage return result - + + if __name__ == '__main__': import matplotlib.pyplot as plt - + np.random.seed(0) noise = generate_perlin_noise_2d((256, 256), (8, 8)) plt.imshow(noise, cmap='gray', interpolation='lanczos') plt.colorbar() - + np.random.seed(0) noise = generate_fractal_noise_2d((256, 256), (8, 8), 5) plt.figure() diff --git a/parcels/tools/perlin3d.py b/parcels/tools/perlin3d.py index 7424e97ff8..c24306779f 100644 --- a/parcels/tools/perlin3d.py +++ b/parcels/tools/perlin3d.py @@ -3,48 +3,50 @@ from numpy.random import RandomState, SeedSequence from scipy import ndimage from time import time - + + def generate_perlin_noise_3d(shape, res): def f(t): return 6*t**5 - 15*t**4 + 10*t**3 - + delta = (res[0] / shape[0], res[1] / shape[1], res[2] / shape[2]) d = (shape[0] // res[0], shape[1] // res[1], shape[2] // res[2]) - grid = np.mgrid[0:res[0]:delta[0],0:res[1]:delta[1],0:res[2]:delta[2]] + grid = np.mgrid[0:res[0]:delta[0], 0:res[1]:delta[1], 0:res[2]:delta[2]] grid = grid.transpose(1, 2, 3, 0) % 1 # Gradients theta = 2*np.pi*np.random.rand(res[0]+1, res[1]+1, res[2]+1) phi = 2*np.pi*np.random.rand(res[0]+1, res[1]+1, res[2]+1) gradients = np.stack((np.sin(phi)*np.cos(theta), np.sin(phi)*np.sin(theta), np.cos(phi)), axis=3) - g000 = gradients[0:-1,0:-1,0:-1].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) - g100 = gradients[1: ,0:-1,0:-1].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) - g010 = gradients[0:-1,1: ,0:-1].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) - g110 = gradients[1: ,1: ,0:-1].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) - g001 = gradients[0:-1,0:-1,1: ].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) - g101 = gradients[1: ,0:-1,1: ].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) - g011 = gradients[0:-1,1: ,1: ].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) - g111 = gradients[1: ,1: ,1: ].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g000 = gradients[0:-1, 0:-1, 0:-1].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g100 = gradients[1:, 0:-1, 0:-1].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g010 = gradients[0:-1, 1:, 0:-1].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g110 = gradients[1:, 1:, 0:-1].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g001 = gradients[0:-1, 0:-1, 1:].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g101 = gradients[1:, 0:-1, 1:].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g011 = gradients[0:-1, 1:, 1:].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) + g111 = gradients[1:, 1:, 1:].repeat(d[0], 0).repeat(d[1], 1).repeat(d[2], 2) # Ramps - n000 = np.sum(np.stack((grid[:,:,:,0] , grid[:,:,:,1] , grid[:,:,:,2] ), axis=3) * g000, 3) - n100 = np.sum(np.stack((grid[:,:,:,0]-1, grid[:,:,:,1] , grid[:,:,:,2] ), axis=3) * g100, 3) - n010 = np.sum(np.stack((grid[:,:,:,0] , grid[:,:,:,1]-1, grid[:,:,:,2] ), axis=3) * g010, 3) - n110 = np.sum(np.stack((grid[:,:,:,0]-1, grid[:,:,:,1]-1, grid[:,:,:,2] ), axis=3) * g110, 3) - n001 = np.sum(np.stack((grid[:,:,:,0] , grid[:,:,:,1] , grid[:,:,:,2]-1), axis=3) * g001, 3) - n101 = np.sum(np.stack((grid[:,:,:,0]-1, grid[:,:,:,1] , grid[:,:,:,2]-1), axis=3) * g101, 3) - n011 = np.sum(np.stack((grid[:,:,:,0] , grid[:,:,:,1]-1, grid[:,:,:,2]-1), axis=3) * g011, 3) - n111 = np.sum(np.stack((grid[:,:,:,0]-1, grid[:,:,:,1]-1, grid[:,:,:,2]-1), axis=3) * g111, 3) + n000 = np.sum(np.stack((grid[:, :, :, 0], grid[:, :, :, 1], grid[:, :, :, 2]), axis=3) * g000, 3) + n100 = np.sum(np.stack((grid[:, :, :, 0]-1, grid[:, :, :, 1], grid[:, :, :, 2]), axis=3) * g100, 3) + n010 = np.sum(np.stack((grid[:, :, :, 0], grid[:, :, :, 1]-1, grid[:, :, :, 2]), axis=3) * g010, 3) + n110 = np.sum(np.stack((grid[:, :, :, 0]-1, grid[:, :, :, 1]-1, grid[:, :, :, 2]), axis=3) * g110, 3) + n001 = np.sum(np.stack((grid[:, :, :, 0], grid[:, :, :, 1], grid[:, :, :, 2]-1), axis=3) * g001, 3) + n101 = np.sum(np.stack((grid[:, :, :, 0]-1, grid[:, :, :, 1], grid[:, :, :, 2]-1), axis=3) * g101, 3) + n011 = np.sum(np.stack((grid[:, :, :, 0], grid[:, :, :, 1]-1, grid[:, :, :, 2]-1), axis=3) * g011, 3) + n111 = np.sum(np.stack((grid[:, :, :, 0]-1, grid[:, :, :, 1]-1, grid[:, :, :, 2]-1), axis=3) * g111, 3) # Interpolation t = f(grid) - n00 = n000*(1-t[:,:,:,0]) + t[:,:,:,0]*n100 - n10 = n010*(1-t[:,:,:,0]) + t[:,:,:,0]*n110 - n01 = n001*(1-t[:,:,:,0]) + t[:,:,:,0]*n101 - n11 = n011*(1-t[:,:,:,0]) + t[:,:,:,0]*n111 - n0 = (1-t[:,:,:,1])*n00 + t[:,:,:,1]*n10 - n1 = (1-t[:,:,:,1])*n01 + t[:,:,:,1]*n11 - return ((1-t[:,:,:,2])*n0 + t[:,:,:,2]*n1) - + n00 = n000*(1-t[:, :, :, 0]) + t[:, :, :, 0]*n100 + n10 = n010*(1-t[:, :, :, 0]) + t[:, :, :, 0]*n110 + n01 = n001*(1-t[:, :, :, 0]) + t[:, :, :, 0]*n101 + n11 = n011*(1-t[:, :, :, 0]) + t[:, :, :, 0]*n111 + n0 = (1-t[:, :, :, 1])*n00 + t[:, :, :, 1]*n10 + n1 = (1-t[:, :, :, 1])*n01 + t[:, :, :, 1]*n11 + return (1-t[:, :, :, 2])*n0 + t[:, :, :, 2]*n1 + + def generate_fractal_noise_3d(shape, res, octaves=1, persistence=0.5): - rs = RandomState( MT19937(SeedSequence(int(round(time() * 1000)))) ) + RandomState(MT19937(SeedSequence(int(round(time() * 1000))))) noise = np.zeros(shape) frequency = 1 amplitude = 1 @@ -54,8 +56,9 @@ def generate_fractal_noise_3d(shape, res, octaves=1, persistence=0.5): amplitude *= persistence return noise + def generate_fractal_noise_temporal3d(shape, tsteps, res, octaves=1, persistence=0.5, max_shift=(1, 1, 1)): - rs = RandomState( MT19937(SeedSequence(int(round(time() * 1000)))) ) + RandomState(MT19937(SeedSequence(int(round(time() * 1000))))) noise = np.zeros(shape) frequency = 1 amplitude = 1 @@ -63,12 +66,11 @@ def generate_fractal_noise_temporal3d(shape, tsteps, res, octaves=1, persistence noise += amplitude * generate_perlin_noise_3d(shape, (frequency*res[0], frequency*res[1], frequency*res[2])) frequency *= 2 amplitude *= persistence - ishape = (tsteps, )+shape result = np.zeros(ishape) timage = np.zeros(noise.shape) for i in range(0, tsteps): - result[i,:,:] = noise + result[i, :, :] = noise sx = np.random.randint(-max_shift[0], max_shift[0], dtype=np.int32) sy = np.random.randint(-max_shift[1], max_shift[1], dtype=np.int32) sz = np.random.randint(-max_shift[2], max_shift[2], dtype=np.int32) @@ -76,13 +78,14 @@ def generate_fractal_noise_temporal3d(shape, tsteps, res, octaves=1, persistence noise = timage return result + if __name__ == '__main__': import matplotlib.pyplot as plt import matplotlib.animation as animation - + np.random.seed(0) noise = generate_fractal_noise_3d((32, 256, 256), (1, 4, 4), 4) - + fig = plt.figure() images = [[plt.imshow(layer, cmap='gray', interpolation='lanczos', animated=True)] for layer in noise] animation = animation.ArtistAnimation(fig, images, interval=50, blit=True) diff --git a/parcels/wrapping/__init__.py b/parcels/wrapping/__init__.py new file mode 100644 index 0000000000..ac0df6ac56 --- /dev/null +++ b/parcels/wrapping/__init__.py @@ -0,0 +1,6 @@ +from .code_compiler import * # noqa +from .code_generator import * # noqa +from .code_interface import * # noqa + + +c_lib_register = LibraryRegisterC() diff --git a/parcels/wrapping/code_compiler.py b/parcels/wrapping/code_compiler.py new file mode 100644 index 0000000000..8541ac874b --- /dev/null +++ b/parcels/wrapping/code_compiler.py @@ -0,0 +1,345 @@ +import os +import subprocess +from struct import calcsize +# from pathlib import Path + +try: + from mpi4py import MPI +except: + MPI = None + + +class Compiler_parameters(object): + def __init__(self): + self._compiler = "" + self._cppargs = [] + self._ldargs = [] + self._incdirs = [] + self._libdirs = [] + self._libs = [] + self._dynlib_ext = "" + self._stclib_ext = "" + self._obj_ext = "" + self._exe_ext = "" + + @property + def compiler(self): + return self._compiler + + @property + def cppargs(self): + return self._cppargs + + @property + def ldargs(self): + return self._ldargs + + @property + def incdirs(self): + return self._incdirs + + @property + def libdirs(self): + return self._libdirs + + @property + def libs(self): + return self._libs + + @property + def dynlib_ext(self): + return self._dynlib_ext + + @property + def stclib_ext(self): + return self._stclib_ext + + @property + def obj_ext(self): + return self._obj_ext + + @property + def exe_ext(self): + return self._exe_ext + + +class GNU_parameters(Compiler_parameters): + def __init__(self, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None): + super(GNU_parameters, self).__init__() + if cppargs is None: + cppargs = [] + if ldargs is None: + ldargs = [] + if incdirs is None: + incdirs = [] + if libdirs is None: + libdirs = [] + if libs is None: + libs = [] + + Iflags = [] + if incdirs is not None and isinstance(incdirs, list): + for i, dir in enumerate(incdirs): + Iflags.append("-I"+dir) + Lflags = [] + if libdirs is not None and isinstance(libdirs, list): + for i, dir in enumerate(libdirs): + Lflags.append("-L"+dir) + lflags = [] + if libs is not None and isinstance(libs, list): + for i, lib in enumerate(libs): + lflags.append("-l" + lib) + + cc_env = os.getenv('CC') + self._compiler = "mpicc" if MPI else "gcc" if cc_env is None else cc_env + opt_flags = ['-g', '-O3'] + arch_flag = ['-m64' if calcsize("P") == 8 else '-m32'] + self._cppargs = ['-Wall', '-fPIC'] + if len(incdirs) > 0: + self._cppargs += Iflags + self._cppargs += opt_flags + cppargs + arch_flag + self._ldargs = ['-shared'] + if len(Lflags) > 0: + self._ldargs += Lflags + if len(lflags) > 0: + self._ldargs += lflags + self._ldargs += ldargs + if len(Lflags) > 0: + self._ldargs += ['-Wl,-rpath=%s' % (":".join(libdirs))] + self._ldargs += arch_flag + self._incdirs = incdirs + self._libdirs = libdirs + self._libs = libs + self._dynlib_ext = "so" + self._stclib_ext = "a" + self._obj_ext = "o" + self._exe_ext = "" + + +class Clang_parameters(Compiler_parameters): + def __init__(self, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None): + super(Clang_parameters, self).__init__() + if cppargs is None: + cppargs = [] + if ldargs is None: + ldargs = [] + if incdirs is None: + incdirs = [] + if libdirs is None: + libdirs = [] + if libs is None: + libs = [] + self._compiler = "cc" + self._cppargs = cppargs + self._ldargs = ldargs + self._incdirs = incdirs + self._libdirs = libdirs + self._libs = libs + self._dynlib_ext = "dynlib" + self._stclib_ext = "a" + self._obj_ext = "o" + self._exe_ext = "exe" + + +class MinGW_parameters(Compiler_parameters): + def __init__(self, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None): + super(MinGW_parameters, self).__init__() + if cppargs is None: + cppargs = [] + if ldargs is None: + ldargs = [] + if incdirs is None: + incdirs = [] + if libdirs is None: + libdirs = [] + if libs is None: + libs = [] + self._compiler = "gcc" + self._cppargs = cppargs + self._ldargs = ldargs + self._incdirs = incdirs + self._libdirs = libdirs + self._libs = libs + self._dynlib_ext = "so" + self._stclib_ext = "a" + self._obj_ext = "o" + self._exe_ext = "exe" + + +class VS_parameters(Compiler_parameters): + def __init__(self, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None): + super(VS_parameters, self).__init__() + if cppargs is None: + cppargs = [] + if ldargs is None: + ldargs = [] + if incdirs is None: + incdirs = [] + if libdirs is None: + libdirs = [] + if libs is None: + libs = [] + self._compiler = "cl" + self._cppargs = cppargs + self._ldargs = ldargs + self._incdirs = incdirs + self._libdirs = libdirs + self._libs = libs + self._dynlib_ext = "dll" + self._stclib_ext = "lib" + self._obj_ext = "obj" + self._exe_ext = "exe" + + +class CCompiler(object): + """A compiler object for creating and loading shared libraries. + + :arg cc: C compiler executable (uses environment variable ``CC`` if not provided). + :arg cppargs: A list of arguments to the C compiler (optional). + :arg ldargs: A list of arguments to the linker (optional).""" + + def __init__(self, cc=None, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None, tmp_dir=os.getcwd()): + if cppargs is None: + cppargs = [] + if ldargs is None: + ldargs = [] + + self._cc = os.getenv('CC') if cc is None else cc + self._cppargs = cppargs + self._ldargs = ldargs + self._dynlib_ext = "" + self._stclib_ext = "" + self._obj_ext = "" + self._exe_ext = "" + self._tmp_dir = tmp_dir + + def compile(self, src, obj, log): + pass + + def _create_compile_process_(self, cmd, src, log): + with open(log, 'w') as logfile: + try: + subprocess.check_call(cmd, stdout=logfile, stderr=logfile) + except OSError: + err = """OSError during compilation +Please check if compiler exists: %s""" % self._cc + raise RuntimeError(err) + except subprocess.CalledProcessError: + with open(log, 'r') as logfile2: + err = """Error during compilation: +Compilation command: %s +Source/Destination file: %s +Log file: %s + +Log output: %s""" % (" ".join(cmd), src, logfile.name, logfile2.read()) + raise RuntimeError(err) + return True + + +class CCompiler_SS(CCompiler): + """ + single-stage C-compiler; used for a SINGLE source file + """ + def __init__(self, cc=None, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None, tmp_dir=os.getcwd()): + super(CCompiler_SS, self).__init__(cc=cc, cppargs=cppargs, ldargs=ldargs, incdirs=incdirs, libdirs=libdirs, libs=libs, tmp_dir=tmp_dir) + + def __str__(self): + output = "[CCompiler_SS]: " + output += "('cc': {}), ".format(self._cc) + output += "('cppargs': {}), ".format(self._cppargs) + output += "('ldargs': {}), ".format(self._ldargs) + output += "('incdirs': {}), ".format(self._incdirs) + output += "('libdirs': {}), ".format(self._libdirs) + output += "('libs': {}), ".format(self._libs) + output += "('tmp_dir': {}), ".format(self._tmp_dir) + return output + + def compile(self, src, obj, log): + cc = [self._cc] + self._cppargs + ['-o', obj, src] + self._ldargs + with open(log, 'w') as logfile: + logfile.write("Compiling: %s\n" % " ".join(cc)) + self._create_compile_process_(cc, src, log) + + +class CCompiler_MS(CCompiler): + """ + multi-stage C-compiler: used for multiple source files + """ + def __init__(self, cc=None, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None, tmp_dir=os.getcwd()): + super(CCompiler_MS, self).__init__(cc=cc, cppargs=cppargs, ldargs=ldargs, incdirs=incdirs, libdirs=libdirs, libs=libs, tmp_dir=tmp_dir) + + def compile(self, src, obj, log): + objs = [] + for src_file in src: + src_file_wo_ext = os.path.splitext(src_file)[0] + src_file_wo_ppath = os.path.basename(src_file_wo_ext) + if MPI and MPI.COMM_WORLD.Get_size() > 1: + src_file_wo_ppath = "%s_%d" % (src_file_wo_ppath, MPI.COMM_WORLD.Get_rank()) + obj_file = os.path.join(self._tmp_dir, src_file_wo_ppath) + "." + self._obj_ext + objs.append(obj_file) + slog_file = os.path.join(self._tmp_dir, src_file_wo_ppath) + "_o" + "." + "log" + # cc = [self._cc] + self._cppargs + ["-c", src_file] + cc = [self._cc] + self._cppargs + ['-c', src_file] + ['-o', obj_file] + with open(log, 'w') as logfile: + logfile.write("Compiling: %s\n" % " ".join(cc)) + success = self._create_compile_process_(cc, src_file, slog_file) + if success and os._exists(slog_file): + os.remove(slog_file) + cc = [self._cc] + objs + ['-o', obj] + self._ldargs + with open(log, 'a') as logfile: + logfile.write("Linking: %s\n" % " ".join(cc)) + self._create_compile_process_(cc, obj, log) + for fpath in objs: + if os.path.exists(fpath): + os.remove(fpath) + + +class GNUCompiler_SS(CCompiler_SS): + """A compiler object for the GNU Linux toolchain. + + :arg cppargs: A list of arguments to pass to the C compiler + (optional). + :arg ldargs: A list of arguments to pass to the linker (optional).""" + def __init__(self, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None, tmp_dir=os.getcwd()): + c_params = GNU_parameters(cppargs, ldargs, incdirs, libdirs, libs) + super(GNUCompiler_SS, self).__init__(c_params.compiler, cppargs=c_params.cppargs, ldargs=c_params.ldargs, incdirs=c_params.incdirs, libdirs=c_params.libdirs, libs=c_params.libs, tmp_dir=tmp_dir) + self._dynlib_ext = c_params.dynlib_ext + self._stclib_ext = c_params.stclib_ext + self._obj_ext = c_params.obj_ext + self._exe_ext = c_params.exe_ext + + def compile(self, src, obj, log): + lib_pathfile = os.path.basename(obj) + lib_pathdir = os.path.dirname(obj) + if lib_pathfile[0:3] != "lib": + lib_pathfile = "lib"+lib_pathfile + obj = os.path.join(lib_pathdir, lib_pathfile) + + super(GNUCompiler_SS, self).compile(src, obj, log) + + +class GNUCompiler_MS(CCompiler_MS): + """A compiler object for the GNU Linux toolchain. + + :arg cppargs: A list of arguments to pass to the C compiler + (optional). + :arg ldargs: A list of arguments to pass to the linker (optional).""" + def __init__(self, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None, tmp_dir=os.getcwd()): + c_params = GNU_parameters(cppargs, ldargs, incdirs, libdirs, libs) + super(GNUCompiler_MS, self).__init__(c_params.compiler, cppargs=c_params.cppargs, ldargs=c_params.ldargs, incdirs=c_params.incdirs, libdirs=c_params.libdirs, libs=c_params.libs, tmp_dir=tmp_dir) + self._dynlib_ext = c_params.dynlib_ext + self._stclib_ext = c_params.stclib_ext + self._obj_ext = c_params.obj_ext + self._exe_ext = c_params.exe_ext + + def compile(self, src, obj, log): + lib_pathfile = os.path.basename(obj) + lib_pathdir = os.path.dirname(obj) + if lib_pathfile[0:3] != "lib": + lib_pathfile = "lib"+lib_pathfile + obj = os.path.join(lib_pathdir, lib_pathfile) + + super(GNUCompiler_MS, self).compile(src, obj, log) + + +GNUCompiler = GNUCompiler_SS diff --git a/parcels/codegenerator.py b/parcels/wrapping/code_generator.py similarity index 82% rename from parcels/codegenerator.py rename to parcels/wrapping/code_generator.py index 46341ad5cf..827c58e5c8 100644 --- a/parcels/codegenerator.py +++ b/parcels/wrapping/code_generator.py @@ -1,11 +1,11 @@ import ast import collections import math -import numpy as np import random from copy import copy import cgen as c +import numpy as np from parcels.field import Field from parcels.field import NestedField @@ -13,6 +13,8 @@ from parcels.field import VectorField from parcels.tools.loggers import logger +# from Node import Node, NodeJIT + class IntrinsicNode(ast.AST): def __init__(self, obj, ccode): @@ -172,7 +174,7 @@ def __getattr__(self, attr): attr = self.symbol_map[attr] return IntrinsicNode(None, ccode=attr) else: - raise AttributeError("""Unknown error code encountered: %s""" + raise AttributeError("""Unknown math function encountered: %s""" % attr) @@ -205,7 +207,7 @@ class IntrinsicTransformer(ast.NodeTransformer): names, such as 'particle' or 'fieldset', inserts placeholder objects and propagates attribute access.""" - def __init__(self, fieldset, ptype): + def __init__(self, ptype, fieldset=None): self.fieldset = fieldset self.ptype = ptype @@ -224,7 +226,7 @@ def get_tmp(self): def visit_Name(self, node): """Inject IntrinsicNode objects into the tree according to keyword""" - if node.id == 'fieldset': + if node.id == 'fieldset' and self.fieldset is not None: node = FieldSetNode(self.fieldset, ccode='fset') elif node.id == 'particle': node = ParticleNode(self.ptype, ccode='particle') @@ -369,7 +371,7 @@ class KernelGenerator(ast.NodeVisitor): kernel_vars = ['particle', 'fieldset', 'time', 'output_time', 'tol'] array_vars = [] - def __init__(self, fieldset, ptype): + def __init__(self, ptype, fieldset=None): self.fieldset = fieldset self.ptype = ptype self.field_args = collections.OrderedDict() @@ -378,7 +380,7 @@ def __init__(self, fieldset, ptype): def generate(self, py_ast, funcvars): # Replace occurences of intrinsic objects in Python AST - transformer = IntrinsicTransformer(self.fieldset, self.ptype) + transformer = IntrinsicTransformer(self.ptype, self.fieldset) py_ast = transformer.visit(py_ast) # Untangle Pythonic tuple-assignment statements @@ -869,11 +871,168 @@ def _isdocstr(node): node.ccode = node.s -class LoopGenerator(object): +class NodeLoopGenerator(object): + """Code generator class that adds type definitions and the outer + loop around kernel functions to generate compilable C code.""" + + def __init__(self, ptype=None, fieldset=None): + self.fieldset = fieldset + self.ptype = ptype + + def generate(self, funcname, field_args, const_args, kernel_ast, c_include): + ccode = [] + + # Add include for Parcels and math header + ccode += [str(c.Include("parcels.h", system=False))] + ccode += [str(c.Include("math.h", system=False))] + ccode += [str(c.Include('node.h', system=False))] + ccode += [str(c.Assign('double _next_dt', '0'))] + ccode += [str(c.Assign('size_t _next_dt_set', '0'))] + + # Generate type definition for particle type + vdecl = [] + for v in self.ptype.variables: + if v.dtype == np.uint64: + vdecl.append(c.Pointer(c.POD(np.void, v.name))) + else: + vdecl.append(c.POD(v.dtype, v.name)) + + ccode += [str(c.Typedef(c.GenerableStruct("", vdecl, declname=self.ptype.name)))] + + args = [c.Pointer(c.Value(self.ptype.name, "particle_backup")), + c.Pointer(c.Value(self.ptype.name, "particle"))] + p_back_set_decl = c.FunctionDeclaration(c.Static(c.DeclSpecifier(c.Value("void", "set_particle_backup"), + spec='inline')), args) + body = [] + for v in self.ptype.variables: + if v.dtype != np.uint64 and v.name not in ['dt', 'state']: + body += [c.Assign(("particle_backup->%s" % v.name), ("particle->%s" % v.name))] + p_back_set_body = c.Block(body) + p_back_set = str(c.FunctionBody(p_back_set_decl, p_back_set_body)) + ccode += [p_back_set] + + args = [c.Pointer(c.Value(self.ptype.name, "particle_backup")), + c.Pointer(c.Value(self.ptype.name, "particle"))] + p_back_get_decl = c.FunctionDeclaration(c.Static(c.DeclSpecifier(c.Value("void", "get_particle_backup"), + spec='inline')), args) + body = [] + for v in self.ptype.variables: + if v.dtype != np.uint64 and v.name not in ['dt', 'state']: + body += [c.Assign(("particle->%s" % v.name), ("particle_backup->%s" % v.name))] + p_back_get_body = c.Block(body) + p_back_get = str(c.FunctionBody(p_back_get_decl, p_back_get_body)) + ccode += [p_back_get] + + update_next_dt_decl = c.FunctionDeclaration(c.Static(c.DeclSpecifier(c.Value("void", "update_next_dt"), + spec='inline')), [c.Value('double', 'dt')]) + if 'update_next_dt' in str(kernel_ast): + body = [] + body += [c.Assign("_next_dt", "dt")] + body += [c.Assign("_next_dt_set", "1")] + update_next_dt_body = c.Block(body) + update_next_dt = str(c.FunctionBody(update_next_dt_decl, update_next_dt_body)) + ccode += [update_next_dt] + + if c_include: + ccode += [c_include] + + # Insert kernel code + ccode += [str(kernel_ast)] + + # Generate outer loop for repeated kernel invocation + args = [c.Value("int", "num_particles"), + c.Pointer(c.Value("NodeJIT", "node_begin")), + # c.Pointer(c.Value(self.ptype.name, "particles")), + c.Value("double", "endtime"), + c.Value("double", "dt")] + if field_args is not None: + for field, _ in field_args.items(): + args += [c.Pointer(c.Value("CField", "%s" % field))] + else: + pass + if const_args is not None: + for const, _ in const_args.items(): + args += [c.Value("double", const)] + else: + pass + + fargs_lst = ['particle->time'] + if field_args is not None: + # fargs_str += ", ".join( list(field_args.keys()) ) + fargs_lst += list(field_args.keys()) + if const_args is not None: + # fargs_str += ", ".join( list(const_args.keys()) ) + fargs_lst += list(const_args.keys()) + fargs_str = ", ".join(fargs_lst) + + # Inner loop nest for forward runs + progress_loop = c.Assign("node", "(%s*)(node->_c_next_p)" % ("NodeJIT")) + reset_res_state = c.Assign("res", "particle->state") + update_state = c.Assign("particle->state", "res") + sign_dt = c.Assign("sign_dt", "dt > 0 ? 1 : -1") + particle_backup = c.Statement("%s particle_backup" % self.ptype.name) + sign_end_part = c.Assign("sign_end_part", "endtime - particle->time > 0 ? 1 : -1") + dt_pos = c.Assign("__dt", "fmin(fabs(particle->dt), fabs(endtime - particle->time))") + pdt_eq_dt_pos = c.Assign("__pdt_prekernels", "__dt * sign_dt") + partdt = c.Assign("particle->dt", "__pdt_prekernels") + dt_0_break = c.If("is_zero_dbl(particle->dt)", c.Statement("break")) + notstarted_continue = c.If("(( sign_end_part != sign_dt) || is_close_dbl(__dt, 0) ) && !is_zero_dbl(particle->dt)", + c.Block([ + c.If("fabs(particle->time) >= fabs(endtime)", + c.Assign("particle->state", "SUCCESS")), + progress_loop, + c.Statement("continue") + ])) + body = [c.Statement("set_particle_backup(&particle_backup, particle)")] + body += [pdt_eq_dt_pos] + body += [partdt] + body += [c.Value("ErrorCode", "state_prev"), c.Assign("state_prev", "particle->state")] + body += [c.Assign("res", "%s(particle, %s)" % (funcname, fargs_str))] + body += [c.If("(res==SUCCESS) && (particle->state != state_prev)", c.Assign("res", "particle->state"))] + check_pdt = c.If("(res == SUCCESS) & !is_equal_dbl(__pdt_prekernels, particle->dt)", c.Assign("res", "REPEAT")) + body += [check_pdt] + update_pdt = c.If("_next_dt_set == 1", + c.Block([c.Assign("_next_dt_set", "0"), c.Assign("particle->dt", "_next_dt")])) + body += [c.If("res == SUCCESS || res == DELETE", c.Block([c.Statement("particle->time += particle->dt"), + update_pdt, + dt_pos, + sign_end_part, + c.If("(res != DELETE) && !is_close_dbl(__dt, 0) && (sign_dt == sign_end_part)", + c.Assign("res", "EVALUATE")), + c.If("sign_dt != sign_end_part", c.Assign("__dt", "0")), + update_state, + dt_0_break + ]), + c.Block([c.Statement("get_particle_backup(&particle_backup, particle)"), + dt_pos, + sign_end_part, + c.If("sign_dt != sign_end_part", c.Assign("__dt", "0")), + update_state, + c.Statement("break")]) + )] + + time_loop = c.While("(particle->state == EVALUATE || particle->state == REPEAT) || is_zero_dbl(particle->dt)", c.Block(body)) + node_loop = c.While("node != NULL", c.Block([c.Assign("particle", "(%s*)(node->_c_data_p)" % (self.ptype.name)), + sign_end_part, reset_res_state, dt_pos, notstarted_continue, time_loop, progress_loop])) + + fbody = c.Block([c.Value("int", "sign_dt, sign_end_part"), # p, + c.Value("ErrorCode", "res"), + c.Value("double", "__pdt_prekernels"), + c.Value("double", "__dt"), + # c.Value("double", "__tol"), c.Assign("__tol", "1.e-6"), # 1e-8 = built-in tolerance for np.isclose() + c.Pointer(c.Value("NodeJIT", "node")), c.Assign("node", "node_begin"), + c.Pointer(c.Value(self.ptype.name, "particle")), c.Assign("particle", "(%s*)(node->_c_data_p)" % (self.ptype.name)), + sign_dt, particle_backup, node_loop]) # part_loop + fdecl = c.FunctionDeclaration(c.Value("void", "particle_loop"), args) + ccode += [str(c.FunctionBody(fdecl, fbody))] + return "\n\n".join(ccode) + + +class VectorizedLoopGenerator(object): """Code generator class that adds type definitions and the outer loop around kernel functions to generate compilable C code.""" - def __init__(self, fieldset, ptype=None): + def __init__(self, ptype=None, fieldset=None): self.fieldset = fieldset self.ptype = ptype diff --git a/parcels/wrapping/code_interface.py b/parcels/wrapping/code_interface.py new file mode 100644 index 0000000000..f5855869ac --- /dev/null +++ b/parcels/wrapping/code_interface.py @@ -0,0 +1,190 @@ +import os +import sys +import _ctypes +from time import sleep +import numpy.ctypeslib as npct +from parcels.tools import get_cache_dir, get_package_dir +from .code_compiler import * +# from parcels.tools.loggers import logger + +try: + from mpi4py import MPI +except: + MPI = None + +try: + from mpi4py import MPI +except: + MPI = None + +__all__ = ['LibraryRegisterC', 'InterfaceC'] + + +class LibraryRegisterC: + _data = {} + + def __init__(self): + self._data = {} + + def __del__(self): + for entry in self._data: + while entry.register_count > 0: + sleep(0.1) + entry.unload_library() + del entry + + def load(self, libname, src_dir=get_package_dir()): + if libname is None or (libname in self._data.keys() and self._data[libname].is_loaded()): + return + if libname not in self._data.keys(): + # cppargs = ['-DDOUBLE_COORD_VARIABLES'] if self.lonlatdepth_dtype == np.float64 else None + cppargs = [] + # , libs=["node"] + ccompiler = GNUCompiler(cppargs=cppargs, incdirs=[os.path.join(get_package_dir(), 'include'), os.path.join(get_package_dir(), 'nodes'), "."], libdirs=[".", get_cache_dir()]) + self._data[libname] = InterfaceC("node", ccompiler, src_dir) + if not self._data[libname].is_compiled(): + self._data[libname].compile_library() + if not self._data[libname].is_loaded(): + self._data[libname].load_library() + + def unload(self, libname): + if libname in self._data.keys(): + self._data[libname].unload_library() + # del self._data[libname] + + def is_created(self, libname): + return libname in self._data.keys() + + def is_registered(self, libname): + return self._data[libname].is_registered() + + def is_loaded(self, libname): + return self._data[libname].is_loaded() + + def is_compiled(self, libname): + return self._data[libname].is_compiled() + + def __getitem__(self, item): + return self.get(item) + + def get(self, libname): + #if libname not in self._data.keys(): + # self.load(libname) + if libname in self._data.keys(): + return self._data[libname] + return None + + def register(self, libname): + #if libname not in self._data.keys(): + # self.load(libname) + if libname in self._data.keys(): + self._data[libname].register() + + def deregister(self, libname): + if libname in self._data.keys(): + self._data[libname].unregister() + # if self._data[libname].register_count <= 0: + # self.unload(libname) + +class InterfaceC: + + def __init__(self, c_file_name, compiler, src_dir=get_package_dir()): + basename = c_file_name + src_pathfile = c_file_name + if isinstance(basename, list) and len(basename) > 0: + basename = basename[0] + lib_path = basename + lib_pathfile = os.path.basename(basename) + lib_pathdir = os.path.dirname(basename) + if lib_pathfile[0:3] != "lib": + lib_pathfile = "lib"+lib_pathfile + lib_path = os.path.join(lib_pathdir, lib_pathfile) + if MPI and MPI.COMM_WORLD.Get_size() > 1: + lib_pathfile = "%s_%d" % (lib_pathfile, MPI.COMM_WORLD.Get_rank()) + lib_path = os.path.join(lib_pathdir, lib_pathfile) + if isinstance(src_pathfile, list): + self.src_file = [] + if isinstance(src_dir, list): + for fdir, fname in zip(src_dir, src_pathfile): + self.src_file.append("%s.c" % os.path.join(fdir, fname)) + else: + for fname in src_pathfile: + self.src_file = "%s.c" % os.path.join(src_dir, fname) + else: + self.src_file = "%s.c" % os.path.join(src_dir, src_pathfile) + self.lib_file = "%s.%s" % (os.path.join(get_cache_dir(), lib_path), 'dll' if sys.platform == 'win32' else 'so') + self.log_file = "%s.log" % os.path.join(get_cache_dir(), basename) + if os.path.exists(self.lib_file): + self.compiled = True + + # self.compiler = GNUCompiler() + self.compiler = compiler + self.compiled = False + self.loaded = False + self.libc = None + self.register_count = 0 + + def __del__(self): + self.unload_library() + self.cleanup_files() + + def is_compiled(self): + return self.compiled + + def is_loaded(self): + return self.loaded + + def is_registered(self): + return self.register_count > 0 + + def compile_library(self): + """ Writes kernel code to file and compiles it.""" + if not self.compiled: + self.compiler.compile(self.src_file, self.lib_file, self.log_file) + # logger.info("Compiled %s ==> %s" % (self.name, self.lib_file)) + # self._cleanup_files = finalize(self, package_globals.cleanup_remove_files, self.lib_file, self.log_file) + self.compiled = True + + def cleanup_files(self): + if os.path.isfile(self.lib_file): + [os.remove(s) for s in [self.lib_file, self.log_file] if os._exists(s)] + + def unload_library(self): + if self.libc is not None and self.compiled and self.loaded: + _ctypes.FreeLibrary(self.libc._handle) if sys.platform == 'win32' else _ctypes.dlclose(self.libc._handle) + del self.libc + self.libc = None + self.loaded = False + + def load_library(self): + if self.libc is None and self.compiled and not self.loaded: + self.libc = npct.load_library(self.lib_file, '.') + # self._cleanup_lib = finalize(self, package_globals.cleanup_unload_lib, self.libc) + self.loaded = True + + def register(self): + self.register_count += 1 + # print("lib '{}' register (count: {})".format(self.lib_file, self.register_count)) + + def unregister(self): + self.register_count -= 1 + # print("lib '{}' de-register (count: {})".format(self.lib_file, self.register_count)) + + def load_functions(self, function_param_array=[]): + """ + + :param function_name_array: array of dictionary {"name": str, "return": type, "arguments": [type, ...]} + :return: dict (function_name -> function_handler) + """ + result = dict() + if self.libc is None or not self.compiled or not self.loaded: + return result + for function_param in function_param_array: + if isinstance(function_param, dict) and \ + isinstance(function_param["name"], str) and \ + isinstance(function_param["return"], type) or function_param["return"] is None and \ + isinstance(function_param["arguments"], list): + result[function_param["name"]] = self.libc[function_param["name"]] + result[function_param["name"]].restype = function_param["return"] + result[function_param["name"]].argtypes = function_param["arguments"] + return result diff --git a/performance/benchmark_CMEMS.py b/performance/benchmark_CMEMS.py index 6959eed652..5feeb4a7d7 100644 --- a/performance/benchmark_CMEMS.py +++ b/performance/benchmark_CMEMS.py @@ -5,9 +5,11 @@ from parcels import AdvectionEE, AdvectionRK45, AdvectionRK4_3D from parcels import FieldSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, ErrorCode -from parcels.particleset_benchmark import ParticleSet_Benchmark as ParticleSet +from parcels.particleset_node_benchmark import ParticleSet_Benchmark as ParticleSetN +from parcels.particleset_vectorized_benchmark import ParticleSet_Benchmark as ParticleSetV from parcels import rng as random from parcels.field import VectorField, NestedField, SummedField +from parcels.tools import idgen # from parcels import plotTrajectoriesFile_loadedField from datetime import timedelta as delta from datetime import datetime @@ -74,9 +76,9 @@ def periodicBC(particle, fieldSet, time): particle.lat = min(particle.lat, 90.0) particle.lat = max(particle.lat, -80.0) # if particle.lat > 90.0: - # particle.lat -= 170.0 + # particle.lat -= 160.0 # if particle.lat < -80.0: - # particle.lat += 170.0 + # particle.lat += 160.0 def initialize(particle, fieldset, time): if particle.initialized_dynamic < 1: @@ -118,8 +120,12 @@ def perIterGC(): parser.add_argument("-sN", "--start_n_particles", dest="start_nparticles", type=str, default="96", help="(optional) number of particles generated per release cycle (if --rt is set) (default: 96)") parser.add_argument("-m", "--mode", dest="compute_mode", choices=['jit','scipy'], default="jit", help="computation mode = [JIT, SciPp]") parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + parser.add_argument("--vmode", dest="vmode", action='store_true', default=False, help="use vectorized- instead of node-ParticleSet (default: False)") args = parser.parse_args() + ParticleSet = ParticleSetN + if args.vmode: + ParticleSet = ParticleSetV imageFileName=args.imageFileName periodicFlag=args.periodic backwardSimulation = args.backwards @@ -153,7 +159,7 @@ def perIterGC(): nowtime = datetime.now() random.seed(nowtime.microsecond) - branch = "benchmarking" + branch = "nodes" computer_env = "local/unspecified" scenario = "CMEMS" headdir = "" @@ -185,6 +191,8 @@ def perIterGC(): dirread_top = os.path.join(datahead, 'CMEMS/GLOBAL_REANALYSIS_PHY_001_030/') print("running {} on {} (uname: {}) - branch '{}' - (target) N: {} - argv: {}".format(scenario, computer_env, os.uname()[1], branch, target_N, sys.argv[1:])) + idgen.setTimeLine(0, delta(days=time_in_days).total_seconds()) + if os.path.sep in imageFileName: head_dir = os.path.dirname(imageFileName) if head_dir[0] == os.path.sep: @@ -239,15 +247,37 @@ def perIterGC(): if agingParticles: if repeatdtFlag: pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * 360.0 -180.0, lat=np.random.rand(start_N_particles, 1) * 160.0 - 80.0, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) - psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * 360.0 -180.0, lat=np.random.rand(int(Nparticle/2.0), 1) * 160.0 - 80.0, time=simStart) - pset.add(psetA) + if args.vmode: + # ==== VECTOR VERSION OF THE ADD ==== # + psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * 360.0 -180.0, lat=np.random.rand(int(Nparticle/2.0), 1) * 160.0 - 80.0, time=simStart) + pset.add(psetA) + else: + # ==== NODE VERSION OF THE ADD ==== # + lonlat_field = np.random.rand(int(Nparticle/2.0), 2) + lonlat_field *= np.array([360.0, 160.0]) + lonlat_field[:, 0] -= 180.0 + lonlat_field[:, 1] -= 80.0 + time_field = np.ones((int(Nparticle/2.0), 1), dtype=np.float64) * simStart + pdata = np.concatenate( (lonlat_field, time_field), axis=1 ) + pset.add(pdata) else: pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * 360.0 -180.0, lat=np.random.rand(Nparticle, 1) * 160.0 - 80.0, time=simStart) else: if repeatdtFlag: pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * 360.0 -180.0, lat=np.random.rand(start_N_particles, 1) * 160.0 - 80.0, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) - psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * 360.0 -180.0, lat=np.random.rand(int(Nparticle/2.0), 1) * 160.0 - 80.0, time=simStart) - pset.add(psetA) + if args.vmode: + # ==== VECTOR VERSION OF THE ADD ==== # + psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * 360.0 -180.0, lat=np.random.rand(int(Nparticle/2.0), 1) * 160.0 - 80.0, time=simStart) + pset.add(psetA) + else: + # ==== NODE VERSION OF THE ADD ==== # + lonlat_field = np.random.rand(int(Nparticle/2.0), 2) + lonlat_field *= np.array([360.0, 160.0]) + lonlat_field[:, 0] -= 180.0 + lonlat_field[:, 1] -= 80.0 + time_field = np.ones((int(Nparticle/2.0), 1), dtype=np.float64) * simStart + pdata = np.concatenate( (lonlat_field, time_field), axis=1 ) + pset.add(pdata) else: pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * 360.0 -180.0, lat=np.random.rand(Nparticle, 1) * 160.0 - 80.0, time=simStart) else: @@ -255,15 +285,37 @@ def perIterGC(): if agingParticles: if repeatdtFlag: pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * 360.0 -180.0, lat=np.random.rand(start_N_particles, 1) * 160.0 - 80.0, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) - psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * 360.0 -180.0, lat=np.random.rand(int(Nparticle/2.0), 1) * 160.0 - 80.0, time=simStart) - pset.add(psetA) + if args.vmode: + # ==== VECTOR VERSION OF THE ADD ==== # + psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * 360.0 -180.0, lat=np.random.rand(int(Nparticle/2.0), 1) * 160.0 - 80.0, time=simStart) + pset.add(psetA) + else: + # ==== NODE VERSION OF THE ADD ==== # + lonlat_field = np.random.rand(int(Nparticle/2.0), 2) + lonlat_field *= np.array([360.0, 160.0]) + lonlat_field[:, 0] -= 180.0 + lonlat_field[:, 1] -= 80.0 + time_field = np.ones((int(Nparticle/2.0), 1), dtype=np.float64) * simStart + pdata = np.concatenate( (lonlat_field, time_field), axis=1 ) + pset.add(pdata) else: pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * 360.0 -180.0, lat=np.random.rand(Nparticle, 1) * 160.0 - 80.0, time=simStart) else: if repeatdtFlag: pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * 360.0 -180.0, lat=np.random.rand(start_N_particles, 1) * 160.0 - 80.0, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) - psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * 360.0 -180.0, lat=np.random.rand(int(Nparticle/2.0), 1) * 160.0 - 80.0, time=simStart) - pset.add(psetA) + if args.vmode: + # ==== VECTOR VERSION OF THE ADD ==== # + psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * 360.0 -180.0, lat=np.random.rand(int(Nparticle/2.0), 1) * 160.0 - 80.0, time=simStart) + pset.add(psetA) + else: + # ==== NODE VERSION OF THE ADD ==== # + lonlat_field = np.random.rand(int(Nparticle/2.0), 2) + lonlat_field *= np.array([360.0, 160.0]) + lonlat_field[:, 0] -= 180.0 + lonlat_field[:, 1] -= 80.0 + time_field = np.ones((int(Nparticle/2.0), 1), dtype=np.float64) * simStart + pdata = np.concatenate( (lonlat_field, time_field), axis=1 ) + pset.add(pdata) else: pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * 360.0 -180.0, lat=np.random.rand(Nparticle, 1) * 160.0 - 80.0, time=simStart) @@ -324,10 +376,8 @@ def perIterGC(): mpi_comm = MPI.COMM_WORLD mpi_rank = mpi_comm.Get_rank() if mpi_rank==0: - #endtime = MPI.Wtime() endtime = ostime.process_time() else: - #endtime = ostime.time() endtime = ostime.process_time() if args.write_out: @@ -373,3 +423,7 @@ def perIterGC(): pset.plot_and_log(memory_used=Nmem, nparticles=Nparticles, target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 150]) else: pset.plot_and_log(target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 150]) + # idgen.close() + # del idgen + # print('Execution finished') + exit(0) diff --git a/performance/benchmark_bickleyjet.py b/performance/benchmark_bickleyjet.py index 7d04d36d38..792d886814 100644 --- a/performance/benchmark_bickleyjet.py +++ b/performance/benchmark_bickleyjet.py @@ -5,8 +5,8 @@ from parcels import AdvectionEE, AdvectionRK45, AdvectionRK4 from parcels import FieldSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, ErrorCode -from parcels.particleset_benchmark import ParticleSet_Benchmark as ParticleSet -from parcels.particleset import ParticleSet as DryParticleSet +from parcels.particleset_node_benchmark import ParticleSet_Benchmark as ParticleSetN +from parcels.particleset_vectorized_benchmark import ParticleSet_Benchmark as ParticleSetV from parcels.field import Field, VectorField, NestedField, SummedField # from parcels import plotTrajectoriesFile_loadedField from parcels import rng as random @@ -189,8 +189,12 @@ def Age(particle, fieldset, time): parser.add_argument("-sN", "--start_n_particles", dest="start_nparticles", type=str, default="96", help="(optional) number of particles generated per release cycle (if --rt is set) (default: 96)") parser.add_argument("-m", "--mode", dest="compute_mode", choices=['jit','scipy'], default="jit", help="computation mode = [JIT, SciPp]") parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + parser.add_argument("--vmode", dest="vmode", action='store_true', default=False, help="use vectorized- instead of node-ParticleSet (default: False)") args = parser.parse_args() + ParticleSet = ParticleSetN + if args.vmode: + ParticleSet = ParticleSetV imageFileName=args.imageFileName periodicFlag=args.periodic backwardSimulation = args.backwards @@ -230,7 +234,7 @@ def Age(particle, fieldset, time): nowtime = datetime.datetime.now() random.seed(nowtime.microsecond) - branch = "benchmarking" + branch = "nodes" computer_env = "local/unspecified" scenario = "bickleyjet" odir = "" @@ -340,6 +344,8 @@ def Age(particle, fieldset, time): out_fname += "_MPI" else: out_fname += "_noMPI" + if periodicFlag: + out_fname += "_p" out_fname += "_n"+str(Nparticle) if backwardSimulation: out_fname += "_bwd" diff --git a/performance/benchmark_deep_migration_NPacific.py b/performance/benchmark_deep_migration_NPacific.py index 904f2a7c07..e50446739a 100644 --- a/performance/benchmark_deep_migration_NPacific.py +++ b/performance/benchmark_deep_migration_NPacific.py @@ -1,21 +1,24 @@ -from parcels import FieldSet, JITParticle, ScipyParticle, AdvectionRK4_3D, AdvectionRK4, ErrorCode, ParticleFile, Variable, Field, NestedField, VectorField, timer -from parcels import ParticleSet_Benchmark +from parcels import FieldSet, JITParticle, ScipyParticle, AdvectionRK4_3D, AdvectionRK4, ErrorCode, Variable, Field, NestedField, VectorField, timer +from parcels.particleset_node_benchmark import ParticleSet_Benchmark +from parcels.particlefile_node import ParticleFile +# from parcels.particleset_vectorized_benchmark import ParticleSet_Benchmark +# from parcels.particlefile_vectorized import ParticleFile +from parcels.tools import idgen, GenerateID_Service, SpatioTemporalIdGenerator, SequentialIdGenerator from parcels.kernels import seawaterdensity from argparse import ArgumentParser from datetime import timedelta as delta -from datetime import datetime +# from datetime import datetime import time as ostime -import numpy as np +import os +import sys import math +import numpy as np +import scipy.linalg +# from numpy import * from glob import glob -import matplotlib.pyplot as plt import fnmatch import warnings -import pickle -import matplotlib.ticker as mtick -from numpy import * -import scipy.linalg -import math as math +import pickle import gc warnings.filterwarnings("ignore") @@ -38,14 +41,16 @@ lon_release = np.tile(np.linspace(-165,-156,10),[10,1]) z_release = np.tile(1,[10,10]) +time0 = 0 # Choose: simdays = 50.0 * 365.0 #simdays = 5 -time0 = 0 simhours = 1 simmins = 30 secsdt = 30 -hrsoutdt = 5 +# hrsoutdt = 5 # this is the original that should be used +hrsoutdt = 24 +hrscbdt = 12 # callback-dt (if callback is garbage collection) should be less than the output #--------- Choose below: NOTE- MUST ALSO MANUALLY CHANGE IT IN THE KOOI KERNAL BELOW ----- rho_pl = 920. # density of plastic (kg m-3): DEFAULT FOR FIG 1 in Kooi: 920 but full range is: 840, 920, 940, 1050, 1380 (last 2 are initially non-buoyant) @@ -189,7 +194,7 @@ def DeleteParticle(particle, fieldset, time): def perIterGC(): gc.collect() -def getclosest_ij(lats,lons,latpt,lonpt): +def getclosest_ij(lats,lons,latpt,lonpt): """Function to find the index of the closest point to a certain lon/lat value.""" dist_sq = (lats-latpt)**2 + (lons-lonpt)**2 # find squared distance of every point on grid minindex_flattened = dist_sq.argmin() # 1D index of minimum dist_sq element @@ -227,6 +232,7 @@ def Profiles(particle, fieldset, time): particle.sw_visc = fieldset.SV[time,particle.depth,particle.lat,particle.lon] particle.w = fieldset.W[time,particle.depth,particle.lat,particle.lon] +# test with t={1, 2, 4, 7, 11, 16, 22, 29, 37, 46, 56} if __name__ == "__main__": parser = ArgumentParser(description="Example of particle advection using in-memory stommel test case") parser.add_argument("-i", "--imageFileName", dest="imageFileName", type=str, default="mpiChunking_plot_MPI.png", help="image file name of the plot") @@ -234,6 +240,7 @@ def Profiles(particle, fieldset, time): # parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=int, default=365, help="runtime in days (default: 365)") parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=str, default="1*365", help="runtime in days (default: 1*365)") parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + parser.add_argument("-w", "--writeout", dest="write_out", action='store_false', default=True, help="write data in outfile (default: true)") args = parser.parse_args() imageFileName=args.imageFileName @@ -241,7 +248,11 @@ def Profiles(particle, fieldset, time): time_in_days = int(float(eval(args.time_in_days))) with_GC = args.useGC - branch = "benchmarking" + global idgen + idgen = GenerateID_Service(SequentialIdGenerator) + idgen.setTimeLine(0, delta(days=time_in_days).total_seconds()) + + branch = "nodes" computer_env = "local/unspecified" scenario = "deep_migration" headdir = "" @@ -259,15 +270,14 @@ def Profiles(particle, fieldset, time): dirread_bgc = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/means/') dirread_mesh = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/domain/') computer_env = "Gemini" - # elif fnmatch.fnmatchcase(os.uname()[1], "int?.*"): # Cartesius elif fnmatch.fnmatchcase(os.uname()[1], "*.bullx*"): # Cartesius CARTESIUS_SCRATCH_USERNAME = 'ckehluu' headdir = "/scratch/shared/{}/experiments/deep_migration_behaviour".format(CARTESIUS_SCRATCH_USERNAME) - odir = os.path.join(headdir, "/BENCHres") + odir = os.path.join(headdir, "BENCHres") datahead = "/projects/0/topios/hydrodynamic_data" dirread = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/means/') dirread_bgc = os.path.join(datahead, 'NEMO-MEDUSA_BGC/ORCA0083-N006/means/') - dirread_mesh = os.path.join(datahead, 'NEMO-MEDUSA_BGC/ORCA0083-N006/domain/') + dirread_mesh = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/domain/') computer_env = "Cartesius" else: headdir = "/var/scratch/dlobelle" @@ -320,7 +330,6 @@ def Profiles(particle, fieldset, time): 'cons_temperature': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': tsfiles}, 'abs_salinity': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': wfiles[0], 'data': tsfiles}} - variables = {'U': 'uo', 'V': 'vo', 'W': 'wo', @@ -345,10 +354,34 @@ def Profiles(particle, fieldset, time): 'cons_temperature': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw','time': 'time_counter'}, 'abs_salinity': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthw','time': 'time_counter'}} - chs = {'time_counter': 1, 'depthu': 75, 'depthv': 75, 'depthw': 75, 'deptht': 75, 'y': 200, 'x': 200} - fieldset = FieldSet.from_nemo(filenames, variables, dimensions, allow_time_extrapolation=False, field_chunksize=chs, time_periodic=delta(days=365)) + chs = {'time_counter': 1, 'depthu': 25, 'depthv': 25, 'depthw': 25, 'deptht': 25, 'y': 64, 'x': 64} + nchs = { + 'U': {'lon': ('x', 64), 'lat': ('y', 48), 'depth': ('depthu', 25), 'time': ('time_counter', 1)}, + 'V': {'lon': ('x', 64), 'lat': ('y', 48), 'depth': ('depthv', 25), 'time': ('time_counter', 1)}, + 'W': {'lon': ('x', 64), 'lat': ('y', 48), 'depth': ('depthw', 25), 'time': ('time_counter', 1)}, + 'd_phy': {'lon': ('x', 64), 'lat': ('y', 48), 'depth': ('deptht', 25), 'time': ('time_counter', 1)}, # pfiles + 'nd_phy': {'lon': ('x', 64), 'lat': ('y', 48), 'depth': ('deptht', 25), 'time': ('time_counter', 1)}, # pfiles + 'euph_z': {'lon': ('x', 64), 'lat': ('y', 48), 'depth': ('deptht', 25), 'time': ('time_counter', 1)}, # dfiles + # 'd_tpp': {'lon': ('x', 96), 'lat': ('y', 48), 'time': ('time_counter', 1)}, # dfiles + # 'nd_tpp': {'lon': ('x', 96), 'lat': ('y', 48), 'time': ('time_counter', 1)}, # dfiles + 'tpp3': {'lon': ('x', 64), 'lat': ('y', 48), 'depth': ('deptht', 25), 'time': ('time_counter', 1)}, # dfiles + 'cons_temperature': {'lon': ('x', 64), 'lat': ('y', 48), 'depth': ('deptht', 25), 'time': ('time_counter', 1)}, # tfiles + 'abs_salinity': {'lon': ('x', 64), 'lat': ('y', 48), 'depth': ('deptht', 25), 'time': ('time_counter', 1)}, # tfiles + } + if periodicFlag: + try: + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, allow_time_extrapolation=False, field_chunksize=chs, time_periodic=delta(days=366)) + except (SyntaxError, ): + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, allow_time_extrapolation=False, chunksize=nchs, time_periodic=delta(days=366)) + else: + try: + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, allow_time_extrapolation=True, field_chunksize=chs) + except (SyntaxError, ): + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, allow_time_extrapolation=True, chunksize=nchs) depths = fieldset.U.depth + idgen.setDepthLimits(np.min(depths), np.max(depths)) + outfile = 'Kooi+NEMO_3D_grid10by10_rho'+str(int(rho_pl))+'_r'+ r_pl+'_'+str(simdays)+'days_'+str(secsdt)+'dtsecs_'+str(hrsoutdt)+'hrsoutdt' dirwrite = os.path.join(odir, "rho_"+str(int(rho_pl))+"kgm-3") if not os.path.exists(dirwrite): @@ -359,37 +392,48 @@ def Profiles(particle, fieldset, time): # profile_auxin_path = '/scratch/ckehl/experiments/deep_migration_behaviour/aux_in/profiles.pickle' profile_auxin_path = os.path.join(headdir, 'aux_in/profiles.pickle') with open(profile_auxin_path, 'rb') as f: - depth,T_z,S_z,rho_z,upsilon_z,mu_z = pickle.load(f) + depth,T_z,S_z,rho_z,epsilon_z,mu_z = pickle.load(f) v_lon = np.array([minlon,maxlon]) v_lat = np.array([minlat,maxlat]) - kv_or = np.transpose(np.tile(np.array(upsilon_z),(len(v_lon),len(v_lat),1)), (2,0,1)) # kinematic viscosity + print("|lon| = {}; |lat| = {}".format(lon_release.shape[0], lat_release.shape[0])) + + kv_or = np.transpose(np.tile(np.array(epsilon_z),(len(v_lon),len(v_lat),1)), (2,0,1)) # kinematic viscosity sv_or = np.transpose(np.tile(np.array(mu_z),(len(v_lon),len(v_lat),1)), (2,0,1)) # dynamic viscosity of seawater - KV = Field('KV', kv_or, lon=v_lon, lat=v_lat, depth=depths, mesh='spherical', field_chunksize=False)#,transpose="True") #,fieldtype='U') - SV = Field('SV', sv_or, lon=v_lon, lat=v_lat, depth=depths, mesh='spherical', field_chunksize=False)#,transpose="True") #,fieldtype='U') + try: + KV = Field('KV', kv_or, lon=v_lon, lat=v_lat, depth=depths, mesh='spherical', field_chunksize=False)#,transpose="True") #,fieldtype='U') + SV = Field('SV', sv_or, lon=v_lon, lat=v_lat, depth=depths, mesh='spherical', field_chunksize=False)#,transpose="True") #,fieldtype='U') + except (SyntaxError, ): + KV = Field('KV', kv_or, lon=v_lon, lat=v_lat, depth=depths, mesh='spherical', chunksize=False)#,transpose="True") #,fieldtype='U') + SV = Field('SV', sv_or, lon=v_lon, lat=v_lat, depth=depths, mesh='spherical', chunksize=False)#,transpose="True") #,fieldtype='U') fieldset.add_field(KV, 'KV') fieldset.add_field(SV, 'SV') """ Defining the particle set """ - pset = ParticleSet_Benchmark.from_list(fieldset=fieldset, # the fields on which the particles are advected - pclass=plastic_particle, # the type of particles (JITParticle or ScipyParticle) - lon= lon_release, #-160., # a vector of release longitudes - lat= lat_release, #36., + pset = ParticleSet_Benchmark.from_list(fieldset=fieldset, # the fields on which the particles are advected + pclass=plastic_particle, # the type of particles (JITParticle or ScipyParticle) + lon= lon_release, # a vector of release longitudes + lat= lat_release, time = time0, - depth = z_release) #[1.] + depth = z_release) # perflog = PerformanceLog() # perflog.pset = pset - #postProcessFuncs = [perflog.advance,] + # postProcessFuncs = [perflog.advance,] """ Kernal + Execution""" postProcessFuncs = [] if with_GC: postProcessFuncs.append(perIterGC) + output_fpath = None + if args.write_out: + output_fpath = os.path.join(dirwrite, outfile) # kernels = pset.Kernel(AdvectionRK4_3D) + pset.Kernel(seawaterdensity.polyTEOS10_bsq) + pset.Kernel(Profiles) + pset.Kernel(Kooi) kernels = pset.Kernel(AdvectionRK4_3D_vert) + pset.Kernel(seawaterdensity.polyTEOS10_bsq) + pset.Kernel(Profiles) + pset.Kernel(Kooi) - pfile= ParticleFile(os.path.join(dirwrite, outfile), pset, outputdt=delta(hours = hrsoutdt)) + pfile = None + if args.write_out: + pfile= ParticleFile(output_fpath, pset, outputdt=delta(hours = hrsoutdt)) starttime = 0 endtime = 0 @@ -405,7 +449,7 @@ def Profiles(particle, fieldset, time): starttime = ostime.process_time() # postIterationCallbacks = postProcessFuncs, callbackdt = delta(hours=hrsoutdt) - pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(seconds = secsdt), output_file=pfile, verbose_progress=True, recovery={ErrorCode.ErrorOutOfBounds: DeleteParticle}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours = hrsoutdt)) + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(seconds = secsdt), output_file=pfile, verbose_progress=True, recovery={ErrorCode.ErrorOutOfBounds: DeleteParticle}, postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours = hrscbdt)) if MPI: mpi_comm = MPI.COMM_WORLD @@ -418,11 +462,14 @@ def Profiles(particle, fieldset, time): # endtime = ostime.time() endtime = ostime.process_time() + if args.write_out: + pfile.close() + + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart - 1) if MPI: mpi_comm = MPI.COMM_WORLD mpi_comm.Barrier() - size_Npart = len(pset.nparticle_log) - Npart = pset.nparticle_log.get_param(size_Npart - 1) Npart = mpi_comm.reduce(Npart, op=MPI.SUM, root=0) if mpi_comm.Get_rank() == 0: if size_Npart>0: @@ -431,16 +478,12 @@ def Profiles(particle, fieldset, time): avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time*1000.0)) else: - size_Npart = len(pset.nparticle_log) - Npart = pset.nparticle_log.get_param(size_Npart-1) if size_Npart > 0: sys.stdout.write("final # particles: {}\n".format( Npart )) sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime - starttime)) avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time * 1000.0)) - pfile.close() - if MPI: mpi_comm = MPI.COMM_WORLD # mpi_comm.Barrier() @@ -452,3 +495,4 @@ def Profiles(particle, fieldset, time): pset.plot_and_log(target_N=1, imageFilePath=imageFileName, odir=odir) print('Execution finished') + exit(0) diff --git a/performance/benchmark_doublegyre.py b/performance/benchmark_doublegyre.py index 8733e6a02f..ed74ca55f0 100644 --- a/performance/benchmark_doublegyre.py +++ b/performance/benchmark_doublegyre.py @@ -5,8 +5,8 @@ from parcels import AdvectionEE, AdvectionRK45, AdvectionRK4 from parcels import FieldSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, ErrorCode -from parcels.particleset_benchmark import ParticleSet_Benchmark as ParticleSet -from parcels.particleset import ParticleSet as DryParticleSet +from parcels.particleset_node_benchmark import ParticleSet_Benchmark as ParticleSetN +from parcels.particleset_vectorized_benchmark import ParticleSet_Benchmark as ParticleSetV from parcels.field import Field, VectorField, NestedField, SummedField # from parcels import plotTrajectoriesFile_loadedField from parcels import rng as random @@ -183,8 +183,12 @@ def Age(particle, fieldset, time): parser.add_argument("-sN", "--start_n_particles", dest="start_nparticles", type=str, default="96", help="(optional) number of particles generated per release cycle (if --rt is set) (default: 96)") parser.add_argument("-m", "--mode", dest="compute_mode", choices=['jit','scipy'], default="jit", help="computation mode = [JIT, SciPp]") parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + parser.add_argument("--vmode", dest="vmode", action='store_true', default=False, help="use vectorized- instead of node-ParticleSet (default: False)") args = parser.parse_args() + ParticleSet = ParticleSetN + if args.vmode: + ParticleSet = ParticleSetV imageFileName=args.imageFileName periodicFlag=args.periodic backwardSimulation = args.backwards @@ -224,7 +228,7 @@ def Age(particle, fieldset, time): nowtime = datetime.datetime.now() random.seed(nowtime.microsecond) - branch = "benchmarking" + branch = "nodes" computer_env = "local/unspecified" scenario = "doublegyre" odir = "" @@ -335,6 +339,8 @@ def Age(particle, fieldset, time): out_fname += "_MPI" else: out_fname += "_noMPI" + if periodicFlag: + out_fname += "_p" out_fname += "_n"+str(Nparticle) if backwardSimulation: out_fname += "_bwd" diff --git a/performance/benchmark_galapagos_backwards.py b/performance/benchmark_galapagos_backwards.py index c74f0b5d89..dbce831621 100644 --- a/performance/benchmark_galapagos_backwards.py +++ b/performance/benchmark_galapagos_backwards.py @@ -1,5 +1,7 @@ from parcels import FieldSet, JITParticle, AdvectionRK4, ErrorCode, Variable -from parcels import ParticleSet_Benchmark +# from parcels.particleset_node_benchmark import ParticleSet_Benchmark +from parcels.particleset_vectorized_benchmark import ParticleSet_Benchmark +from parcels.tools import idgen from datetime import timedelta as delta from glob import glob import numpy as np @@ -33,19 +35,35 @@ def create_galapagos_fieldset(datahead, periodic_wrap, use_stokes): 'V': {'lon': meshfile, 'lat': meshfile, 'data': vfiles}} nemo_variables = {'U': 'uo', 'V': 'vo'} nemo_dimensions = {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'} - period = delta(days=366) if periodic_wrap else False + period = delta(days=366*11) if periodic_wrap else False # 10 years period extrapolation = False if periodic_wrap else True # ==== Because the stokes data is a different grid, we actually need to define the chunking ==== # # fieldset_nemo = FieldSet.from_nemo(nemofiles, nemovariables, nemodimensions, field_chunksize='auto') nemo_chs = {'time_counter': 1, 'depthu': 75, 'depthv': 75, 'depthw': 75, 'deptht': 75, 'y': 100, 'x': 100} - fieldset_nemo = FieldSet.from_nemo(nemo_files, nemo_variables, nemo_dimensions, field_chunksize=nemo_chs, time_periodic=period, allow_time_extrapolation=extrapolation) + nemo_nchs = { + 'U': {'lon': ('x', 64), 'lat': ('y', 32), 'depth': ('depthu', 25), 'time': ('time_counter', 1)}, # + 'V': {'lon': ('x', 64), 'lat': ('y', 32), 'depth': ('depthv', 25), 'time': ('time_counter', 1)}, # + } + try: + fieldset_nemo = FieldSet.from_nemo(nemo_files, nemo_variables, nemo_dimensions, field_chunksize=nemo_chs, time_periodic=period, allow_time_extrapolation=extrapolation) + except (SyntaxError, ): + fieldset_nemo = FieldSet.from_nemo(nemo_files, nemo_variables, nemo_dimensions, chunksize=nemo_nchs, time_periodic=period, allow_time_extrapolation=extrapolation) if wstokes: stokes_files = sorted(glob(datahead+"/WaveWatch3data/CFSR/WW3-*_uss.nc")) stokes_variables = {'U': 'uuss', 'V': 'vuss'} stokes_dimensions = {'lat': 'latitude', 'lon': 'longitude', 'time': 'time'} stokes_chs = {'time': 1, 'latitude': 16, 'longitude': 32} - fieldset_stokes = FieldSet.from_netcdf(stokes_files, stokes_variables, stokes_dimensions, field_chunksize=stokes_chs, time_periodic=period, allow_time_extrapolation=extrapolation) + stokes_nchs = { + 'U': {'lon': ('longitude', 32), 'lat': ('latitude', 16), 'time': ('time', 1)}, + 'V': {'lon': ('longitude', 32), 'lat': ('latitude', 16), 'time': ('time', 1)} + } + stokes_period = delta(days=366+2*31) if periodic_wrap else False # 14 month period + fieldset_stokes = None + try: + fieldset_stokes = FieldSet.from_netcdf(stokes_files, stokes_variables, stokes_dimensions, field_chunksize=stokes_chs, time_periodic=stokes_period, allow_time_extrapolation=extrapolation) + except (SyntaxError, ): + fieldset_stokes = FieldSet.from_netcdf(stokes_files, stokes_variables, stokes_dimensions, chunksize=stokes_nchs, time_periodic=stokes_period, allow_time_extrapolation=extrapolation) fieldset_stokes.add_periodic_halo(zonal=True, meridional=False, halosize=5) fieldset = FieldSet(U=fieldset_nemo.U+fieldset_stokes.U, V=fieldset_nemo.V+fieldset_stokes.V) @@ -76,6 +94,12 @@ def Age(particle, fieldset, time): def DeleteParticle(particle, fieldset, time): particle.delete() +def RenewParticle(particle, fieldset, time): + dlon = -89.0 + 91.8 + dlat = 0.7 + 1.4 + particle.lat = np.random.rand() * dlon -91.8 + particle.lon = np.random.rand() * dlat -1.4 + def periodicBC(particle, fieldSet, time): dlon = -89.0 + 91.8 @@ -98,6 +122,7 @@ def periodicBC(particle, fieldSet, time): # parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=int, default=365, help="runtime in days (default: 365)") parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=str, default="1*365", help="runtime in days (default: 1*365)") parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + parser.add_argument("-w", "--writeout", dest="write_out", action='store_false', default=True, help="write data in outfile (default: true)") args = parser.parse_args() wstokes = args.stokes @@ -106,7 +131,9 @@ def periodicBC(particle, fieldSet, time): time_in_days = int(float(eval(args.time_in_days))) with_GC = args.useGC - branch = "benchmarking" + idgen.setTimeLine(0, delta(days=time_in_days).total_seconds()) + + branch = "nodes" computer_env = "local/unspecified" scenario = "galapagos_backwards" headdir = "" @@ -126,7 +153,7 @@ def periodicBC(particle, fieldSet, time): elif fnmatch.fnmatchcase(os.uname()[1], "*.bullx*"): # Cartesius CARTESIUS_SCRATCH_USERNAME = 'ckehluu' headdir = "/scratch/shared/{}/experiments/galapagos".format(CARTESIUS_SCRATCH_USERNAME) - odir = os.path.join(headdir, "/BENCHres") + odir = os.path.join(headdir, "BENCHres") datahead = "/projects/0/topios/hydrodynamic_data" ddir_head = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/') computer_env = "Cartesius" @@ -135,6 +162,7 @@ def periodicBC(particle, fieldSet, time): odir = os.path.join(headdir, "BENCHres") datahead = "/data" ddir_head = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/') + print("running {} on {} (uname: {}) - branch '{}' - argv: {}".format(scenario, computer_env, os.uname()[1], branch, sys.argv[1:])) @@ -146,14 +174,18 @@ def periodicBC(particle, fieldSet, time): startlon, startlat = np.meshgrid(np.arange(galapagos_extent[0], galapagos_extent[1], 0.2), np.arange(galapagos_extent[2], galapagos_extent[3], 0.2)) - print("running {} on {} (uname: {}) - branch '{}' - (target) N: {} - argv: {}".format(scenario, computer_env, os.uname()[1], branch, startlon.shape[0], sys.argv[1:])) + print("|lon| = {}; |lat| = {}".format(startlon.shape[0], startlat.shape[0])) pset = ParticleSet_Benchmark(fieldset=fieldset, pclass=GalapagosParticle, lon=startlon, lat=startlat, time=fU.grid.time[-1], repeatdt=delta(days=7)) """ Kernal + Execution""" postProcessFuncs = [] if with_GC: postProcessFuncs.append(perIterGC) - outfile = pset.ParticleFile(name=fname, outputdt=delta(days=1)) + output_fpath = None + outfile = None + if args.write_out: + output_fpath = fname + outfile = pset.ParticleFile(name=output_fpath, outputdt=delta(days=1)) kernel = pset.Kernel(AdvectionRK4)+pset.Kernel(Age)+pset.Kernel(periodicBC) starttime = 0 @@ -182,11 +214,14 @@ def periodicBC(particle, fieldSet, time): # endtime = ostime.time() endtime = ostime.process_time() + if args.write_out: + outfile.close() + + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart - 1) if MPI: mpi_comm = MPI.COMM_WORLD mpi_comm.Barrier() - size_Npart = len(pset.nparticle_log) - Npart = pset.nparticle_log.get_param(size_Npart - 1) Npart = mpi_comm.reduce(Npart, op=MPI.SUM, root=0) if mpi_comm.Get_rank() == 0: if size_Npart>0: @@ -195,16 +230,12 @@ def periodicBC(particle, fieldSet, time): avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time*1000.0)) else: - size_Npart = len(pset.nparticle_log) - Npart = pset.nparticle_log.get_param(size_Npart-1) if size_Npart > 0: sys.stdout.write("final # particles: {}\n".format( Npart )) sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime - starttime)) avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time * 1000.0)) - outfile.close() - if MPI: mpi_comm = MPI.COMM_WORLD # mpi_comm.Barrier() @@ -214,3 +245,6 @@ def periodicBC(particle, fieldSet, time): pset.plot_and_log(memory_used=Nmem, nparticles=Nparticles, target_N=1, imageFilePath=imageFileName, odir=odir) else: pset.plot_and_log(target_N=1, imageFilePath=imageFileName, odir=odir) + + print('Execution finished') + exit(0) diff --git a/performance/benchmark_palaeo_Y2K.py b/performance/benchmark_palaeo_Y2K.py index e71b030a78..e2376c2a0f 100644 --- a/performance/benchmark_palaeo_Y2K.py +++ b/performance/benchmark_palaeo_Y2K.py @@ -9,8 +9,10 @@ from parcels import (FieldSet, JITParticle, AdvectionRK4_3D, Field, ErrorCode, ParticleFile, Variable) # from parcels import ParticleSet -from parcels import ParticleSet_Benchmark - +# from parcels.particleset_vectorized_benchmark import ParticleSet_Benchmark +from parcels.particleset_node_benchmark import ParticleSet_Benchmark +from parcels.tools import idgen +from parcels.tools import GenerateID_Service, SpatioTemporalIdGenerator, SequentialIdGenerator from argparse import ArgumentParser from datetime import timedelta as delta from datetime import datetime @@ -41,7 +43,7 @@ odir = "" -def set_nemo_fieldset(ufiles, vfiles, wfiles, tfiles, pfiles, dfiles, ifiles, bfile, mesh_mask='/scratch/ckehl/experiments/palaeo-parcels/NEMOdata/domain/coordinates.nc'): +def set_nemo_fieldset(ufiles, vfiles, wfiles, tfiles, pfiles, dfiles, ifiles, bfile, mesh_mask='/scratch/ckehl/experiments/palaeo-parcels/NEMOdata/domain/coordinates.nc', periodicFlag=False): filenames = { 'U': {'lon': mesh_mask, 'lat': mesh_mask, 'depth': [ufiles[0]], @@ -56,29 +58,24 @@ def set_nemo_fieldset(ufiles, vfiles, wfiles, tfiles, pfiles, dfiles, ifiles, bf 'data':wfiles}, 'S' : {'lon': mesh_mask, 'lat': mesh_mask, - 'depth': [tfiles[0]], 'data':tfiles}, 'T' : {'lon': mesh_mask, 'lat': mesh_mask, - 'depth': [tfiles[0]], 'data':tfiles}, 'NO3':{'lon': mesh_mask, 'lat': mesh_mask, 'depth': [pfiles[0]], 'data':pfiles}, - 'PP':{'lon': mesh_mask, - 'lat': mesh_mask, - 'depth': [dfiles[0]], - 'data':dfiles}, 'ICE':{'lon': mesh_mask, 'lat': mesh_mask, - 'depth': [ifiles[0]], 'data':ifiles}, 'ICEPRES':{'lon': mesh_mask, - 'lat': mesh_mask, - 'depth': [ifiles[0]], - 'data':ifiles}, + 'lat': mesh_mask, + 'data':ifiles}, 'CO2':{'lon': mesh_mask, + 'lat': mesh_mask, + 'data':dfiles}, + 'PP':{'lon': mesh_mask, 'lat': mesh_mask, 'depth': [dfiles[0]], 'data':dfiles}, @@ -101,27 +98,68 @@ def set_nemo_fieldset(ufiles, vfiles, wfiles, tfiles, pfiles, dfiles, ifiles, bf 'W': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'depthu', 'time': 'time_counter'}, # 'T': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, 'S': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, - 'NO3': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, - 'PP': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, + 'NO3': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'deptht', 'time': 'time_counter'}, 'ICE': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, 'ICEPRES': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, - 'CO2': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'} } #, + 'CO2': {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'}, + 'PP': {'lon': 'glamf', 'lat': 'gphif', 'depth': 'deptht', 'time': 'time_counter'}, + } bfiles = {'lon': mesh_mask, 'lat': mesh_mask, 'data': [bfile, ]} bvariables = ('B', 'Bathymetry') bdimensions = {'lon': 'glamf', 'lat': 'gphif'} bchs = False - chs = {'time_counter': 1, 'depthu': 75, 'depthv': 75, 'depthw': 75, 'deptht': 75, 'y': 200, 'x': 200} - # - #chs = (1, 75, 200, 200) - # - #dask.config.set({'array.chunk-size': '6MiB'}) - #chs = 'auto' + # ==== depth-split need to be 75 so all (2D- and 3D) fields are chunked the same way + chs = {'time_counter': 1, 'depthu': 80, 'depthv': 80, 'depthw': 80, 'deptht': 80, 'y': 200, 'x': 200} + # chs = {'U': {'depthu': 80, 'depthv': 80, 'depthw': 80, 'deptht': 80, 'y': 64, 'x': 128, 'time_counter': 1}, + # 'V': {'depthu': 80, 'depthv': 80, 'depthw': 80, 'deptht': 80, 'y': 64, 'x': 128, 'time_counter': 1}, + # 'W': {'depthu': 80, 'depthv': 80, 'depthw': 80, 'deptht': 80, 'y': 64, 'x': 128, 'time_counter': 1}, + # 'T': {'x': 128, 'y': 64, 'deptht': 80, 'time_counter': 1}, # tfiles + # 'S': {'x': 128, 'y': 64, 'deptht': 80, 'time_counter': 1}, # tfiles + # 'NO3': {'x': 128, 'y': 64, 'deptht': 80, 'time_counter': 1}, # pfiles + # 'PP': {'x': 128, 'y': 64, 'deptht': 80, 'time_counter': 1}, # dfiles + # 'ICE': False, # ifiles + # 'ICEPRES': False, # ifiles + # 'CO2': {'x': 128, 'y': 64, 'deptht': 80, 'time_counter': 1}, # dfiles + # } + + # dask.config.set({'array.chunk-size': '6MiB'}) + # chs = 'auto' + nchs = { + 'U': {'lon': ('x', 128), 'lat': ('y', 96), 'depth': ('depthu', 25), 'time': ('time_counter', 1)}, # ufiles + 'V': {'lon': ('x', 128), 'lat': ('y', 96), 'depth': ('depthv', 25), 'time': ('time_counter', 1)}, # vfiles + 'W': {'lon': ('x', 128), 'lat': ('y', 96), 'depth': ('depthw', 25), 'time': ('time_counter', 1)}, # wfiles + 'T': {'lon': ('x', 128), 'lat': ('y', 96), 'time': ('time_counter', 1)}, # tfiles + 'S': {'lon': ('x', 128), 'lat': ('y', 96), 'time': ('time_counter', 1)}, # tfiles + 'NO3': {'lon': ('x', 128), 'lat': ('y', 96), 'depth': ('deptht', 25), 'time': ('time_counter', 1)}, # pfiles + 'ICE': {'lon': ('x', 128), 'lat': ('y', 96), 'time': ('time_counter', 1)}, # ifiles + 'ICEPRES': {'lon': ('x', 128), 'lat': ('y', 96), 'time': ('time_counter', 1)}, # ifiles + 'CO2': {'lon': ('x', 128), 'lat': ('y', 96), 'time': ('time_counter', 1)}, # dfiles + 'PP': {'lon': ('x', 128), 'lat': ('y', 96), 'depth': ('deptht', 25), 'time': ('time_counter', 1)}, # dfiles + } if mesh_mask: # and isinstance(bfile, list) and len(bfile) > 0: - # fieldset = FieldSet.from_nemo(filenames, variables, dimensions, allow_time_extrapolation=False, field_chunksize='auto') - fieldset = FieldSet.from_nemo(filenames, variables, dimensions, allow_time_extrapolation=True, field_chunksize=chs) - Bfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + if not periodicFlag: + try: + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, allow_time_extrapolation=True, field_chunksize=chs) + # Tfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + # Sfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + # NO3field = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + # PPfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + # ICEfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + # ICEPRESfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + # CO2field = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + Bfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + except (SyntaxError, ): + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, allow_time_extrapolation=True, chunksize=nchs) + Bfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', chunksize=bchs) + else: + try: + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, time_periodic=delta(days=366), field_chunksize=chs) + Bfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', field_chunksize=bchs) + except (SyntaxError, ): + fieldset = FieldSet.from_nemo(filenames, variables, dimensions, time_periodic=delta(days=366), chunksize=nchs) + Bfield = Field.from_netcdf(bfiles, bvariables, bdimensions, allow_time_extrapolation=True, interp_method='cgrid_tracer', chunksize=bchs) fieldset.add_field(Bfield, 'B') fieldset.U.vmax = 10 fieldset.V.vmax = 10 @@ -131,8 +169,17 @@ def set_nemo_fieldset(ufiles, vfiles, wfiles, tfiles, pfiles, dfiles, ifiles, bf filenames.pop('B') variables.pop('B') dimensions.pop('B') - # fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, allow_time_extrapolation=False, field_chunksize=chs) - fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, allow_time_extrapolation=True, field_chunksize=chs) + if not periodicFlag: + try: + # fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, allow_time_extrapolation=False, field_chunksize=chs) + fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, allow_time_extrapolation=True, field_chunksize=chs) + except (SyntaxError, ): + fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, allow_time_extrapolation=True, chunksize=nchs) + else: + try: + fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, time_periodic=delta(days=366), field_chunksize=chs) + except (SyntaxError, ): + fieldset = FieldSet.from_netcdf(filenames, variables, dimensions, time_periodic=delta(days=366), chunksize=nchs) fieldset.U.vmax = 10 fieldset.V.vmax = 10 fieldset.W.vmax = 10 @@ -162,6 +209,8 @@ def Sink(particle, fieldset, time): def Age(particle, fieldset, time): if particle.state == ErrorCode.Evaluate: particle.age = particle.age + math.fabs(particle.dt) + # if particle.age > fieldset.maxage: + # particle.delete() def DeleteParticle(particle, fieldset, time): particle.delete() @@ -202,6 +251,8 @@ class DinoParticle(JITParticle): # parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=int, default=365, help="runtime in days (default: 365)") parser.add_argument("-t", "--time_in_days", dest="time_in_days", type=str, default="1*365", help="runtime in days (default: 1*365)") parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + parser.add_argument("-pr", "--profiling", dest="profiling", action='store_true', default=False, help="tells that the profiling of the script is activates") + parser.add_argument("-w", "--writeout", dest="write_out", action='store_false', default=True, help="write data in outfile (default: true)") args = parser.parse_args() sp = args.sp # The sinkspeed m/day @@ -209,9 +260,20 @@ class DinoParticle(JITParticle): imageFileName=args.imageFileName periodicFlag=args.periodic time_in_days = int(float(eval(args.time_in_days))) + time_in_years = int(time_in_days/366.0) with_GC = args.useGC - branch = "benchmarking" + outdt = 24 + cbdt = 24 # np.infty + + # ==== this is not a good choice for long-running simulations (e.g. 10y+) - needs to be adapted to scale ==== # + # ==== also, it is a backward simulation, so the high-value should be first. ==== # + global idgen + idgen = GenerateID_Service(SequentialIdGenerator) + idgen.setTimeLine(0, delta(days=time_in_days).total_seconds()) + # idgen.setTimeLine(delta(days=time_in_days).total_seconds(), 0) + + branch = "nodes" computer_env = "local/unspecified" scenario = "palaeo-parcels" headdir = "" @@ -233,7 +295,7 @@ class DinoParticle(JITParticle): elif fnmatch.fnmatchcase(os.uname()[1], "*.bullx*"): # Cartesius CARTESIUS_SCRATCH_USERNAME = 'ckehluu' headdir = "/scratch/shared/{}/experiments/palaeo-parcels".format(CARTESIUS_SCRATCH_USERNAME) - odir = os.path.join(headdir, "/BENCHres") + odir = os.path.join(headdir, "BENCHres") dirread_pal = os.path.join(headdir,'NEMOdata') datahead = "/projects/0/topios/hydrodynamic_data" dirread_top = os.path.join(datahead, 'NEMO-MEDUSA/ORCA0083-N006/') @@ -250,7 +312,22 @@ class DinoParticle(JITParticle): # dirread_pal = '/projects/0/palaeo-parcels/NEMOdata/' - outfile = 'grid_dd' + str(int(dd)) + '_sp' + str(int(sp)) + outfile = 'grid_dd' + str(int(dd)) + outfile += '_sp' + str(int(sp)) + if periodicFlag: + outfile += '_p' + if time_in_years != 1: + outfile += '_' + str(time_in_years) + 'y' + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_size = mpi_comm.Get_size() + outfile += '_n' + str(mpi_size) + if args.profiling: + outfile += '_prof' + if with_GC: + outfile += '_wGC' + else: + outfile += '_woGC' dirwrite = os.path.join(odir, "sp%d_dd%d" % (int(sp),int(dd))) if not os.path.exists(dirwrite): os.mkdir(dirwrite) @@ -264,16 +341,21 @@ class DinoParticle(JITParticle): assert ~(np.isnan(latsz)).any(), 'locations should not contain any NaN values' dep = dd * np.ones(latsz.shape) - times = np.array([datetime(2000, 12, 25) - delta(days=x) for x in range(0,int(365),3)]) - time = np.empty(shape=(0)) + # timesz = np.array([datetime(2000, 12, 25) - delta(days=x) for x in range(0,int(365),3)]) + timesz = np.array([datetime(2000, 12, 31) - delta(days=x) for x in range(0, time_in_days, 3)]) + # timesz = np.array([datetime(2000, 12, 25) - delta(days=x) for x in range(0, time_in_days, 3)]) + times = np.empty(shape=(0)) depths = np.empty(shape=(0)) lons = np.empty(shape=(0)) lats = np.empty(shape=(0)) - for i in range(len(times)): + for i in range(len(timesz)): lons = np.append(lons,lonsz) lats = np.append(lats, latsz) depths = np.append(depths, np.zeros(len(lonsz), dtype=np.float32)) - time = np.append(time, np.full(len(lonsz),times[i])) + times = np.append(times, np.full(len(lonsz),timesz[i])) + del timesz + del lonsz + del latsz print("running {} on {} (uname: {}) - branch '{}' - (target) N: {} - argv: {}".format(scenario, computer_env, os.uname()[1], branch, lons.shape[0], sys.argv[1:])) @@ -298,20 +380,30 @@ class DinoParticle(JITParticle): ifiles = sorted(glob(dirread_top + 'means/ORCA0083-N06_2000????d05I.nc')) bfile = dirread_top + 'domain/bathymetry_ORCA12_V3.3.nc' - fieldset = set_nemo_fieldset(ufiles, vfiles, wfiles, tfiles, pfiles, dfiles, ifiles, bfile, os.path.join(dirread_pal, "domain/coordinates.nc")) + fieldset = set_nemo_fieldset(ufiles, vfiles, wfiles, tfiles, pfiles, dfiles, ifiles, bfile, os.path.join(dirread_pal, "domain/coordinates.nc"), periodicFlag=periodicFlag) fieldset.add_periodic_halo(zonal=True) fieldset.add_constant('dwellingdepth', np.float(dd)) fieldset.add_constant('sinkspeed', sp/86400.) fieldset.add_constant('maxage', 300000.*86400) fieldset.add_constant('surface', 2.5) - pset = ParticleSet_Benchmark.from_list(fieldset=fieldset, pclass=DinoParticle, lon=lons.tolist(), lat=lats.tolist(), depth=depths.tolist(), time = time) + print("|lon| = {}; |lat| = {}; |depth| = {}, |times| = {}".format(lonsz.shape[0], latsz.shape[0], dep.shape[0], times.shape[0])) + + # ==== Set min/max depths in the fieldset ==== # + fs_depths = fieldset.U.depth + idgen.setDepthLimits(np.min(fs_depths), np.max(fs_depths)) + + pset = ParticleSet_Benchmark.from_list(fieldset=fieldset, pclass=DinoParticle, lon=lons.tolist(), lat=lats.tolist(), depth=depths.tolist(), time = times) """ Kernal + Execution""" postProcessFuncs = [] if with_GC: postProcessFuncs.append(perIterGC) - pfile = pset.ParticleFile(os.path.join(dirwrite, outfile), convert_at_end=True, write_ondelete=True) + output_fpath = None + pfile = None + if args.write_out: + output_fpath = os.path.join(dirwrite, outfile) + pfile = pset.ParticleFile(output_fpath, outputdt=outdt, convert_at_end=True, write_ondelete=True) kernels = pset.Kernel(initials) + Sink + Age + pset.Kernel(AdvectionRK4_3D) + Age starttime = 0 @@ -326,12 +418,11 @@ class DinoParticle(JITParticle): else: # starttime = ostime.time() starttime = ostime.process_time() - # pset.execute(kernels, runtime=delta(days=365*9), dt=delta(minutes=-20), output_file=pfile, verbose_progress=False, # recovery={ErrorCode.ErrorOutOfBounds: DeleteParticle}, postIterationCallbacks=postProcessFuncs) # postIterationCallbacks=postProcessFuncs, callbackdt=delta(hours=12) - pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(hours=-12), output_file=pfile, verbose_progress=False, recovery={ErrorCode.ErrorOutOfBounds: DeleteParticle}, postIterationCallbacks=postProcessFuncs, callbackdt=np.infty) - + pset.execute(kernels, runtime=delta(days=time_in_days), dt=delta(hours=-12), output_file=pfile, verbose_progress=False, recovery={ErrorCode.ErrorOutOfBounds: DeleteParticle}, postIterationCallbacks=postProcessFuncs, callbackdt=cbdt) + if MPI: mpi_comm = MPI.COMM_WORLD mpi_rank = mpi_comm.Get_rank() @@ -343,11 +434,14 @@ class DinoParticle(JITParticle): #endtime = ostime.time() endtime = ostime.process_time() + if args.write_out: + pfile.close() + + size_Npart = len(pset.nparticle_log) + Npart = pset.nparticle_log.get_param(size_Npart - 1) if MPI: mpi_comm = MPI.COMM_WORLD mpi_comm.Barrier() - size_Npart = len(pset.nparticle_log) - Npart = pset.nparticle_log.get_param(size_Npart - 1) Npart = mpi_comm.reduce(Npart, op=MPI.SUM, root=0) if mpi_comm.Get_rank() == 0: if size_Npart>0: @@ -356,16 +450,12 @@ class DinoParticle(JITParticle): avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time*1000.0)) else: - size_Npart = len(pset.nparticle_log) - Npart = pset.nparticle_log.get_param(size_Npart-1) if size_Npart > 0: sys.stdout.write("final # particles: {}\n".format( Npart )) sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime - starttime)) avg_time = np.mean(np.array(pset.total_log.get_values(), dtype=np.float64)) sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time * 1000.0)) - pfile.close() - if MPI: mpi_comm = MPI.COMM_WORLD # mpi_comm.Barrier() @@ -377,6 +467,7 @@ class DinoParticle(JITParticle): pset.plot_and_log(target_N=1, imageFilePath=imageFileName, odir=odir) print('Execution finished') + exit(0) diff --git a/performance/benchmark_perlin.py b/performance/benchmark_perlin.py index 67622b4909..3115328e66 100644 --- a/performance/benchmark_perlin.py +++ b/performance/benchmark_perlin.py @@ -4,11 +4,13 @@ """ from parcels import AdvectionEE, AdvectionRK45, AdvectionRK4 -from parcels import FieldSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, ErrorCode -from parcels.particleset_benchmark import ParticleSet_Benchmark as ParticleSet +from parcels import FieldSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, ErrorCode # , RectilinearZGrid +from parcels.particleset_node_benchmark import ParticleSet_Benchmark as ParticleSetN +from parcels.particleset_vectorized_benchmark import ParticleSet_Benchmark as ParticleSetV from parcels.field import Field, VectorField, NestedField, SummedField # from parcels import plotTrajectoriesFile_loadedField from parcels import rng as random +from parcels.tools import idgen from datetime import timedelta as delta import math from argparse import ArgumentParser @@ -47,8 +49,8 @@ noctaves=3 #noctaves=4 # formerly -perlinres=(1,32,8) -shapescale=(4,8,8) +perlinres=(1,24,12) # (1,32,8) +shapescale=(4,4,4) # (4,8,8) #shapescale=(8,6,6) # formerly perlin_persistence=0.6 img_shape = (int(math.pow(2,noctaves))*perlinres[1]*shapescale[1], int(math.pow(2,noctaves))*perlinres[2]*shapescale[2]) @@ -91,29 +93,15 @@ def perlin_fieldset_from_numpy(periodic_wrap=False, write_out=False): # Coordinates of the test fieldset (on A-grid in deg) lon = np.linspace(-a*0.5, a*0.5, img_shape[0], dtype=np.float32) - # sys.stdout.write("lon field: {}\n".format(lon.size)) lat = np.linspace(-b*0.5, b*0.5, img_shape[1], dtype=np.float32) - # sys.stdout.write("lat field: {}\n".format(lat.size)) totime = tsteps*tscale*24.0*60.0*60.0 time = np.linspace(0., totime, tsteps, dtype=np.float64) - # sys.stdout.write("time field: {}\n".format(time.size)) # Define arrays U (zonal), V (meridional) U = perlin2d.generate_fractal_noise_temporal2d(img_shape, tsteps, (perlinres[1], perlinres[2]), noctaves, perlin_persistence, max_shift=((-1, 2), (-1, 2))) U = np.transpose(U, (0,2,1)) - # U = np.swapaxes(U, 1, 2) - # print("U-statistics - min: {:10.7f}; max: {:10.7f}; avg. {:10.7f}; std_dev: {:10.7f}".format(U.min(initial=0), U.max(initial=0), U.mean(), U.std())) V = perlin2d.generate_fractal_noise_temporal2d(img_shape, tsteps, (perlinres[1], perlinres[2]), noctaves, perlin_persistence, max_shift=((-1, 2), (-1, 2))) V = np.transpose(V, (0,2,1)) - # V = np.swapaxes(V, 1, 2) - # print("V-statistics - min: {:10.7f}; max: {:10.7f}; avg. {:10.7f}; std_dev: {:10.7f}".format(V.min(initial=0), V.max(initial=0), V.mean(), V.std())) - - # U = perlin3d.generate_fractal_noise_3d(img_shape, perlinres, noctaves, perlin_persistence) * scalefac - # U = np.transpose(U, (0,2,1)) - # sys.stdout.write("U field shape: {} - [tdim][ydim][xdim]=[{}][{}][{}]\n".format(U.shape, time.shape[0], lat.shape[0], lon.shape[0])) - # V = perlin3d.generate_fractal_noise_3d(img_shape, perlinres, noctaves, perlin_persistence) * scalefac - # V = np.transpose(V, (0,2,1)) - # sys.stdout.write("V field shape: {} - [tdim][ydim][xdim]=[{}][{}][{}]\n".format(V.shape, time.shape[0], lat.shape[0], lon.shape[0])) U *= scalefac V *= scalefac @@ -147,8 +135,7 @@ def perlin_fieldset_from_xarray(periodic_wrap=False): totime = img_shape[0] * 24.0 * 60.0 * 60.0 time = np.linspace(0., totime, img_shape[0], dtype=np.float32) - # Define arrays U (zonal), V (meridional), W (vertical) and P (sea - # surface height) all on A-grid + # Define arrays U (zonal), V (meridional), W (vertical) U = perlin3d.generate_fractal_noise_3d(img_shape, perlinres, noctaves, perlin_persistence) * scalefac U = np.transpose(U, (0,2,1)) V = perlin3d.generate_fractal_noise_3d(img_shape, perlinres, noctaves, perlin_persistence) * scalefac @@ -157,7 +144,7 @@ def perlin_fieldset_from_xarray(periodic_wrap=False): dimensions = {'time': time, 'lon': lon, 'lat': lat} dims = ('time', 'lat', 'lon') data = {'Uxr': xr.DataArray(U, coords=dimensions, dims=dims), - 'Vxr': xr.DataArray(V, coords=dimensions, dims=dims)} #,'Pxr': xr.DataArray(P, coords=dimensions, dims=dims) + 'Vxr': xr.DataArray(V, coords=dimensions, dims=dims)} ds = xr.Dataset(data) variables = {'U': 'Uxr', 'V': 'Vxr'} @@ -212,8 +199,12 @@ def Age(particle, fieldset, time): parser.add_argument("-sN", "--start_n_particles", dest="start_nparticles", type=str, default="96", help="(optional) number of particles generated per release cycle (if --rt is set) (default: 96)") parser.add_argument("-m", "--mode", dest="compute_mode", choices=['jit','scipy'], default="jit", help="computation mode = [JIT, SciPp]") parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + parser.add_argument("--vmode", dest="vmode", action='store_true', default=False, help="use vectorized- instead of node-ParticleSet (default: False)") args = parser.parse_args() + ParticleSet = ParticleSetN + if args.vmode: + ParticleSet = ParticleSetV imageFileName=args.imageFileName periodicFlag=args.periodic backwardSimulation = args.backwards @@ -234,6 +225,9 @@ def Age(particle, fieldset, time): # cycle_scaler = 3.0 / 2.0 cycle_scaler = 7.0 / 4.0 start_N_particles = int(float(eval(args.start_nparticles))) + + idgen.setTimeLine(0, delta(days=time_in_days).total_seconds()) + if MPI: mpi_comm = MPI.COMM_WORLD if mpi_comm.Get_rank() == 0: @@ -253,7 +247,7 @@ def Age(particle, fieldset, time): nowtime = datetime.datetime.now() random.seed(nowtime.microsecond) - branch = "benchmarking" + branch = "nodes" computer_env = "local/unspecified" scenario = "perlin" odir = "" @@ -329,15 +323,53 @@ def Age(particle, fieldset, time): if agingParticles: if repeatdtFlag: pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) - psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) - pset.add(psetA) + if args.vmode: + # ==== VECTOR VERSION OF THE ADD ==== # + psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + pset.add(psetA) + + # for i in range(int(Nparticle/2.0)): + # lon = np.random.random() * a + # lat = np.random.random() * b + # pdepth = 0 + # ptime = simStart + # pindex = idgen.total_length + # pid = idgen.nextID(lon, lat, pdepth, ptime) + # pdata = age_ptype[(args.compute_mode).lower()](lon=lon, lat=lat, pid=pid, fieldset=fieldset, depth=pdepth, time=ptime, index=pindex) + # pset.add(pdata) + else: + # ==== NODE VERSION OF THE ADD ==== # + lonlat_field = np.random.rand(int(Nparticle/2.0), 2) + lonlat_field *= np.array([a, b]) + time_field = np.ones((int(Nparticle/2.0), 1), dtype=np.float64) * simStart + pdata = np.concatenate( (lonlat_field, time_field), axis=1 ) + pset.add(pdata) else: pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) else: if repeatdtFlag: pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) - psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) - pset.add(psetA) + if args.vmode: + # ==== VECTOR VERSION OF THE ADD ==== # + psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + pset.add(psetA) + + # for i in range(int(Nparticle/2.0)): + # lon = np.random.random() * a + # lat = np.random.random() * b + # pdepth = 0 + # ptime = simStart + # pindex = idgen.total_length + # pid = idgen.nextID(lon, lat, pdepth, ptime) + # pdata = ptype[(args.compute_mode).lower()](lon=lon, lat=lat, pid=pid, fieldset=fieldset, depth=pdepth, time=ptime, index=pindex) + # pset.add(pdata) + else: + # ==== NODE VERSION OF THE ADD ==== # + lonlat_field = np.random.rand(int(Nparticle/2.0), 2) + lonlat_field *= np.array([a, b]) + time_field = np.ones((int(Nparticle/2.0), 1), dtype=np.float64) * simStart + pdata = np.concatenate( (lonlat_field, time_field), axis=1 ) + pset.add(pdata) else: pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) else: @@ -345,15 +377,55 @@ def Age(particle, fieldset, time): if agingParticles: if repeatdtFlag: pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) - psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) - pset.add(psetA) + if args.vmode: + # ==== VECTOR VERSION OF THE ADD ==== # + psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + pset.add(psetA) + + # for i in range(int(Nparticle/2.0)): + # lon = np.random.random() * a + # lat = np.random.random() * b + # pdepth = 0 + # ptime = simStart + # pindex = idgen.total_length + # pid = idgen.nextID(lon, lat, pdepth, ptime) + # pdata = age_ptype[(args.compute_mode).lower()](lon=lon, lat=lat, pid=pid, fieldset=fieldset, depth=pdepth, time=ptime, index=pindex) + # pset.add(pdata) + + else: + # ==== NODE VERSION OF THE ADD ==== # + lonlat_field = np.random.rand(int(Nparticle/2.0), 2) + lonlat_field *= np.array([a, b]) + time_field = np.ones((int(Nparticle/2.0), 1), dtype=np.float64) * simStart + pdata = np.concatenate( (lonlat_field, time_field), axis=1 ) + pset.add(pdata) else: pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) else: if repeatdtFlag: pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) - psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) - pset.add(psetA) + if args.vmode: + # ==== VECTOR VERSION OF THE ADD ==== # + psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + pset.add(psetA) + + # for i in range(int(Nparticle/2.0)): + # lon = np.random.random() * a + # lat = np.random.random() * b + # pdepth = 0 + # ptime = simStart + # pindex = idgen.total_length + # pid = idgen.nextID(lon, lat, pdepth, ptime) + # pdata = ptype[(args.compute_mode).lower()](lon=lon, lat=lat, pid=pid, fieldset=fieldset, depth=pdepth, time=ptime, index=pindex) + # pset.add(pdata) + + else: + # ==== NODE VERSION OF THE ADD ==== # + lonlat_field = np.random.rand(int(Nparticle/2.0), 2) + lonlat_field *= np.array([a, b]) + time_field = np.ones((int(Nparticle/2.0), 1), dtype=np.float64) * simStart + pdata = np.concatenate( (lonlat_field, time_field), axis=1 ) + pset.add(pdata) else: pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) @@ -458,4 +530,6 @@ def Age(particle, fieldset, time): else: pset.plot_and_log(target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 150]) - + # idgen.close() + # del idgen + exit(0) diff --git a/performance/benchmark_stommel.py b/performance/benchmark_stommel.py index a336cacf2d..20223a712b 100644 --- a/performance/benchmark_stommel.py +++ b/performance/benchmark_stommel.py @@ -4,10 +4,11 @@ """ from parcels import AdvectionEE, AdvectionRK45, AdvectionRK4 -from parcels import FieldSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, ErrorCode -from parcels.particleset_benchmark import ParticleSet_Benchmark as ParticleSet -# from parcels.kernel_benchmark import Kernel_Benchmark as Kernel +from parcels import FieldSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, RectilinearZGrid, ErrorCode +from parcels.particleset_node_benchmark import ParticleSet_Benchmark as ParticleSetN +from parcels.particleset_vectorized_benchmark import ParticleSet_Benchmark as ParticleSetV from parcels.field import VectorField, NestedField, SummedField +from parcels.tools import idgen # from parcels import plotTrajectoriesFile_loadedField from datetime import timedelta as delta import math @@ -15,7 +16,6 @@ import datetime import numpy as np import xarray as xr -# import pytest import fnmatch import gc import os @@ -29,7 +29,7 @@ from mpi4py import MPI except: MPI = None -with_GC = False +# with_GC = False pset = None @@ -221,8 +221,12 @@ def Age(particle, fieldset, time): parser.add_argument("-sN", "--start_n_particles", dest="start_nparticles", type=str, default="96", help="(optional) number of particles generated per release cycle (if --rt is set) (default: 96)") parser.add_argument("-m", "--mode", dest="compute_mode", choices=['jit','scipy'], default="jit", help="computation mode = [JIT, SciPp]") parser.add_argument("-G", "--GC", dest="useGC", action='store_true', default=False, help="using a garbage collector (default: false)") + parser.add_argument("--vmode", dest="vmode", action='store_true', default=False, help="use vectorized- instead of node-ParticleSet (default: False)") args = parser.parse_args() + ParticleSet = ParticleSetN + if args.vmode: + ParticleSet = ParticleSetV imageFileName=args.imageFileName periodicFlag=args.periodic backwardSimulation = args.backwards @@ -238,6 +242,9 @@ def Age(particle, fieldset, time): np_scaler = 3.0 / 2.0 cycle_scaler = 7.0 / 4.0 start_N_particles = int(float(eval(args.start_nparticles))) + + idgen.setTimeLine(0, delta(days=time_in_days).total_seconds()) + if MPI: mpi_comm = MPI.COMM_WORLD if mpi_comm.Get_rank() == 0: @@ -252,12 +259,12 @@ def Age(particle, fieldset, time): sys.stdout.write("N: {}\n".format(Nparticle)) dt_minutes = 60 - #dt_minutes = 20 - #random.seed(123456) + # dt_minutes = 20 + # random.seed(123456) nowtime = datetime.datetime.now() random.seed(nowtime.microsecond) - branch = "benchmarking" + branch = "nodes" computer_env = "local/unspecified" scenario = "stommel" odir = "" @@ -294,11 +301,8 @@ def Age(particle, fieldset, time): mpi_comm = MPI.COMM_WORLD mpi_rank = mpi_comm.Get_rank() if mpi_rank==0: - # global_t_0 = ostime.time() - # global_t_0 = MPI.Wtime() global_t_0 = ostime.process_time() else: - # global_t_0 = ostime.time() global_t_0 = ostime.process_time() simStart = None @@ -328,15 +332,47 @@ def Age(particle, fieldset, time): if agingParticles: if repeatdtFlag: pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) - psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) - pset.add(psetA) + # psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + # pset.add(psetA) + + # for i in range(int(Nparticle/2.0)): + # lon = np.random.random() * a + # lat = np.random.random() * b + # pdepth = 0 + # ptime = simStart + # pindex = idgen.total_length + # pid = idgen.nextID(lon, lat, pdepth, ptime) + # pdata = age_ptype[(args.compute_mode).lower()](lon=lon, lat=lat, pid=pid, fieldset=fieldset, depth=pdepth, time=ptime, index=pindex) + # pset.add(pdata) + + lonlat_field = np.random.rand(int(Nparticle/2.0), 2) + lonlat_field *= np.array([a, b]) + time_field = np.ones((int(Nparticle/2.0), 1), dtype=np.float64) * simStart + pdata = np.concatenate( (lonlat_field, time_field), axis=1 ) + pset.add(pdata) else: pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) else: if repeatdtFlag: pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) - psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) - pset.add(psetA) + # psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + # pset.add(psetA) + + # for i in range(int(Nparticle/2.0)): + # lon = np.random.random() * a + # lat = np.random.random() * b + # pdepth = 0 + # ptime = simStart + # pindex = idgen.total_length + # pid = idgen.nextID(lon, lat, pdepth, ptime) + # pdata = ptype[(args.compute_mode).lower()](lon=lon, lat=lat, pid=pid, fieldset=fieldset, depth=pdepth, time=ptime, index=pindex) + # pset.add(pdata) + + lonlat_field = np.random.rand(int(Nparticle/2.0), 2) + lonlat_field *= np.array([a, b]) + time_field = np.ones((int(Nparticle/2.0), 1), dtype=np.float64) * simStart + pdata = np.concatenate( (lonlat_field, time_field), axis=1 ) + pset.add(pdata) else: pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) else: @@ -344,15 +380,47 @@ def Age(particle, fieldset, time): if agingParticles: if repeatdtFlag: pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) - psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) - pset.add(psetA) + # psetA = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + # pset.add(psetA) + + # for i in range(int(Nparticle/2.0)): + # lon = np.random.random() * a + # lat = np.random.random() * b + # pdepth = 0 + # ptime = simStart + # pindex = idgen.total_length + # pid = idgen.nextID(lon, lat, pdepth, ptime) + # pdata = age_ptype[(args.compute_mode).lower()](lon=lon, lat=lat, pid=pid, fieldset=fieldset, depth=pdepth, time=ptime, index=pindex) + # pset.add(pdata) + + lonlat_field = np.random.rand(int(Nparticle/2.0), 2) + lonlat_field *= np.array([a, b]) + time_field = np.ones((int(Nparticle/2.0), 1), dtype=np.float64) * simStart + pdata = np.concatenate( (lonlat_field, time_field), axis=1 ) + pset.add(pdata) else: pset = ParticleSet(fieldset=fieldset, pclass=age_ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) else: if repeatdtFlag: pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(start_N_particles, 1) * a, lat=np.random.rand(start_N_particles, 1) * b, time=simStart, repeatdt=delta(minutes=repeatRateMinutes)) - psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) - pset.add(psetA) + # psetA = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(int(Nparticle/2.0), 1) * a, lat=np.random.rand(int(Nparticle/2.0), 1) * b, time=simStart) + # pset.add(psetA) + + # for i in range(int(Nparticle/2.0)): + # lon = np.random.random() * a + # lat = np.random.random() * b + # pdepth = 0 + # ptime = simStart + # pindex = idgen.total_length + # pid = idgen.nextID(lon, lat, pdepth, ptime) + # pdata = ptype[(args.compute_mode).lower()](lon=lon, lat=lat, pid=pid, fieldset=fieldset, depth=pdepth, time=ptime, index=pindex) + # pset.add(pdata) + + lonlat_field = np.random.rand(int(Nparticle/2.0), 2) + lonlat_field *= np.array([a, b]) + time_field = np.ones((int(Nparticle/2.0), 1), dtype=np.float64) * simStart + pdata = np.concatenate( (lonlat_field, time_field), axis=1 ) + pset.add(pdata) else: pset = ParticleSet(fieldset=fieldset, pclass=ptype[(args.compute_mode).lower()], lon=np.random.rand(Nparticle, 1) * a, lat=np.random.rand(Nparticle, 1) * b, time=simStart) @@ -377,9 +445,9 @@ def Age(particle, fieldset, time): out_fname += "_MPI" else: out_fname += "_noMPI" - out_fname += "_n"+str(Nparticle) if periodicFlag: out_fname += "_p" + out_fname += "_n"+str(Nparticle) if backwardSimulation: out_fname += "_bwd" else: @@ -398,11 +466,8 @@ def Age(particle, fieldset, time): mpi_comm = MPI.COMM_WORLD mpi_rank = mpi_comm.Get_rank() if mpi_rank==0: - # starttime = ostime.time() - # starttime = MPI.Wtime() starttime = ostime.process_time() else: - # starttime = ostime.time() starttime = ostime.process_time() kernels = pset.Kernel(AdvectionRK4,delete_cfiles=True) if agingParticles: @@ -426,41 +491,13 @@ def Age(particle, fieldset, time): mpi_comm = MPI.COMM_WORLD mpi_rank = mpi_comm.Get_rank() if mpi_rank==0: - # endtime = ostime.time() - # endtime = MPI.Wtime() endtime = ostime.process_time() else: - #endtime = ostime.time() endtime = ostime.process_time() if args.write_out: output_file.close() - # if MPI: - # mpi_comm = MPI.COMM_WORLD - # if mpi_comm.Get_rank() == 0: - # dt_time = [] - # for i in range(len(perflog.times_steps)): - # if i==0: - # dt_time.append( (perflog.times_steps[i]-global_t_0) ) - # else: - # dt_time.append( (perflog.times_steps[i]-perflog.times_steps[i-1]) ) - # sys.stdout.write("final # particles: {}\n".format(perflog.Nparticles_step[len(perflog.Nparticles_step)-1])) - # sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime-starttime)) - # avg_time = np.mean(np.array(dt_time, dtype=np.float64)) - # sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time*1000.0)) - # else: - # dt_time = [] - # for i in range(len(perflog.times_steps)): - # if i == 0: - # dt_time.append((perflog.times_steps[i] - global_t_0)) - # else: - # dt_time.append((perflog.times_steps[i] - perflog.times_steps[i - 1])) - # sys.stdout.write("final # particles: {}\n".format(perflog.Nparticles_step[len(perflog.Nparticles_step)-1])) - # sys.stdout.write("Time of pset.execute(): {} sec.\n".format(endtime - starttime)) - # avg_time = np.mean(np.array(dt_time, dtype=np.float64)) - # sys.stdout.write("Avg. kernel update time: {} msec.\n".format(avg_time * 1000.0)) - if MPI: mpi_comm = MPI.COMM_WORLD mpi_comm.Barrier() @@ -501,5 +538,8 @@ def Age(particle, fieldset, time): pset.plot_and_log(memory_used=Nmem, nparticles=Nparticles, target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 120]) else: pset.plot_and_log(target_N=target_N, imageFilePath=imageFileName, odir=odir, xlim_range=[0, 730], ylim_range=[0, 150]) - + # idgen.close() + # del idgen + # print('Execution finished') + exit(0) diff --git a/setup.py b/setup.py index d51bf82fea..e7519a8869 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,7 @@ setup_requires=['setuptools_scm', 'setuptools_scm_git_archive'], packages=find_packages(), package_data={'parcels': ['include/*', + 'nodes/*', 'examples/*']}, entry_points={'console_scripts': [ 'parcels_get_examples = parcels.scripts.get_examples:main', diff --git a/tests/test_advection.py b/tests/test_advection.py index c2577ea684..641acdca35 100644 --- a/tests/test_advection.py +++ b/tests/test_advection.py @@ -1,4 +1,5 @@ -from parcels import FieldSet, ParticleSet, ScipyParticle, JITParticle, ErrorCode +from parcels import FieldSet, ScipyParticle, JITParticle, ErrorCode +from parcels.particleset_vectorized import ParticleSet from parcels import AdvectionEE, AdvectionRK4, AdvectionRK45, AdvectionRK4_3D import numpy as np import pytest diff --git a/tests/test_diffusion.py b/tests/test_diffusion.py index 334228fc66..98bb2394a7 100644 --- a/tests/test_diffusion.py +++ b/tests/test_diffusion.py @@ -1,5 +1,6 @@ -from parcels import (FieldSet, Field, RectilinearZGrid, ParticleSet, BrownianMotion2D, +from parcels import (FieldSet, Field, RectilinearZGrid, BrownianMotion2D, SpatiallyVaryingBrownianMotion2D, JITParticle, ScipyParticle, Variable) +from parcels.particleset_vectorized import ParticleSet from parcels import rng as random from datetime import timedelta as delta import numpy as np diff --git a/tests/test_fieldset.py b/tests/test_fieldset.py index 38b0398820..9d6ab53540 100644 --- a/tests/test_fieldset.py +++ b/tests/test_fieldset.py @@ -1,4 +1,5 @@ -from parcels import FieldSet, ParticleSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, AdvectionRK4_3D, RectilinearZGrid, ErrorCode, OutOfTimeError +from parcels import FieldSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, AdvectionRK4_3D, RectilinearZGrid, ErrorCode, OutOfTimeError +from parcels.particleset_vectorized import ParticleSet from parcels.field import Field, VectorField from parcels.tools.converters import TimeConverter, _get_cftime_calendars, _get_cftime_datetimes, UnitConverter, GeographicPolar import dask.array as da diff --git a/tests/test_fieldset_sampling.py b/tests/test_fieldset_sampling.py index c9abfc5d63..e9e9ae625a 100644 --- a/tests/test_fieldset_sampling.py +++ b/tests/test_fieldset_sampling.py @@ -1,5 +1,6 @@ -from parcels import (FieldSet, Field, NestedField, ParticleSet, ScipyParticle, JITParticle, Geographic, +from parcels import (FieldSet, Field, NestedField, ScipyParticle, JITParticle, Geographic, AdvectionRK4, AdvectionRK4_3D, Variable, ErrorCode) +from parcels.particleset_vectorized import ParticleSet import numpy as np import pytest from math import cos, pi diff --git a/tests/test_grids.py b/tests/test_grids.py index 54e0b50dfd..a42063ed23 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -1,4 +1,5 @@ -from parcels import (FieldSet, Field, ParticleSet, ScipyParticle, JITParticle, Variable, AdvectionRK4, AdvectionRK4_3D, ErrorCode) +from parcels import (FieldSet, Field, ScipyParticle, JITParticle, Variable, AdvectionRK4, AdvectionRK4_3D, ErrorCode) +from parcels.particleset_vectorized import ParticleSet from parcels import RectilinearZGrid, RectilinearSGrid, CurvilinearZGrid import numpy as np import xarray as xr diff --git a/tests/test_kernel_execution.py b/tests/test_kernel_execution.py index 800c53eb7c..2ad3e0e4e8 100644 --- a/tests/test_kernel_execution.py +++ b/tests/test_kernel_execution.py @@ -1,10 +1,9 @@ from os import path -from parcels import ( - FieldSet, ParticleSet, ScipyParticle, JITParticle, ErrorCode, KernelError, - OutOfBoundsError, AdvectionRK4 -) +from parcels import (FieldSet, ScipyParticle, JITParticle, ErrorCode, KernelError,OutOfBoundsError, AdvectionRK4) +from parcels.particleset_vectorized import ParticleSet import numpy as np import pytest +from parcels.tools import idgen from parcels.tools import logger from os import getpid @@ -324,3 +323,4 @@ def run_test_execution_runtime(fset, mode, start, end, substeps, dt, npart=10): run_test_execution_runtime(fset, 'scipy', 2., 16., 5, 3.) run_test_execution_runtime(fset, 'scipy', 20., 10., 4, -1.) run_test_execution_runtime(fset, 'scipy', 20., -10., 7, -2.) + # idgen.close() diff --git a/tests/test_kernel_language.py b/tests/test_kernel_language.py index 723a7da054..b840f82754 100644 --- a/tests/test_kernel_language.py +++ b/tests/test_kernel_language.py @@ -1,9 +1,13 @@ -from parcels import FieldSet, ParticleSet, ScipyParticle, JITParticle, Kernel, Variable, ErrorCode -from parcels.kernels.seawaterdensity import polyTEOS10_bsq, UNESCO_Density +import random as py_random +#from parcels import random as parcels_random => this doesn't work, and we should not 'ghost' up which random module is used; the module shoudl really get a different name from parcels import random as parcels_random +from parcels.rng import parcels_random as pa_random import numpy as np +from parcels import FieldSet, ScipyParticle, JITParticle, Variable, ErrorCode +from parcels.particleset_vectorized import ParticleSet +from parcels.kernel_vectorized import Kernel +from parcels.kernels.seawaterdensity import polyTEOS10_bsq, UNESCO_Density import pytest -import random as py_random from os import path import sys @@ -13,7 +17,8 @@ def expr_kernel(name, pset, expr): pycode = """def %s(particle, fieldset, time): - particle.p = %s""" % (name, expr) + particle.p = %s + return ErrorCode.Success""" % (name, expr) return Kernel(pset.fieldset, pset.ptype, pyfunc=None, funccode=pycode, funcname=name, funcvars=['particle']) @@ -240,11 +245,12 @@ def kernel2(particle, fieldset, time): def random_series(npart, rngfunc, rngargs, mode): - random = parcels_random if mode == 'jit' else py_random - random.seed(1234) - func = getattr(random, rngfunc) + random_mod = parcels_random if mode == 'jit' else py_random + print(repr(random_mod)) + random_mod.seed(1234) + func = getattr(random_mod, rngfunc) series = [func(*rngargs) for _ in range(npart)] - random.seed(1234) # Reset the RNG seed + random_mod.seed(1234) # Reset the RNG seed return series @@ -257,7 +263,7 @@ def random_series(npart, rngfunc, rngargs, mode): def test_random_float(fieldset, mode, rngfunc, rngargs, npart=10): """ Test basic random number generation """ class TestParticle(ptype[mode]): - p = Variable('p', dtype=np.float32 if rngfunc == 'randint' else np.float32) + p = Variable('p', dtype=np.int32 if rngfunc == 'randint' else np.float32) pset = ParticleSet(fieldset, pclass=TestParticle, lon=np.linspace(0., 1., npart), lat=np.zeros(npart) + 0.5) @@ -399,4 +405,4 @@ class DensParticle(ptype[mode]): if(pressure == 0): assert np.allclose(pset[0].density, 1005.9465) elif(pressure == 10): - assert np.allclose(pset[0].density, 1006.4179) + assert np.allclose(pset[0].density, 1006.4179) \ No newline at end of file diff --git a/tests/test_node_processing.py b/tests/test_node_processing.py new file mode 100644 index 0000000000..e7ec486bc5 --- /dev/null +++ b/tests/test_node_processing.py @@ -0,0 +1,199 @@ +from parcels import (FieldSet, Field, ScipyParticle, JITParticle, + Variable, ErrorCode, CurvilinearZGrid, AdvectionRK4) +from parcels.particleset_node import ParticleSet +from parcels.particlefile_node import ParticleFile +from parcels.kernel_node import Kernel +from parcels.nodes.Node import Node, NodeJIT +from parcels.tools import idgen +from parcels.tools import get_cache_dir +from os import path +# from parcels.tools import logger +import numpy as np +import pytest + + +try: + from mpi4py import MPI +except: + MPI = None + +ptype = {'scipy': ScipyParticle, 'jit': JITParticle} + + +def fieldset(xdim=40, ydim=100): + U = np.zeros((ydim, xdim), dtype=np.float32) + V = np.zeros((ydim, xdim), dtype=np.float32) + lon = np.linspace(0, 1, xdim, dtype=np.float32) + lat = np.linspace(-60, 60, ydim, dtype=np.float32) + depth = np.zeros(1, dtype=np.float32) + data = {'U': np.array(U, dtype=np.float32), 'V': np.array(V, dtype=np.float32)} + dimensions = {'lat': lat, 'lon': lon, 'depth': depth} + return FieldSet.from_data(data, dimensions) + + +@pytest.fixture(name="fieldset") +def fieldset_fixture(xdim=40, ydim=100): + return fieldset(xdim=xdim, ydim=ydim) + + +#@pytest.mark.parametrize('mode', ['scipy', 'jit']) +@pytest.mark.parametrize('mode', ['jit']) +def test_pset_repeated_release(fieldset, mode, npart=10): + time = np.arange(0, npart, 1) # release 1 particle every second + pset = ParticleSet(fieldset, lon=np.zeros(npart), lat=np.zeros(npart), + pclass=ptype[mode], time=time) + assert np.allclose([n.data.time for n in pset.data], time) + + def IncrLon(particle, fieldset, time): + particle.lon += 1. + pset.execute(IncrLon, dt=1., runtime=npart) + assert np.allclose([n.data.lon for n in pset.data], np.arange(npart, 0, -1)) + + +@pytest.mark.parametrize('mode', ['scipy', 'jit']) +def test_pset_dt0(fieldset, mode, npart=10): + pset = ParticleSet(fieldset, lon=np.zeros(npart), lat=np.zeros(npart), + pclass=ptype[mode]) + + def IncrLon(particle, fieldset, time): + particle.lon += 1 + pset.execute(IncrLon, dt=0., runtime=npart) + assert np.allclose([n.data.lon for n in pset.data], 1.) + assert np.allclose([n.data.time for n in pset.data], 0.) + + +@pytest.mark.parametrize('mode', ['scipy', 'jit']) +def test_pset_custom_ptype(fieldset, mode, npart=100): + class TestParticle(ptype[mode]): + user_vars = {'p': np.float32, 'n': np.int32} + + def __init__(self, *args, **kwargs): + super(TestParticle, self).__init__(*args, **kwargs) + self.p = 0.33 + self.n = 2 + + pset = ParticleSet(fieldset, pclass=TestParticle, + lon=np.linspace(0, 1, npart, dtype=np.float32), + lat=np.linspace(1, 0, npart, dtype=np.float32)) + assert(pset.size == 100) + assert np.allclose([n.data.p - 0.33 for n in pset.data], np.zeros(npart), rtol=1e-12) + assert np.allclose([n.data.n - 2 for n in pset.data], np.zeros(npart), rtol=1e-12) + + +@pytest.mark.parametrize('mode', ['scipy', 'jit']) +def test_pset_add_explicit(fieldset, mode, npart=100): + nclass = Node + if mode == 'jit': + nclass = NodeJIT + lon = np.linspace(0, 1, npart, dtype=np.float64) + lat = np.linspace(1, 0, npart, dtype=np.float64) + pset = ParticleSet(fieldset=fieldset, pclass=ptype[mode], lon=[], lat=[], lonlatdepth_dtype=np.float64) + index_mapping = {} + for i in range(0, npart): + index = idgen.total_length + id = idgen.getID(lon[i], lat[i], 0., 0.) + index_mapping[i] = id + pdata = ptype[mode](lon[i], lat[i], pid=id, fieldset=fieldset, index=index) + ndata = nclass(id=id, data=pdata) + pset.add(ndata) + assert(pset.size == 100) + # ==== of course this is not working as the order in pset.data and lon is not the same ==== # + # assert np.allclose([n.data.lon for n in pset.data], lon, rtol=1e-12) + assert np.allclose([pset.get_by_id(index_mapping[i]).data.lon for i in index_mapping.keys()], lon, rtol=1e-12) + assert np.allclose([pset.get_by_id(index_mapping[i]).data.lat for i in index_mapping.keys()], lat, rtol=1e-12) + if MPI: + mpi_comm = MPI.COMM_WORLD + mpi_comm.Barrier() + + +@pytest.mark.parametrize('mode', ['scipy', 'jit']) +def test_pset_node_execute(fieldset, mode, tmpdir, npart=100): + nclass = Node + if mode == 'jit': + nclass = NodeJIT + lon = np.linspace(0, 1, npart, dtype=np.float64) + lat = np.linspace(1, 0, npart, dtype=np.float64) + outfilepath = tmpdir.join("pfile_node_execute_test.nc") + pset = ParticleSet(fieldset=fieldset, lon=[], lat=[], pclass=ptype[mode], lonlatdepth_dtype=np.float64) + pfile = pset.ParticleFile(name=outfilepath, outputdt=1.) + index_mapping = {} + for i in range(npart): + index = idgen.getID(lon[i], lat[i], 0., 0.) + index_mapping[i] = index + pdata = ptype[mode](lon[i], lat[i], pid=index, fieldset=fieldset) + ndata = nclass(id=index, data=pdata) + pset.add(ndata) + pset.execute(AdvectionRK4, runtime=0., dt=10., output_file=pfile) + pfile.close() + assert(pset.size == 100) + # ==== of course this is not working as the order in pset.data and lon is not the same ==== # + # assert np.allclose([n.data.lon for n in pset.data], lon, rtol=1e-12) + assert np.allclose([pset.get_by_id(index_mapping[i]).data.lon for i in index_mapping.keys()], lon, rtol=1e-12) + assert np.allclose([pset.get_by_id(index_mapping[i]).data.lat for i in index_mapping.keys()], lat, rtol=1e-12) + + +def run_test_pset_add_explicit(fset, mode, npart=100): + nclass = Node + if mode == 'jit': + nclass = NodeJIT + + lon = np.linspace(0, 1, npart, dtype=np.float64) + lat = np.linspace(1, 0, npart, dtype=np.float64) + outfilepath = path.join(get_cache_dir(), "add_explicit", "pfile_node_add_explicit.nc") + pset = ParticleSet(fieldset=fset, pclass=ptype[mode], lon=lon, lat=lat, lonlatdepth_dtype=np.float64) + pfile = pset.ParticleFile(name=outfilepath, outputdt=1.) + + index_mapping = {} + for i in range(0, npart): + index = idgen.total_length + id = idgen.getID(lon[i], lat[i], None, None) + index_mapping[i] = id + pdata = ptype[mode](lon[i], lat[i], pid=id, fieldset=fset, index=index) + ndata = nclass(id=id, data=pdata) + pset.add(ndata) + # assert(pset.size >= npart) + # print("# particles: {}".format(pset.size)) + # logger.info("# particles: {}".format(pset.size)) + for tstep in range(3): + pfile.write(pset, tstep) + pfile.close() + # logger.info("# particles: {}".format(pset.size)) + assert (pset.size > 0) + assert (pset.size <= 2* npart) + # ==== of course this is not working as the order in pset.data and lon is not the same ==== # + # assert np.allclose([n.data.lon for n in pset.data], lon, rtol=1e-12) + # ==== makes no sence in MPI ==== # + if MPI is None: + assert np.allclose([pset.get_by_id(index_mapping[i]).data.lon for i in index_mapping.keys()], lon, rtol=1e-12) + assert np.allclose([pset.get_by_id(index_mapping[i]).data.lat for i in index_mapping.keys()], lat, rtol=1e-12) + + +def run_test_pset_node_execute(fset, mode, npart=10000): + nclass = Node + if mode == 'jit': + nclass = NodeJIT + lon = np.linspace(0, 1, npart, dtype=np.float64) + lat = np.linspace(1, 0, npart, dtype=np.float64) + outfilepath = path.join(get_cache_dir(), "pfile_node_execute_test.nc") + pset = ParticleSet(fieldset=fset, lon=lon, lat=lat, pclass=ptype[mode], lonlatdepth_dtype=np.float64) + pfile = pset.ParticleFile(name=outfilepath, outputdt=360.0) + for i in range(npart): + index = idgen.getID(lon[i], lat[i], 0., 0.) + pdata = ptype[mode](lon[i], lat[i], pid=index, fieldset=fset) + ndata = nclass(id=index, data=pdata) + pset.add(ndata) + pset.execute(AdvectionRK4, runtime=0., dt=(360.0 * 24.0 * 10.0), output_file=pfile) + pfile.close() + # logger.info("# particles: {}".format(pset.size)) + assert (pset.size > 0) + assert (pset.size <= 2* npart) + + +if __name__ == '__main__': + fset = fieldset() + run_test_pset_add_explicit(fset, 'jit') + run_test_pset_add_explicit(fset, 'scipy') + run_test_pset_node_execute(fset, 'jit') + run_test_pset_node_execute(fset, 'scipy') + # idgen.close() + # del idgen diff --git a/tests/test_particle_file.py b/tests/test_particle_file.py index d538cf6ca8..67cf586f97 100644 --- a/tests/test_particle_file.py +++ b/tests/test_particle_file.py @@ -1,5 +1,5 @@ -from parcels import (FieldSet, ParticleSet, ScipyParticle, JITParticle, - Variable, ErrorCode) +from parcels import (FieldSet, ScipyParticle, JITParticle, Variable, ErrorCode) +from parcels.particleset_vectorized import ParticleSet from parcels.particlefile import _set_calendar from parcels.tools.converters import _get_cftime_calendars, _get_cftime_datetimes import numpy as np diff --git a/tests/test_particle_sets.py b/tests/test_particle_sets.py index bda1e55398..9fac78a6af 100644 --- a/tests/test_particle_sets.py +++ b/tests/test_particle_sets.py @@ -1,8 +1,9 @@ -from parcels import (FieldSet, ParticleSet, Field, ScipyParticle, JITParticle, - Variable, ErrorCode, CurvilinearZGrid, AdvectionRK4) +from parcels import (FieldSet, Field, ScipyParticle, JITParticle, + Variable, ErrorCode, CurvilinearZGrid) +from parcels.particleset_vectorized import ParticleSet import numpy as np import pytest -from parcels.compiler import get_cache_dir +from parcels.tools.global_statics import get_cache_dir from parcels.tools import logger from os import path diff --git a/tests/test_particles.py b/tests/test_particles.py index 39a4a66b5f..1f4f29376a 100644 --- a/tests/test_particles.py +++ b/tests/test_particles.py @@ -1,4 +1,5 @@ -from parcels import FieldSet, ParticleSet, ScipyParticle, JITParticle, Variable, AdvectionRK4 +from parcels import FieldSet, ScipyParticle, JITParticle, Variable, AdvectionRK4 +from parcels.particleset_vectorized import ParticleSet import numpy as np import pytest from operator import attrgetter diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 64fef3f527..76d4abd611 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -1,5 +1,5 @@ -from parcels import (FieldSet, ParticleSet, JITParticle, AdvectionRK4, - plotTrajectoriesFile) +from parcels import (FieldSet, JITParticle, AdvectionRK4, plotTrajectoriesFile) +from parcels.particleset_vectorized import ParticleSet from datetime import timedelta as delta import numpy as np import pytest