diff --git a/parcels/__init__.py b/parcels/__init__.py index 2b55c9b35d..e0a9271db2 100644 --- a/parcels/__init__.py +++ b/parcels/__init__.py @@ -8,6 +8,7 @@ from parcels.kernel import * # noqa import parcels.rng as ParcelsRandom # noqa from parcels.particlefile import * # noqa +from parcels.compilation import * # noqa from parcels.kernels import * # noqa from parcels.scripts import * # noqa from parcels.gridset import * # noqa diff --git a/parcels/compilation/__init__.py b/parcels/compilation/__init__.py new file mode 100644 index 0000000000..8fe787b361 --- /dev/null +++ b/parcels/compilation/__init__.py @@ -0,0 +1,2 @@ +from .codegenerator import * # noqa +from .codecompiler import * # noqa diff --git a/parcels/compilation/codecompiler.py b/parcels/compilation/codecompiler.py new file mode 100644 index 0000000000..79917906e8 --- /dev/null +++ b/parcels/compilation/codecompiler.py @@ -0,0 +1,271 @@ +import os +import subprocess +from struct import calcsize + +try: + from mpi4py import MPI +except: + MPI = None + + +class Compiler_parameters(object): + def __init__(self): + self._compiler = "" + self._cppargs = [] + self._ldargs = [] + self._incdirs = [] + self._libdirs = [] + self._libs = [] + self._dynlib_ext = "" + self._stclib_ext = "" + self._obj_ext = "" + self._exe_ext = "" + + @property + def compiler(self): + return self._compiler + + @property + def cppargs(self): + return self._cppargs + + @property + def ldargs(self): + return self._ldargs + + @property + def incdirs(self): + return self._incdirs + + @property + def libdirs(self): + return self._libdirs + + @property + def libs(self): + return self._libs + + @property + def dynlib_ext(self): + return self._dynlib_ext + + @property + def stclib_ext(self): + return self._stclib_ext + + @property + def obj_ext(self): + return self._obj_ext + + @property + def exe_ext(self): + return self._exe_ext + + +class GNU_parameters(Compiler_parameters): + def __init__(self, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None): + super(GNU_parameters, self).__init__() + if cppargs is None: + cppargs = [] + if ldargs is None: + ldargs = [] + if incdirs is None: + incdirs = [] + if libdirs is None: + libdirs = [] + if libs is None: + libs = [] + + Iflags = [] + if isinstance(incdirs, list): + for i, dir in enumerate(incdirs): + Iflags.append("-I"+dir) + Lflags = [] + if isinstance(libdirs, list): + for i, dir in enumerate(libdirs): + Lflags.append("-L"+dir) + lflags = [] + if isinstance(libs, list): + for i, lib in enumerate(libs): + lflags.append("-l" + lib) + + cc_env = os.getenv('CC') + self._compiler = "mpicc" if MPI else "gcc" if cc_env is None else cc_env + opt_flags = ['-g', '-O3'] + arch_flag = ['-m64' if calcsize("P") == 8 else '-m32'] + self._cppargs = ['-Wall', '-fPIC'] + self._cppargs += Iflags + self._cppargs += opt_flags + cppargs + arch_flag + self._ldargs = ['-shared'] + self._ldargs += Lflags + self._ldargs += lflags + self._ldargs += ldargs + if len(Lflags) > 0: + self._ldargs += ['-Wl, -rpath=%s' % (":".join(libdirs))] + self._ldargs += arch_flag + self._incdirs = incdirs + self._libdirs = libdirs + self._libs = libs + self._dynlib_ext = "so" + self._stclib_ext = "a" + self._obj_ext = "o" + self._exe_ext = "" + + +class Clang_parameters(Compiler_parameters): + def __init__(self, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None): + super(Clang_parameters, self).__init__() + if cppargs is None: + cppargs = [] + if ldargs is None: + ldargs = [] + if incdirs is None: + incdirs = [] + if libdirs is None: + libdirs = [] + if libs is None: + libs = [] + self._compiler = "cc" + self._cppargs = cppargs + self._ldargs = ldargs + self._incdirs = incdirs + self._libdirs = libdirs + self._libs = libs + self._dynlib_ext = "dynlib" + self._stclib_ext = "a" + self._obj_ext = "o" + self._exe_ext = "exe" + + +class MinGW_parameters(Compiler_parameters): + def __init__(self, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None): + super(MinGW_parameters, self).__init__() + if cppargs is None: + cppargs = [] + if ldargs is None: + ldargs = [] + if incdirs is None: + incdirs = [] + if libdirs is None: + libdirs = [] + if libs is None: + libs = [] + self._compiler = "gcc" + self._cppargs = cppargs + self._ldargs = ldargs + self._incdirs = incdirs + self._libdirs = libdirs + self._libs = libs + self._dynlib_ext = "so" + self._stclib_ext = "a" + self._obj_ext = "o" + self._exe_ext = "exe" + + +class VS_parameters(Compiler_parameters): + def __init__(self, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None): + super(VS_parameters, self).__init__() + if cppargs is None: + cppargs = [] + if ldargs is None: + ldargs = [] + if incdirs is None: + incdirs = [] + if libdirs is None: + libdirs = [] + if libs is None: + libs = [] + self._compiler = "cl" + self._cppargs = cppargs + self._ldargs = ldargs + self._incdirs = incdirs + self._libdirs = libdirs + self._libs = libs + self._dynlib_ext = "dll" + self._stclib_ext = "lib" + self._obj_ext = "obj" + self._exe_ext = "exe" + + +class CCompiler(object): + """A compiler object for creating and loading shared libraries. + + :arg cc: C compiler executable (uses environment variable ``CC`` if not provided). + :arg cppargs: A list of arguments to the C compiler (optional). + :arg ldargs: A list of arguments to the linker (optional).""" + + def __init__(self, cc=None, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None, tmp_dir=os.getcwd()): + if cppargs is None: + cppargs = [] + if ldargs is None: + ldargs = [] + + self._cc = os.getenv('CC') if cc is None else cc + self._cppargs = cppargs + self._ldargs = ldargs + self._dynlib_ext = "" + self._stclib_ext = "" + self._obj_ext = "" + self._exe_ext = "" + self._tmp_dir = tmp_dir + + def compile(self, src, obj, log): + pass + + def _create_compile_process_(self, cmd, src, log): + with open(log, 'w') as logfile: + try: + subprocess.check_call(cmd, stdout=logfile, stderr=logfile) + except OSError: + err = """OSError during compilation +Please check if compiler exists: %s""" % self._cc + raise RuntimeError(err) + except subprocess.CalledProcessError: + with open(log, 'r') as logfile2: + err = """Error during compilation: +Compilation command: %s +Source/Destination file: %s +Log file: %s + +Log output: %s""" % (" ".join(cmd), src, logfile.name, logfile2.read()) + raise RuntimeError(err) + return True + + +class CCompiler_SS(CCompiler): + """ + single-stage C-compiler; used for a SINGLE source file + """ + def __init__(self, cc=None, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None, tmp_dir=os.getcwd()): + super(CCompiler_SS, self).__init__(cc=cc, cppargs=cppargs, ldargs=ldargs, incdirs=incdirs, libdirs=libdirs, libs=libs, tmp_dir=tmp_dir) + + def compile(self, src, obj, log): + cc = [self._cc] + self._cppargs + ['-o', obj, src] + self._ldargs + with open(log, 'w') as logfile: + logfile.write("Compiling: %s\n" % " ".join(cc)) + self._create_compile_process_(cc, src, log) + + +class GNUCompiler_SS(CCompiler_SS): + """A compiler object for the GNU Linux toolchain. + + :arg cppargs: A list of arguments to pass to the C compiler + (optional). + :arg ldargs: A list of arguments to pass to the linker (optional).""" + def __init__(self, cppargs=None, ldargs=None, incdirs=None, libdirs=None, libs=None, tmp_dir=os.getcwd()): + c_params = GNU_parameters(cppargs, ldargs, incdirs, libdirs, libs) + super(GNUCompiler_SS, self).__init__(c_params.compiler, cppargs=c_params.cppargs, ldargs=c_params.ldargs, incdirs=c_params.incdirs, libdirs=c_params.libdirs, libs=c_params.libs, tmp_dir=tmp_dir) + self._dynlib_ext = c_params.dynlib_ext + self._stclib_ext = c_params.stclib_ext + self._obj_ext = c_params.obj_ext + self._exe_ext = c_params.exe_ext + + def compile(self, src, obj, log): + lib_pathfile = os.path.basename(obj) + lib_pathdir = os.path.dirname(obj) + obj = os.path.join(lib_pathdir, lib_pathfile) + + super(GNUCompiler_SS, self).compile(src, obj, log) + + +GNUCompiler = GNUCompiler_SS diff --git a/parcels/codegenerator.py b/parcels/compilation/codegenerator.py similarity index 100% rename from parcels/codegenerator.py rename to parcels/compilation/codegenerator.py diff --git a/parcels/compiler.py b/parcels/compiler.py deleted file mode 100644 index 760f6c7b98..0000000000 --- a/parcels/compiler.py +++ /dev/null @@ -1,86 +0,0 @@ -import subprocess -from os import getenv -from os import path -from struct import calcsize -from tempfile import gettempdir - -try: - from os import getuid -except: - # Windows does not have getuid(), so define to simply return 'tmp' - def getuid(): - return 'tmp' -try: - from mpi4py import MPI -except: - MPI = None -from pathlib import Path - - -def get_package_dir(): - return path.abspath(path.dirname(__file__)) - - -def get_cache_dir(): - directory = path.join(gettempdir(), "parcels-%s" % getuid()) - Path(directory).mkdir(exist_ok=True) - return directory - - -class Compiler(object): - """A compiler object for creating and loading shared libraries. - - :arg cc: C compiler executable (uses environment variable ``CC`` if not provided). - :arg cppargs: A list of arguments to the C compiler (optional). - :arg ldargs: A list of arguments to the linker (optional).""" - - def __init__(self, cc=None, cppargs=None, ldargs=None): - if cppargs is None: - cppargs = [] - if ldargs is None: - ldargs = [] - - self._cc = getenv('CC') if cc is None else cc - self._cppargs = cppargs - self._ldargs = ldargs - - def compile(self, src, obj, log): - cc = [self._cc] + self._cppargs + ['-o', obj, src] + self._ldargs - with open(log, 'w') as logfile: - logfile.write("Compiling: %s\n" % " ".join(cc)) - try: - subprocess.check_call(cc, stdout=logfile, stderr=logfile) - except OSError: - err = """OSError during compilation -Please check if compiler exists: %s""" % self._cc - raise RuntimeError(err) - except subprocess.CalledProcessError: - with open(log, 'r') as logfile2: - err = """Error during compilation: -Compilation command: %s -Source file: %s -Log file: %s - -Log output: %s""" % (" ".join(cc), src, logfile.name, logfile2.read()) - raise RuntimeError(err) - - -class GNUCompiler(Compiler): - """A compiler object for the GNU Linux toolchain. - - :arg cppargs: A list of arguments to pass to the C compiler - (optional). - :arg ldargs: A list of arguments to pass to the linker (optional).""" - def __init__(self, cppargs=None, ldargs=None): - if cppargs is None: - cppargs = [] - if ldargs is None: - ldargs = [] - - opt_flags = ['-g', '-O3'] - arch_flag = ['-m64' if calcsize("P") == 8 else '-m32'] - cppargs = ['-Wall', '-fPIC', '-I%s' % path.join(get_package_dir(), 'include')] + opt_flags + cppargs - cppargs += arch_flag - ldargs = ['-shared'] + ldargs + arch_flag - compiler = "mpicc" if MPI else "gcc" - super(GNUCompiler, self).__init__(compiler, cppargs=cppargs, ldargs=ldargs) diff --git a/parcels/examples/tutorial_sampling.ipynb b/parcels/examples/tutorial_sampling.ipynb index 32a6c88ff6..d92482dd2f 100644 --- a/parcels/examples/tutorial_sampling.ipynb +++ b/parcels/examples/tutorial_sampling.ipynb @@ -599,8 +599,17 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.11" + }, + "pycharm": { + "stem_cell": { + "cell_type": "raw", + "source": [], + "metadata": { + "collapsed": false + } + } } }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/parcels/kernel.py b/parcels/kernel.py index 6bdcec08dc..b9e61cf947 100644 --- a/parcels/kernel.py +++ b/parcels/kernel.py @@ -24,9 +24,9 @@ except: MPI = None -from parcels.codegenerator import KernelGenerator -from parcels.codegenerator import LoopGenerator -from parcels.compiler import get_cache_dir +from parcels.compilation.codegenerator import KernelGenerator +from parcels.compilation.codegenerator import LoopGenerator +from parcels.tools.global_statics import get_cache_dir from parcels.field import Field from parcels.field import FieldOutOfBoundError from parcels.field import FieldOutOfBoundSurfaceError diff --git a/parcels/particleset.py b/parcels/particleset.py index ef214659be..bd515f5d21 100644 --- a/parcels/particleset.py +++ b/parcels/particleset.py @@ -7,9 +7,10 @@ import numpy as np import xarray as xr from operator import attrgetter +from os import path import progressbar -from parcels.compiler import GNUCompiler +from parcels.compilation.codecompiler import GNUCompiler from parcels.field import Field from parcels.field import NestedField from parcels.field import SummedField @@ -19,6 +20,7 @@ from parcels.particle import JITParticle from parcels.particlefile import ParticleFile from parcels.tools.converters import _get_cftime_calendars +from parcels.tools.global_statics import get_package_dir from parcels.tools.statuscodes import OperationCode from parcels.tools.loggers import logger try: @@ -669,7 +671,7 @@ def execute(self, pyfunc=AdvectionRK4, endtime=None, runtime=None, dt=1., if self.ptype.uses_jit: self.kernel.remove_lib() cppargs = ['-DDOUBLE_COORD_VARIABLES'] if self.lonlatdepth_dtype == np.float64 else None - self.kernel.compile(compiler=GNUCompiler(cppargs=cppargs)) + self.kernel.compile(compiler=GNUCompiler(cppargs=cppargs, incdirs=[path.join(get_package_dir(), 'include'), "."])) self.kernel.load_lib() # Convert all time variables to seconds diff --git a/parcels/rng.py b/parcels/rng.py index c2660c2978..2862ff9f33 100644 --- a/parcels/rng.py +++ b/parcels/rng.py @@ -1,15 +1,17 @@ import uuid +import _ctypes from ctypes import c_float from ctypes import c_int from os import path +from os import remove +from sys import platform import numpy.ctypeslib as npct -from parcels.compiler import get_cache_dir -from parcels.compiler import GNUCompiler +from parcels.tools import get_cache_dir, get_package_dir +from parcels.compilation.codecompiler import GNUCompiler from parcels.tools.loggers import logger - __all__ = ['seed', 'random', 'uniform', 'randint', 'normalvariate', 'expovariate', 'vonmisesvariate'] @@ -50,24 +52,76 @@ class RandomC(object): return parcels_vonmisesvariate(mu, kappa); } """ - ccode = stmt_import + fnct_seed - ccode += fnct_random + fnct_uniform + fnct_randint + fnct_normalvariate + fnct_expovariate + fnct_vonmisesvariate - basename = path.join(get_cache_dir(), 'parcels_random_%s' % uuid.uuid4()) - src_file = "%s.c" % basename - lib_file = "%s.so" % basename - log_file = "%s.log" % basename + _lib = None + ccode = None + src_file = None + lib_file = None + log_file = None def __init__(self): self._lib = None - - @property - def lib(self, compiler=GNUCompiler()): + self.ccode = "" + self.ccode += self.stmt_import + self.ccode += self.fnct_seed + self.ccode += self.fnct_random + self.ccode += self.fnct_uniform + self.ccode += self.fnct_randint + self.ccode += self.fnct_normalvariate + self.ccode += self.fnct_expovariate + self.ccode += self.fnct_vonmisesvariate + self._loaded = False + self.compile() + self.load_lib() + + def __del__(self): + self.unload_lib() + self.remove_lib() + + def unload_lib(self): + # Unload the currently loaded dynamic linked library to be secure + if self._lib is not None and self._loaded: + _ctypes.FreeLibrary(self._lib._handle) if platform == 'win32' else _ctypes.dlclose(self._lib._handle) + del self._lib + self._lib = None + self._loaded = False + + def load_lib(self): + self._lib = npct.load_library(self.lib_file, '.') + self._loaded = True + + def remove_lib(self): + # If file already exists, pull new names. This is necessary on a Windows machine, because + # Python's ctype does not deal in any sort of manner well with dynamic linked libraries on this OS. + if path.isfile(self.lib_file): + [remove(s) for s in [self.src_file, self.lib_file, self.log_file]] + + def compile(self, compiler=None): + if self.src_file is None or self.lib_file is None or self.log_file is None: + basename = 'parcels_random_%s' % uuid.uuid4() + lib_filename = "lib" + basename + basepath = path.join(get_cache_dir(), "%s" % basename) + libpath = path.join(get_cache_dir(), "%s" % lib_filename) + self.src_file = "%s.c" % basepath + self.lib_file = "%s.so" % libpath + self.log_file = "%s.log" % basepath + ccompiler = compiler + if ccompiler is None: + cppargs = [] + incdirs = [path.join(get_package_dir(), 'include'), ] + ccompiler = GNUCompiler(cppargs=cppargs, incdirs=incdirs) if self._lib is None: - with open(self.src_file, 'w') as f: + with open(self.src_file, 'w+') as f: f.write(self.ccode) - compiler.compile(self.src_file, self.lib_file, self.log_file) + ccompiler.compile(self.src_file, self.lib_file, self.log_file) logger.info("Compiled %s ==> %s" % ("ParcelsRandom", self.lib_file)) - self._lib = npct.load_library(self.lib_file, '.') + + @property + def lib(self): + if self.src_file is None or self.lib_file is None or self.log_file is None: + self.compile() + if self._lib is None or not self._loaded: + self.load_lib() + # self._lib = npct.load_library(self.lib_file, '.') return self._lib diff --git a/parcels/tools/__init__.py b/parcels/tools/__init__.py index 8e847fee26..5afbc7d9fe 100644 --- a/parcels/tools/__init__.py +++ b/parcels/tools/__init__.py @@ -1,4 +1,5 @@ from .converters import * # noqa +from .global_statics import * # noqa from .statuscodes import * # noqa from .interpolation_utils import * # noqa from .loggers import * # noqa diff --git a/parcels/tools/global_statics.py b/parcels/tools/global_statics.py new file mode 100644 index 0000000000..96b5edd869 --- /dev/null +++ b/parcels/tools/global_statics.py @@ -0,0 +1,36 @@ +import os +import sys +import _ctypes +from tempfile import gettempdir +from pathlib import Path + +try: + from os import getuid +except: + # Windows does not have getuid(), so define to simply return 'tmp' + def getuid(): + return 'tmp' + + +def cleanup_remove_files(lib_file, log_file): + if os.path.isfile(lib_file): + [os.remove(s) for s in [lib_file, log_file]] + + +def cleanup_unload_lib(lib): + # Clean-up the in-memory dynamic linked libraries. + # This is not really necessary, as these programs are not that large, but with the new random + # naming scheme which is required on Windows OS'es to deal with updates to a Parcels' kernel. + if lib is not None: + _ctypes.FreeLibrary(lib._handle) if sys.platform == 'win32' else _ctypes.dlclose(lib._handle) + + +def get_package_dir(): + fpath = Path(__file__) + return fpath.parent.parent + + +def get_cache_dir(): + directory = os.path.join(gettempdir(), "parcels-%s" % getuid()) + Path(directory).mkdir(exist_ok=True) + return directory