Skip to content

Commit

Permalink
icon-meteorology processor using last 4 reference-times
Browse files Browse the repository at this point in the history
  • Loading branch information
@heikoklein
heikoklein committed Dec 4, 2023
1 parent eedce16 commit 31c087c
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 69 deletions.
20 changes: 11 additions & 9 deletions utils/SnapPy/Snappy/EcMeteorologyCalculator.py
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
# SNAP: Servere Nuclear Accident Programme
# Copyright (C) 1992-2017 Norwegian Meteorological Institute
#
# This file is part of SNAP. SNAP is free software: you can
# redistribute it and/or modify it under the terms of the
# GNU General Public License as published by the
#
# This file is part of SNAP. SNAP is free software: you can
# redistribute it and/or modify it under the terms of the
# GNU General Public License as published by the
# Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
Expand Down Expand Up @@ -100,7 +100,7 @@ def add_expected_files(self, date):

def calc(self, proc=None):
'''run the calculation of ec-data if required.
Args:
proc -- A QProcess, which will be used to run a longer process in the background.
STDERR/STDOUT and signal-handler should be set. If proc is None, the
Expand Down Expand Up @@ -164,11 +164,13 @@ def calc(self, proc=None):
return

if __name__ == "__main__":
print(EcMeteorologyCalculator.findECGlobalData(datetime.strptime("2020-04-29T00", "%Y-%m-%dT%H")))
yesterday = datetime.today() - timedelta(days=1)
yesterdaytime = datetime.combine(yesterday, datetime.min.time())
print(EcMeteorologyCalculator.findECGlobalData(yesterdaytime))
try:
EcMeteorologyCalculator.findECGlobalData(datetime.strptime("2010-10-24T00", "%Y-%m-%dT%H"))
except Exception as e:
print(e.args[0])
# print(EcMeteorologyCalculator(Resources(), datetime.strptime("2016-10-24T00", "%Y-%m-%dT%H"), 63, 42, None))
ecmet = EcMeteorologyCalculator(EcMeteorologyCalculator.getGlobalMeteoResources(), datetime.strptime("2020-04-29T00", "%Y-%m-%dT%H"), -159, 20) # hawaii
ecmet = EcMeteorologyCalculator(EcMeteorologyCalculator.getGlobalMeteoResources(), yesterdaytime, -159, 20) # hawaii
print("recalc: ", ecmet.must_calc())
134 changes: 86 additions & 48 deletions utils/SnapPy/Snappy/ICONMeteorologyCalculator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@
@author: heikok
'''

from datetime import datetime
from glob import iglob
from datetime import datetime, timedelta
import netCDF4
import numpy as np
import os
Expand All @@ -33,6 +32,8 @@

class ICONMeteorologyCalculator(Snappy.MeteorologyCalculator.MeteorologyCalculator):
'''Calculate dwd icon-meteorology'''
dir_template = "NRPA_LON{lon0}_LAT{lat0}_ICON"


@staticmethod
def get_valid_timesteps(filename):
Expand Down Expand Up @@ -79,22 +80,49 @@ def getGlobalMeteoResources():
res.timeoffset = 0 # required offset between reference-time and first useful startup-time
return res

# def __init__(self, res: Snappy.MeteorologyCalculator.GlobalMeteoResource, dtime: datetime, domainCenterX, domainCenterY):
# super(res, dtime, domainCenterX, domainCenterY)

def add_expected_files(self, date):
def __init__(self, res: Snappy.MeteorologyCalculator.GlobalMeteoResource, dtime: datetime, domainCenterX, domainCenterY):
self.files = []
self.optFiles = []

# not utc-dependent for ICON-data
super().__init__(res, dtime, domainCenterX, domainCenterY)
self.globalOptionalFiles = []
for hr in (6, 12, 18): # earlier hours
hdate = self.date - timedelta(hours=hr)
datefile = self.findGlobalData(res, hdate)
self.add_expected_files(datefile[0], optional=True)
self.globalOptionalFiles.append(datefile[1])

def add_expected_files(self, date, optional=False):
# only one file expected
self.files.append(os.path.join(self.outputdir,
self.res.output_filename_pattern.format(year=date.year,
month=date.month,
day=date.day,
UTC=date.hour,
resdir=Resources().directory)))
file = os.path.join(
self.outputdir,
self.res.output_filename_pattern.format(year=date.year,
month=date.month,
day=date.day,
UTC=date.hour,
resdir=Resources().directory)
)
if optional:
self.optFiles.append(file)
else:
self.files.append(file)

return

def get_meteorology_files(self):
# The Icon-Meteorology calcuator has the newest file as expected, and
# older ones as optional, while snap expects the newest ones to be last,
# so reversing the files
return reversed(super().get_meteorology_files())

def must_calc(self):
'''check if calculation is required or has been done earlier'''
recalc = False
for f in self.files + self.optFiles:
if (not os.path.isfile(f)):
recalc = True
return recalc


def calc(self, proc=None):
'''run the calculation of ec-data if required.
Expand All @@ -105,47 +133,58 @@ def calc(self, proc=None):
subprocess will be run in the current-process. If proc is set, the caller
needs to wait for the proc to finish before calling other methods of this object
'''
if (not self.must_calc()):
if not self.must_calc():
return

precommand = '''#! /bin/bash
cd {outputdir} || exit 1
echo "Preprocessing 5-7days icon meteorology, please wait ca. 3min"
echo "MET-Input: {globalfile}"
echo "MET-Output: {outputdir}"
command = f"""#! /bin/bash
echo "Preprocessing 2-5days icon meteorology, please wait ca. 5min"
echo "MET-Output-Directory: {self.outputdir}"
cd {self.outputdir} || exit 1
umask 000
touch running
cp {resdir}/icon_fimex.cfg .
cp {Resources().directory}/icon_fimex.cfg .
chmod 666 icon_fimex.cfg
cp {resdir}/icon_sigma_hybrid.ncml .
cp {Resources().directory}/icon_sigma_hybrid.ncml .
chmod 666 icon_sigma_hybrid.ncml
tmpfile=out$$.nc4
fimex -c icon_fimex.cfg \
--input.file={globalfile} \
--interpolate.xAxisValues={xAxisValues} \
--interpolate.yAxisValues={yAxisValues} \
--extract.pickDimension.name=time \
--extract.pickDimension.list={timeStepList} \
--output.file=$tmpfile \
--output.type=nc4 \
&& mv $tmpfile {outputfile}
rm {outputdir}/running
'''
(timesteps, _) = ICONMeteorologyCalculator.get_valid_timesteps(self.globalfile)
timeStepList = ",".join([str(x) for x in timesteps])
command = precommand.format(resdir=Resources().directory,
xAxisValues="{},{},...,{}".format(self.lon0,
self.lon0+self.res.domainDeltaX,
self.lon0+self.res.domainWidth),
yAxisValues="{},{},...,{}".format(self.lat0,
self.lat0+self.res.domainDeltaY,
self.lat0+self.res.domainHeight),
globalfile=self.globalfile,
timeStepList=timeStepList,
outputfile=self.files[0],
outputdir=self.outputdir
)
"""
precommand = """
echo "MET-Input: {file}"
if [ -e "{outputfile}" ]; then
echo "{outputfile} exists, skipping..."
else
tmpfile=out$$.nc4
fimex -c icon_fimex.cfg \
--input.file={file} \
--interpolate.xAxisValues={xAxisValues} \
--interpolate.yAxisValues={yAxisValues} \
--extract.pickDimension.name=time \
--extract.pickDimension.list={timeStepList} \
--output.file=$tmpfile \
--output.type=nc4 \
&& mv $tmpfile {outputfile}
fi
"""
in_files = [self.globalfile] + self.globalOptionalFiles
out_files = self.files + self.optFiles
print(in_files, out_files, self.optFiles)
for i, file in enumerate(in_files):
(timesteps, _) = ICONMeteorologyCalculator.get_valid_timesteps(file)
timeStepList = ",".join([str(x) for x in timesteps])
command += precommand.format(
file=file,
xAxisValues="{},{},...,{}".format(self.lon0,
self.lon0+self.res.domainDeltaX,
self.lon0+self.res.domainWidth),
yAxisValues="{},{},...,{}".format(self.lat0,
self.lat0+self.res.domainDeltaY,
self.lat0+self.res.domainHeight),
timeStepList=timeStepList,
outputfile=out_files[i],
outputdir=self.outputdir
)
command += f"rm {self.outputdir}/running\n"
scriptFile = os.path.join(self.outputdir, "command.sh")
with open(scriptFile, 'w') as script:
script.write(command)
Expand All @@ -159,7 +198,6 @@ def calc(self, proc=None):
return

if __name__ == "__main__":
from datetime import timedelta
yesterday = datetime.today() - timedelta(days=1)
yesterdaytime = datetime.combine(yesterday, datetime.min.time())
for utc in (0, 6, 12, 18):
Expand Down
37 changes: 25 additions & 12 deletions utils/SnapPy/Snappy/MeteorologyCalculator.py
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
# SNAP: Servere Nuclear Accident Programme
# Copyright (C) 1992-2017 Norwegian Meteorological Institute
#
# This file is part of SNAP. SNAP is free software: you can
# redistribute it and/or modify it under the terms of the
# GNU General Public License as published by the
#
# This file is part of SNAP. SNAP is free software: you can
# redistribute it and/or modify it under the terms of the
# GNU General Public License as published by the
# Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
Expand Down Expand Up @@ -67,9 +67,11 @@ def __str__(self):


class MeteorologyCalculator(abc.ABC):
dir_template = "NRPA_LON{lon0}_LAT{lat0}_{utc:02d}"

'''Base-class to pre-calculate/extract meteorology'''
@staticmethod
def findAllGlobalData(res: GlobalMeteoResource):
def findAllGlobalData(res: GlobalMeteoResource):
'''Static method to find all global dataset.
Args:
Expand Down Expand Up @@ -158,18 +160,29 @@ def __init__(self, res: GlobalMeteoResource, dtime: datetime, domainCenterX, dom
lon0 = math.floor((domainCenterX-(res.domainWidth/2.))/10.)*10
self.lat0 = int(lat0)
self.lon0 = int(lon0)
self.outputdir = os.path.join(res.outputdir, "NRPA_LON{x}_LAT{y}_{utc:02d}".format(x=self.lon0, y=self.lat0, utc=utc))
self._set_outputdir(res, self.dir_template, self.lon0, self.lat0, utc)
self.add_expected_files(lastDateFile[0])

def _set_outputdir(self, res, template, lon0, lat0, utc):
self.outputdir = os.path.join(res.outputdir,
template.format(lon0=lon0, lat0=lat0, utc=utc))
self._check_create_outputdir(res)

def _check_create_outputdir(self, res):
"""Create the output-directory and, when already in use, create a temporary one.
In use is defined by the existence of a 'running' file (no locking since
not well enough supported across nodes on lustre)
"""
# try to avoid conflicting processes (not 100% save)
i = 1
while (os.path.isfile(os.path.join(self.outputdir, "running"))):
self.outputdir = os.path.join(res.outputdir, "NRPA_TEMP_{utc:02d}_{i}".format(utc=utc, i=i))
self.outputdir = os.path.join(res.outputdir, f"NRPA_TEMP_{utc}_{i}")
i+=1

if (not os.path.exists(self.outputdir)):
os.makedirs(self.outputdir)

self.add_expected_files(lastDateFile[0])


@abc.abstractmethod
def add_expected_files(self, date):
Expand Down Expand Up @@ -207,7 +220,7 @@ def must_calc(self):
@abc.abstractmethod
def calc(self, proc=None):
'''abstract baseclass to run the calculation of meteo-data if required. Should check if self.must_calc() at the beginning.
Args:
proc -- A QProcess, which will be used to run a longer process in the background.
STDERR/STDOUT and signal-handler should be set. If proc is None, the
Expand Down

0 comments on commit 31c087c

Please sign in to comment.