Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

196 attempt to detect filename when saving #218

Merged
merged 38 commits into from
Feb 14, 2024
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
11ef134
detect filename decorator
EddyCMWF Sep 19, 2023
c75bae2
CDS save tests added
EddyCMWF Sep 19, 2023
28f83ec
CDS save tests added
EddyCMWF Sep 19, 2023
d09f1fd
Merge branch 'develop' into 196-attempt-to-detect-filename-when-saving
EddyCMWF Oct 11, 2023
4b7efce
filename detector
EddyCMWF Oct 12, 2023
21e24eb
filename detector
EddyCMWF Oct 12, 2023
1d72662
filename detector
EddyCMWF Oct 12, 2023
0b77c2b
Merge branch 'develop' into 196-attempt-to-detect-filename-when-saving
EddyCMWF Oct 12, 2023
e247820
merge from develop
EddyCMWF Oct 17, 2023
c649518
qa
EddyCMWF Oct 17, 2023
a9225dd
qa
EddyCMWF Oct 17, 2023
1b86669
qa
EddyCMWF Oct 17, 2023
6169e11
Merge branch 'develop' into 196-attempt-to-detect-filename-when-saving
EddyCMWF Oct 17, 2023
1314550
Do not overwrite file being read
EddyCMWF Oct 17, 2023
e86a242
prevent anyone overwriting the file they are reading
EddyCMWF Oct 17, 2023
abed6af
QA
EddyCMWF Oct 17, 2023
3720f0a
better implementation
EddyCMWF Oct 17, 2023
ba4eac8
even better implementation
EddyCMWF Oct 18, 2023
85ae984
adding source_filename
EddyCMWF Nov 21, 2023
a1e8556
Merge branch 'develop' into 196-attempt-to-detect-filename-when-saving
EddyCMWF Nov 21, 2023
1d4b3d3
use source_filename
EddyCMWF Nov 21, 2023
249b59c
preserve source filename
EddyCMWF Nov 21, 2023
c191be9
preserve source filename
EddyCMWF Nov 21, 2023
58f1ffc
basename only
EddyCMWF Nov 21, 2023
e346b60
qa and warning tests
EddyCMWF Nov 22, 2023
590bf83
typo
EddyCMWF Nov 27, 2023
abb8198
Merge branch 'develop' into 196-attempt-to-detect-filename-when-saving
EddyCMWF Nov 29, 2023
d7b1ab0
Merge branch 'develop' into 196-attempt-to-detect-filename-when-saving
EddyCMWF Dec 13, 2023
37f6c85
merge from develop, with conflict resoved
EddyCMWF Feb 2, 2024
460fbde
remove debug print statement
EddyCMWF Feb 2, 2024
7405fe9
Merge branch 'main' into 196-attempt-to-detect-filename-when-saving
EddyCMWF Feb 8, 2024
beb52aa
save docstring updated
EddyCMWF Feb 8, 2024
acfc548
source_filename as a property
EddyCMWF Feb 8, 2024
21e6041
source_filename as a standard field in Source class
EddyCMWF Feb 8, 2024
05a33bd
Merge branch 'develop' into 196-attempt-to-detect-filename-when-saving
EddyCMWF Feb 8, 2024
f71b895
removing sneaky files
EddyCMWF Feb 8, 2024
5aae0f8
Review comments responded to, test now using the correct filename
EddyCMWF Feb 12, 2024
015e0c2
remove debug prints
EddyCMWF Feb 12, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion earthkit/data/core/fieldlist.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

from earthkit.data.core import Base
from earthkit.data.core.index import Index
from earthkit.data.decorators import cached_method
from earthkit.data.decorators import cached_method, detect_out_filename
from earthkit.data.utils.metadata import metadata_argument


Expand Down Expand Up @@ -1153,6 +1153,7 @@ def _is_shared_grid(self):
)
return False

@detect_out_filename
def save(self, filename, append=False):
r"""Write all the fields into a file.
sandorkertesz marked this conversation as resolved.
Show resolved Hide resolved

Expand Down
13 changes: 13 additions & 0 deletions earthkit/data/decorators.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,19 @@ def wrapped(*args, **kwargs):
return wrapped


def detect_out_filename(func):
@functools.wraps(func)
def wrapped(self, *args, **kwargs):
if len(args) == 0:
try:
args = [os.path.basename(self.path)]
EddyCMWF marked this conversation as resolved.
Show resolved Hide resolved
except AttributeError:
raise TypeError("Please provide a output filename")
EddyCMWF marked this conversation as resolved.
Show resolved Hide resolved
return func(self, *args, **kwargs)

return wrapped


LOCK = threading.RLock()


Expand Down
3 changes: 2 additions & 1 deletion earthkit/data/readers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from earthkit.data.core import Base
from earthkit.data.core.settings import SETTINGS
from earthkit.data.decorators import locked
from earthkit.data.decorators import detect_out_filename, locked

LOG = logging.getLogger(__name__)

Expand Down Expand Up @@ -60,6 +60,7 @@ def ignore(self):
def cache_file(self, *args, **kwargs):
return self.source.cache_file(*args, **kwargs)

@detect_out_filename
def save(self, path):
mode = "wb" if self.binary else "w"
with open(path, mode) as f:
Expand Down
2 changes: 2 additions & 0 deletions earthkit/data/sources/file.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

from earthkit.data import from_source
from earthkit.data.core.caching import CACHE
from earthkit.data.decorators import detect_out_filename
from earthkit.data.readers import reader

from . import Source
Expand Down Expand Up @@ -114,6 +115,7 @@ def to_numpy(self, **kwargs):
def values(self):
return self._reader.values

@detect_out_filename
def save(self, path):
return self._reader.save(path)

Expand Down
58 changes: 57 additions & 1 deletion tests/sources/test_cds.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,12 @@
# nor does it submit to any jurisdiction.
#

import os

import pytest

from earthkit.data import from_source
from earthkit.data.core.temporary import temp_directory
from earthkit.data.testing import NO_CDS


Expand Down Expand Up @@ -64,11 +67,37 @@ def test_cds_grib_3():
assert len(s) == 8


@pytest.mark.long_test
@pytest.mark.download
@pytest.mark.skipif(NO_CDS, reason="No access to CDS")
def test_cds_grib_save():
s = from_source(
"cds",
"reanalysis-era5-single-levels",
variable=["2t", "msl"],
product_type="reanalysis",
area=[50, -50, 20, 50],
date="2012-12-12",
time="12:00",
)
with temp_directory() as tmpdir:
# Check file save to assigned filename
s.save(os.path.join(tmpdir, "test.grib"))
assert os.path.isfile(os.path.join(tmpdir, "test.grib"))

# Check file can be saved in current dir with detected filename:
here = os.curdir
os.chdir(tmpdir)
s.save()
assert os.path.isfile(os.path.basename(s.path))
EddyCMWF marked this conversation as resolved.
Show resolved Hide resolved
os.chdir(here)


@pytest.mark.long_test
@pytest.mark.download
@pytest.mark.skipif(NO_CDS, reason="No access to CDS")
@pytest.mark.parametrize(
"split_on,expected_len",
"split_on, expected_len",
(
["variable", 2],
[("variable", "time"), 4],
Expand Down Expand Up @@ -105,6 +134,33 @@ def test_cds_netcdf():
assert len(s) == 2


@pytest.mark.long_test
@pytest.mark.download
@pytest.mark.skipif(NO_CDS, reason="No access to CDS")
def test_cds_netcdf_save():
s = from_source(
"cds",
"reanalysis-era5-single-levels",
variable=["2t", "msl"],
product_type="reanalysis",
area=[50, -50, 20, 50],
date="2012-12-12",
time="12:00",
format="netcdf",
)
with temp_directory() as tmpdir:
# Check file save to assigned filename
s.save(os.path.join(tmpdir, "test.nc"))
assert os.path.isfile(os.path.join(tmpdir, "test.nc"))

# Check file can be saved in current dir with detected filename:
here = os.curdir
os.chdir(tmpdir)
s.save()
assert os.path.isfile(os.path.basename(s.path))
os.chdir(here)


@pytest.mark.long_test
@pytest.mark.download
@pytest.mark.skipif(NO_CDS, reason="No access to CDS")
Expand Down
14 changes: 14 additions & 0 deletions tests/sources/test_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,20 @@ def test_file_source_netcdf():
assert len(s) == 2


def test_file_source_netcdf_save():
s = from_source("file", earthkit_examples_file("test.nc"))
with temp_directory() as tmpdir:
# Check file save to assigned filename
s.save(os.path.join(tmpdir, "test2.nc"))
assert os.path.isfile(os.path.join(tmpdir, "test2.nc"))
# Check file can be saved in current dir with detected filename:
here = os.curdir
os.chdir(tmpdir)
s.save()
assert os.path.isfile("test.nc")
os.chdir(here)


def test_file_source_odb():
s = from_source("file", earthkit_examples_file("test.odb"))
assert s.path == earthkit_examples_file("test.odb")
Expand Down
3 changes: 2 additions & 1 deletion tests/translators/test_translators.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import xarray as xr

from earthkit.data import from_source, transform, translators, wrappers
from earthkit.data.testing import earthkit_test_data_file
from earthkit.data.translators import ndarray as ndtranslator
from earthkit.data.translators import pandas as pdtranslator
from earthkit.data.translators import string as strtranslator
Expand Down Expand Up @@ -133,7 +134,7 @@ def test_gpd_dataframe_translator():

def test_transform_from_grib_file():
# transform grib-based data object
f = from_source("file", "tests/data/test_single.grib")
f = from_source("file", earthkit_test_data_file("test_single.grib"))
EddyCMWF marked this conversation as resolved.
Show resolved Hide resolved

# np.ndarray
transformed = transform(f, np.ndarray)
Expand Down
13 changes: 7 additions & 6 deletions tests/utils/test_module_inputs_wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

from earthkit.data import from_object, from_source
from earthkit.data.readers import Reader
from earthkit.data.testing import earthkit_test_data_file
from earthkit.data.utils import module_inputs_wrapper

from . import dummy_module
Expand Down Expand Up @@ -58,7 +59,7 @@

def test_transform_function_inputs_reader_to_xarray():
# Check EK GribReader object
EK_GRIB_READER = from_source("file", "tests/data/test_single.grib")
EK_GRIB_READER = from_source("file", earthkit_test_data_file("test_single.grib"))
ek_reader_result = WRAPPED_XR_ONES_LIKE(EK_GRIB_READER)
# Will return a DataSet becuase that is first value in kwarg_types
assert isinstance(ek_reader_result, xr.Dataset)
Expand All @@ -67,7 +68,7 @@ def test_transform_function_inputs_reader_to_xarray():

def test_transform_function_inputs_reader_to_xarray_typesetting():
# Check EK GribReader object
EK_GRIB_READER = from_source("file", "tests/data/test_single.grib")
EK_GRIB_READER = from_source("file", earthkit_test_data_file("test_single.grib"))
ek_reader_result = WRAPPED_XR_ONES_LIKE_TYPE_SETTING(EK_GRIB_READER)
# Will return a dataarray because that is first value in type-set Union
assert isinstance(ek_reader_result, xr.DataArray)
Expand All @@ -76,7 +77,7 @@ def test_transform_function_inputs_reader_to_xarray_typesetting():

def test_transform_module_inputs_reader_to_xarray():
# Check EK GribReader object
EK_GRIB_READER = from_source("file", "tests/data/test_single.grib")
EK_GRIB_READER = from_source("file", earthkit_test_data_file("test_single.grib"))
ek_reader_result = WRAPPED_DUMMY_MODULE.xarray_ones_like(EK_GRIB_READER)
# Data array because type-setting of function has dataarray first
assert isinstance(ek_reader_result, xr.DataArray)
Expand Down Expand Up @@ -108,22 +109,22 @@ def test_transform_module_inputs_wrapper_to_xarray():

def test_transform_function_inputs_reader_to_numpy():
# Test with Earthkit.data GribReader object
EK_GRIB_READER = from_source("file", "tests/data/test_single.grib")
EK_GRIB_READER = from_source("file", earthkit_test_data_file("test_single.grib"))
assert WRAPPED_NP_MEAN(EK_GRIB_READER) == np.mean(EK_GRIB_READER.to_numpy())
assert isinstance(WRAPPED_NP_MEAN(EK_GRIB_READER), np.float64)


def test_transform_function_inputs_reader_to_numpy_typesetting():
# Test with Earthkit.data GribReader object
EK_GRIB_READER = from_source("file", "tests/data/test_single.grib")
EK_GRIB_READER = from_source("file", earthkit_test_data_file("test_single.grib"))
result = WRAPPED_NP_MEAN_TYPE_SETTING(EK_GRIB_READER)
assert result == np.mean(EK_GRIB_READER.to_numpy())
assert isinstance(result, np.float64)


def test_transform_module_inputs_reader_to_numpy():
# Test with Earthkit.data GribReader object
EK_GRIB_READER = from_source("file", "tests/data/test_single.grib")
EK_GRIB_READER = from_source("file", earthkit_test_data_file("test_single.grib"))
result = WRAPPED_DUMMY_MODULE.numpy_mean(EK_GRIB_READER)
assert result == np.mean(EK_GRIB_READER.to_numpy())
assert isinstance(result, np.float64)
Expand Down
Loading