Skip to content

Commit

Permalink
Move more details into MetadataModel
Browse files Browse the repository at this point in the history
  • Loading branch information
gutzbenj committed Jan 7, 2025
1 parent 711dcb7 commit 09c7509
Show file tree
Hide file tree
Showing 54 changed files with 233 additions and 413 deletions.
6 changes: 3 additions & 3 deletions docs/data/parameters.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ from wetterdienst.provider.dwd.observation import DwdObservationMetadata

# print resolutions, datasets and parameters
for resolution in DwdObservationMetadata:
print(f"{2*"\t"}{resolution.name}", "\n")
print(f"{2 * "\t"}{resolution.name}", "\n")
for dataset in resolution:
print(f"{3*"\t"}{dataset.name}", "\n")
print(f"{3 * "\t"}{dataset.name}", "\n")
for parameter in dataset:
print(f"{4*"\t"}{parameter.name}", "\n")
print(f"{4 * "\t"}{parameter.name}", "\n")
```

The amount of distinct parameters and a list of the parameter names:
Expand Down
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ dependencies = [
"portion>=2.4.2,<3",
"pyarrow>=13,<19",
"pydantic>=2.7.3,<3",
"pydantic-extra-types>=2.10.1,<3",
"pydantic-settings>=2.7.0,<3",
"pypdf>=3.12.1,<6",
"python-dateutil>=2,<3",
Expand All @@ -105,6 +106,7 @@ dependencies = [
"stamina>=23,<25",
"tabulate>=0.8,<1",
"tqdm>=4,<5",
"tzdata>=2024.2,<2025",
"tzfpy>=0.15.2,<1",
]

Expand Down
7 changes: 3 additions & 4 deletions tests/provider/dwd/observation/test_api_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
from wetterdienst import Settings
from wetterdienst.metadata.columns import Columns
from wetterdienst.metadata.period import Period
from wetterdienst.metadata.timezone import Timezone
from wetterdienst.provider.dwd.observation.api import DwdObservationRequest
from wetterdienst.provider.dwd.observation.metadata import (
DwdObservationMetadata,
Expand Down Expand Up @@ -117,7 +116,7 @@ def test_request_period_historical_recent_now(default_settings):
]


@freeze_time(dt.datetime(2022, 1, 29, 1, 30, tzinfo=ZoneInfo(Timezone.GERMANY.value)))
@freeze_time(dt.datetime(2022, 1, 29, 1, 30, tzinfo=ZoneInfo("Europe/Berlin")))
def test_request_period_recent_now(default_settings):
request = DwdObservationRequest(
parameters=[("daily", "climate_summary")],
Expand All @@ -127,7 +126,7 @@ def test_request_period_recent_now(default_settings):
assert request.periods == [Period.RECENT, Period.NOW]


@freeze_time(dt.datetime(2022, 1, 29, 2, 30, tzinfo=ZoneInfo(Timezone.GERMANY.value)))
@freeze_time(dt.datetime(2022, 1, 29, 2, 30, tzinfo=ZoneInfo("Europe/Berlin")))
def test_request_period_now(default_settings):
# Now period
request = DwdObservationRequest(
Expand All @@ -138,7 +137,7 @@ def test_request_period_now(default_settings):
assert request.periods == [Period.NOW]


@freeze_time(dt.datetime(2021, 3, 28, 18, 38, tzinfo=ZoneInfo(Timezone.GERMANY.value)))
@freeze_time(dt.datetime(2021, 3, 28, 18, 38, tzinfo=ZoneInfo("Europe/Berlin")))
def test_request_period_now_fixed_date(default_settings):
# Now period
request = DwdObservationRequest(
Expand Down
2 changes: 1 addition & 1 deletion tests/provider/dwd/observation/test_available_datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@
import polars as pl
import pytest

from wetterdienst.metadata.cache import CacheExpiry
from wetterdienst.provider.dwd.observation.metadata import (
DwdObservationMetadata,
)
from wetterdienst.util.cache import CacheExpiry
from wetterdienst.util.network import HTTPFileSystem

SKIP_DATASETS = (
Expand Down
2 changes: 1 addition & 1 deletion tests/provider/dwd/radar/test_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import pytest
from fsspec.implementations.http import HTTPFileSystem

from wetterdienst.provider.dwd.metadata.datetime import DatetimeFormat
from wetterdienst.provider.dwd.metadata import DatetimeFormat
from wetterdienst.provider.dwd.radar.util import RADAR_DT_PATTERN, get_date_from_filename, verify_hdf5

HDF5_EXAMPLE = (
Expand Down
2 changes: 1 addition & 1 deletion tests/provider/dwd/road/test_api.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import pytest

from wetterdienst import Settings
from wetterdienst.metadata.cache import CacheExpiry
from wetterdienst.provider.dwd.road.api import DwdRoadRequest, DwdRoadStationGroup
from wetterdienst.util.cache import CacheExpiry
from wetterdienst.util.eccodes import ensure_eccodes, ensure_pdbufr
from wetterdienst.util.network import list_remote_files_fsspec

Expand Down
2 changes: 1 addition & 1 deletion tests/provider/dwd/test_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@
import pytest

from wetterdienst import Period
from wetterdienst.metadata.cache import CacheExpiry
from wetterdienst.provider.dwd.observation.fileindex import _create_file_index_for_dwd_server, build_path_to_parameter
from wetterdienst.provider.dwd.observation.metadata import (
DwdObservationMetadata,
)
from wetterdienst.settings import Settings
from wetterdienst.util.cache import CacheExpiry
from wetterdienst.util.network import list_remote_files_fsspec


Expand Down
2 changes: 1 addition & 1 deletion tests/util/test_network.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Copyright (C) 2018-2021, earthobservations developers.
# Distributed under the MIT License. See LICENSE for more info.
from wetterdienst.metadata.cache import CacheExpiry
from wetterdienst.settings import Settings
from wetterdienst.util.cache import CacheExpiry
from wetterdienst.util.network import NetworkFilesystemManager


Expand Down
19 changes: 18 additions & 1 deletion uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 0 additions & 4 deletions wetterdienst/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,8 @@

from wetterdienst import boot
from wetterdienst.api import Wetterdienst
from wetterdienst.metadata.kind import Kind
from wetterdienst.metadata.parameter import Parameter
from wetterdienst.metadata.period import Period
from wetterdienst.metadata.provider import Provider
from wetterdienst.metadata.resolution import Resolution
from wetterdienst.settings import Settings

Expand Down Expand Up @@ -65,10 +63,8 @@ def to_dict(self):
"__version__",
"Author",
"Info",
"Kind",
"Parameter",
"Period",
"Provider",
"Resolution",
"Settings",
"Wetterdienst",
Expand Down
34 changes: 0 additions & 34 deletions wetterdienst/core/core.py

This file was deleted.

22 changes: 14 additions & 8 deletions wetterdienst/core/timeseries/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,12 @@
import logging
from collections.abc import Iterable
from dataclasses import dataclass
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Literal

from pydantic import BaseModel, Field, SkipValidation, field_validator
from pydantic_extra_types.timezone_name import (
TimeZoneName, # noqa: TCH002, needs to stay here for pydantic model to work
)

from wetterdienst.metadata.period import Period # noqa: TCH001, needs to stay here for pydantic model to work
from wetterdienst.metadata.resolution import Resolution # noqa: TCH001, needs to stay here for pydantic model to work
Expand Down Expand Up @@ -167,6 +170,15 @@ def __iter__(self) -> Iterator[DatasetModel]:


class MetadataModel(BaseModel):
name_short: str
name_english: str
name_local: str
country: str
copyright: str
url: str
kind: Literal["observation", "forecast"]
timezone: TimeZoneName
timezone_data: TimeZoneName | Literal["dynamic"]
resolutions: list[ResolutionModel]

def __getitem__(self, item: str | int):
Expand Down Expand Up @@ -199,13 +211,7 @@ def __getattr__(self, item):
):
return resolution
else:
available_resolutions = [
f"{resolution.name}/{resolution.name_original}"
if resolution.name != resolution.name_original
else resolution.name
for resolution in self.resolutions
]
raise AttributeError(f"'{item}'. Available resolutions: {', '.join(available_resolutions)}")
return super().__getattr__(item)

def __iter__(self) -> Iterator[ResolutionModel]:
return iter(self.resolutions)
Expand Down
28 changes: 1 addition & 27 deletions wetterdienst/core/timeseries/request.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
from abc import abstractmethod
from collections.abc import Sequence
from hashlib import sha256
from typing import TYPE_CHECKING
from zoneinfo import ZoneInfo

import numpy as np
Expand All @@ -17,7 +16,6 @@
from polars.exceptions import NoDataError
from rapidfuzz import fuzz, process

from wetterdienst.core.core import Core
from wetterdienst.core.timeseries.metadata import (
DatasetModel,
MetadataModel,
Expand All @@ -42,11 +40,6 @@
from wetterdienst.settings import Settings
from wetterdienst.util.python import to_list

if TYPE_CHECKING:
from wetterdienst.metadata.datarange import DataRange
from wetterdienst.metadata.kind import Kind
from wetterdienst.metadata.provider import Provider

try:
from backports.datetime_fromisoformat import MonkeyPatch
except ImportError:
Expand All @@ -69,34 +62,15 @@
_SETTINGS_TYPE = dict | Settings | None


class TimeseriesRequest(Core):
class TimeseriesRequest:
"""Core for stations_result information of a source"""

@property
@abstractmethod
def _provider(self) -> Provider:
"""Optional enumeration for multiple resolutions"""
pass

@property
@abstractmethod
def _kind(self) -> Kind:
"""Optional enumeration for multiple resolutions"""
pass

@property
@abstractmethod
def metadata(self) -> MetadataModel:
"""metadata model"""
pass

@property
@abstractmethod
def _data_range(self) -> DataRange:
"""State whether data from this provider is given in fixed data chunks
or has to be defined over start and end date"""
pass

@property
@abstractmethod
def _values(self):
Expand Down
19 changes: 6 additions & 13 deletions wetterdienst/core/timeseries/values.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
from wetterdienst.core.timeseries.unit import UnitConverter
from wetterdienst.metadata.columns import Columns
from wetterdienst.metadata.resolution import DAILY_AT_MOST, Frequency, Resolution
from wetterdienst.metadata.timezone import Timezone
from wetterdienst.util.logging import TqdmToLogger

if TYPE_CHECKING:
Expand Down Expand Up @@ -105,15 +104,9 @@ def _meta_fields(self) -> dict[str, str]:
# TODO: add data type (mosmix, observation, ...)

@property
def data_tz(self) -> str:
def timezone_data(self) -> str:
"""Timezone of the published data"""
return self._data_tz.value

@property
@abstractmethod
def _data_tz(self) -> Timezone:
"""Timezone enumeration of published data."""
pass
return self.sr.stations.metadata.timezone_data

def _adjust_start_end_date(
self,
Expand Down Expand Up @@ -263,10 +256,10 @@ def _create_empty_station_df(self, station_id: str, dataset: DatasetModel) -> pl
if not self.sr.start_date:
return pl.DataFrame(schema=self._meta_fields)

if self._data_tz == Timezone.DYNAMIC:
if self.timezone_data == "dynamic":
tzinfo = ZoneInfo(self._get_timezone_from_station(station_id))
else:
tzinfo = ZoneInfo(self.data_tz)
tzinfo = ZoneInfo(self.timezone_data)
start_date, end_date = self._adjust_start_end_date(
self.sr.start_date, self.sr.end_date, tzinfo, dataset.resolution.value
)
Expand Down Expand Up @@ -301,10 +294,10 @@ def _build_complete_df(self, df: pl.DataFrame, station_id: str, resolution: Reso
"""
if df.is_empty():
return df
if self._data_tz == Timezone.DYNAMIC:
if self.timezone_data == "dynamic":
tzinfo = ZoneInfo(self._get_timezone_from_station(station_id))
else:
tzinfo = ZoneInfo(self.data_tz)
tzinfo = ZoneInfo(self.timezone_data)
start_date, end_date = self._adjust_start_end_date(self.sr.start_date, self.sr.end_date, tzinfo, resolution)
base_df = self._get_base_df(start_date, end_date, resolution)
data = []
Expand Down
File renamed without changes.
Loading

0 comments on commit 09c7509

Please sign in to comment.