Feat dataset refactor api (#43)

* change dataset_type into separate table

* add fixture for unit table

* fix API response structure

* refactor base dataset reader

* combine forecast and historical into 1 api

* add test for data reader

* add providers filter in measurement api

* fix lint

* update database diagram with dataset_type table

django_project/gap/admin.py

@@ -8,10 +8,20 @@
 
 from .models import (
  Attribute, Country, Provider, Measurement, Station, IngestorSession,
- Dataset, DatasetAttribute, NetCDFFile
+ Dataset, DatasetAttribute, NetCDFFile, DatasetType, Unit
 )
 
 
+@admin.register(Unit)
+class UnitAdmin(admin.ModelAdmin):
+ """Unit admin."""
+
+ list_display = (
+ 'name', 'description'
+ )
+ search_fields = ('name',)
+
+
 @admin.register(Attribute)
 class AttributeAdmin(admin.ModelAdmin):
  """Attribute admin."""
@@ -42,6 +52,15 @@ class ProviderAdmin(admin.ModelAdmin):
  search_fields = ('name',)
 
 
+@admin.register(DatasetType)
+class DatasetTypeAdmin(admin.ModelAdmin):
+ """DatasetType admin."""
+
+ list_display = (
+ 'name', 'type'
+ )
+
+
 @admin.register(Dataset)
 class DatasetAdmin(admin.ModelAdmin):
  """Dataset admin."""
@@ -58,6 +77,7 @@ class DatasetAttributeAdmin(admin.ModelAdmin):
  list_display = (
  'dataset', 'attribute', 'source', 'source_unit',
  )
+ list_filter = ('dataset',)
 
 
 @admin.register(Measurement)

django_project/gap/factories.py

@@ -9,6 +9,8 @@
 
 from core.factories import BaseMetaFactory, BaseFactory
 from gap.models import (
+ CastType,
+ DatasetType,
  Dataset,
  Provider,
  Unit,
@@ -18,7 +20,6 @@
  Station,
  Measurement,
  ObservationType,
- DatasetType,
  DatasetTimeStep,
  DatasetStore,
  NetCDFFile
@@ -37,6 +38,19 @@ class Meta: # noqa
  description = factory.Faker('text')
 
 
+class DatasetTypeFactory(
+ BaseFactory[DatasetType], metaclass=BaseMetaFactory[DatasetType]
+):
+ """Factory class for DatasetType model."""
+
+ class Meta: # noqa
+ model = DatasetType
+
+ name = factory.Faker('company')
+ description = factory.Faker('text')
+ type = CastType.HISTORICAL
+
+
 class DatasetFactory(
  BaseFactory[Dataset], metaclass=BaseMetaFactory[Dataset]
 ):
@@ -47,7 +61,7 @@ class Meta: # noqa
 
  name = factory.Faker('company')
  description = factory.Faker('text')
- type = DatasetType.CLIMATE_REANALYSIS
+ type = factory.SubFactory(DatasetTypeFactory)
  time_step = DatasetTimeStep.DAILY
  store_type = DatasetStore.TABLE
  provider = factory.SubFactory(ProviderFactory)

django_project/gap/fixtures/2.dataset_type.json

@@ -0,0 +1,47 @@
+[
+{
+ "model": "gap.datasettype",
+ "pk": 1,
+ "fields": {
+ "name": "Climate Reanalysis",
+ "description": "",
+ "type": "historical"
+ }
+},
+{
+ "model": "gap.datasettype",
+ "pk": 2,
+ "fields": {
+ "name": "Short-term Forecast",
+ "description": "",
+ "type": "forecast"
+ }
+},
+{
+ "model": "gap.datasettype",
+ "pk": 3,
+ "fields": {
+ "name": "Seasonal Forecast",
+ "description": "",
+ "type": "forecast"
+ }
+},
+{
+ "model": "gap.datasettype",
+ "pk": 4,
+ "fields": {
+ "name": "Ground Observational",
+ "description": "",
+ "type": "historical"
+ }
+},
+{
+ "model": "gap.datasettype",
+ "pk": 5,
+ "fields": {
+ "name": "Airborne Observational",
+ "description": "",
+ "type": "historical"
+ }
+}
+]

django_project/gap/fixtures/3.unit.json

@@ -0,0 +1,74 @@
+[
+{
+ "model": "gap.unit",
+ "pk": 1,
+ "fields": {
+ "name": "mm",
+ "description": null
+ }
+},
+{
+ "model": "gap.unit",
+ "pk": 3,
+ "fields": {
+ "name": "MJ/sqm",
+ "description": null
+ }
+},
+{
+ "model": "gap.unit",
+ "pk": 4,
+ "fields": {
+ "name": "mm day-1",
+ "description": null
+ }
+},
+{
+ "model": "gap.unit",
+ "pk": 5,
+ "fields": {
+ "name": "atm",
+ "description": null
+ }
+},
+{
+ "model": "gap.unit",
+ "pk": 6,
+ "fields": {
+ "name": "g/m3",
+ "description": null
+ }
+},
+{
+ "model": "gap.unit",
+ "pk": 7,
+ "fields": {
+ "name": "W/m2",
+ "description": null
+ }
+},
+{
+ "model": "gap.unit",
+ "pk": 8,
+ "fields": {
+ "name": "°C",
+ "description": null
+ }
+},
+{
+ "model": "gap.unit",
+ "pk": 9,
+ "fields": {
+ "name": "Degrees from North",
+ "description": null
+ }
+},
+{
+ "model": "gap.unit",
+ "pk": 10,
+ "fields": {
+ "name": "m/s",
+ "description": null
+ }
+}
+]

django_project/gap/ingestor/tahmo.py

@@ -18,7 +18,7 @@
 from gap.models import (
  Provider, Station, ObservationType, Country, IngestorSession,
  Attribute, Measurement, Dataset, DatasetType, DatasetTimeStep,
- DatasetStore, DatasetAttribute, Unit
+ DatasetStore, DatasetAttribute, Unit, CastType
 )
 
 
@@ -58,10 +58,16 @@ def __init__(self, session: IngestorSession):
  self.obs_type, _ = ObservationType.objects.get_or_create(
  name='Ground Observations'
  )
+ self.dataset_type, _ = DatasetType.objects.get_or_create(
+ name='Ground Observational',
+ defaults={
+ 'type': CastType.HISTORICAL
+ }
+ )
  self.dataset, _ = Dataset.objects.get_or_create(
- name='Tahmo',
+ name=f'Tahmo {self.dataset_type.name}',
  provider=self.provider,
- type=DatasetType.GROUND_OBSERVATIONAL,
+ type=self.dataset_type,
  time_step=DatasetTimeStep.DAILY,
  store_type=DatasetStore.TABLE
  )

django_project/gap/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2.7 on 2024-07-05 15:16
+# Generated by Django 4.2.7 on 2024-07-14 21:07
 
 import django.contrib.gis.db.models.fields
 from django.db import migrations, models
@@ -45,14 +45,25 @@ class Migration(migrations.Migration):
  ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
  ('name', models.CharField(max_length=512)),
  ('description', models.TextField(blank=True, null=True)),
- ('type', models.CharField(choices=[('Climate Reanalysis', 'Climate Reanalysis'), ('Short-term Forecast', 'Short-term Forecast'), ('Seasonal Forecast', 'Seasonal Forecast'), ('Ground Observational', 'Ground Observational'), ('Airborne Observational', 'Airborne Observational')], max_length=512)),
  ('time_step', models.CharField(choices=[('DAILY', 'DAILY'), ('HOURLY', 'HOURLY')], max_length=512)),
  ('store_type', models.CharField(choices=[('TABLE', 'TABLE'), ('NETCDF', 'NETCDF'), ('EXT_API', 'EXT_API')], max_length=512)),
  ],
  options={
  'abstract': False,
  },
  ),
+ migrations.CreateModel(
+ name='DatasetType',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=512)),
+ ('description', models.TextField(blank=True, null=True)),
+ ('type', models.CharField(choices=[('historical', 'historical'), ('forecast', 'forecast')], max_length=512)),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
  migrations.CreateModel(
  name='IngestorSession',
  fields=[
@@ -140,6 +151,11 @@ class Migration(migrations.Migration):
  name='provider',
  field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gap.provider'),
  ),
+ migrations.AddField(
+ model_name='dataset',
+ name='type',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gap.datasettype'),
+ ),
  migrations.AddField(
  model_name='attribute',
  name='unit',

django_project/gap/models/dataset.py

@@ -11,14 +11,23 @@
 from gap.models.common import Provider
 
 
-class DatasetType:
+class CastType:
+ """Cast type."""
+
+ HISTORICAL = 'historical'
+ FORECAST = 'forecast'
+
+
+class DatasetType(Definition):
  """Dataset type."""
 
- CLIMATE_REANALYSIS = 'Climate Reanalysis'
- SHORT_TERM_FORECAST = 'Short-term Forecast'
- SEASONAL_FORECAST = 'Seasonal Forecast'
- GROUND_OBSERVATIONAL = 'Ground Observational'
- AIRBORNE_OBSERVATIONAL = 'Airborne Observational'
+ type = models.CharField(
+ choices=(
+ (CastType.HISTORICAL, CastType.HISTORICAL),
+ (CastType.FORECAST, CastType.FORECAST),
+ ),
+ max_length=512
+ )
 
 
 class DatasetStore:
@@ -42,21 +51,8 @@ class Dataset(Definition):
  provider = models.ForeignKey(
  Provider, on_delete=models.CASCADE
  )
- type = models.CharField(
- choices=(
- (DatasetType.CLIMATE_REANALYSIS, DatasetType.CLIMATE_REANALYSIS),
- (DatasetType.SHORT_TERM_FORECAST, DatasetType.SHORT_TERM_FORECAST),
- (DatasetType.SEASONAL_FORECAST, DatasetType.SEASONAL_FORECAST),
- (
- DatasetType.GROUND_OBSERVATIONAL,
- DatasetType.GROUND_OBSERVATIONAL
- ),
- (
- DatasetType.AIRBORNE_OBSERVATIONAL,
- DatasetType.AIRBORNE_OBSERVATIONAL
- ),
- ),
- max_length=512
+ type = models.ForeignKey(
+ DatasetType, on_delete=models.CASCADE
  )
  time_step = models.CharField(
  choices=(

django_project/gap/providers/__init__.py

@@ -0,0 +1,33 @@
+# coding=utf-8
+"""
+Tomorrow Now GAP.
+
+.. note:: Helper for reading NetCDF File
+"""
+
+from gap.models import Dataset
+from gap.utils.netcdf import NetCDFProvider
+from gap.providers.cbam import CBAMNetCDFReader
+from gap.providers.salient import SalientNetCDFReader
+from gap.providers.tahmo import TahmoDatasetReader
+
+
+def get_reader_from_dataset(dataset: Dataset):
+ """Create a new Reader from given dataset.
+
+ :param dataset: Dataset to be read
+ :type dataset: Dataset
+ :raises TypeError: if provider is neither CBAM or Salient
+ :return: Reader Class Type
+ :rtype: CBAMNetCDFReader|SalientNetCDFReader
+ """
+ if dataset.provider.name == NetCDFProvider.CBAM:
+ return CBAMNetCDFReader
+ elif dataset.provider.name == NetCDFProvider.SALIENT:
+ return SalientNetCDFReader
+ elif dataset.provider.name == 'Tahmo':
+ return TahmoDatasetReader
+ else:
+ raise TypeError(
+ f'Unsupported provider name: {dataset.provider.name}'
+ )