Skip to content
This repository has been archived by the owner on Jan 10, 2025. It is now read-only.

Commit

Permalink
fix concat
Browse files Browse the repository at this point in the history
  • Loading branch information
floriankrb committed Feb 19, 2024
1 parent 7e3d9d1 commit 28547f7
Show file tree
Hide file tree
Showing 3 changed files with 16 additions and 10 deletions.
1 change: 0 additions & 1 deletion ecml_tools/create/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,6 @@ def cleanup(self):
)
loader.run()


def patch(self, **kwargs):
from .patch import apply_patch

Expand Down
22 changes: 14 additions & 8 deletions ecml_tools/create/input.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ def grid_points(self):
def dates(self):
if self._dates is None:
raise ValueError(f"No dates for {self}")
assert hasattr(self._dates, "values"), (type(self), self._dates)
return self._dates.values

@cached_property
Expand Down Expand Up @@ -523,15 +524,15 @@ def _trace_select(self, dates):


class ConcatResult(Result):
def __init__(self, context, action_path, results):
super().__init__(context, action_path, dates=None)
def __init__(self, context, action_path, dates, results, **kwargs):
super().__init__(context, action_path, dates)
self.results = [r for r in results if not r.empty]

@cached_property
@check_references
@trace_datasource
def datasource(self):
ds = EmptyResult(self.context, None, self.dates).datasource
ds = EmptyResult(self.context, self.action_path, self._dates).datasource
for i in self.results:
ds += i.datasource
assert_is_fieldset(ds), i
Expand Down Expand Up @@ -689,7 +690,12 @@ def __init__(self, context, action_path, previous_step, *args, **kwargs):
class ConcatAction(ActionWithList):
@trace_select
def select(self, dates):
return ConcatResult(self.context, [a.select(dates) for a in self.actions])
return ConcatResult(
self.context,
self.action_path,
dates,
[a.select(dates) for a in self.actions],
)


class JoinAction(ActionWithList):
Expand All @@ -715,18 +721,18 @@ def __init__(self, context, action_path, **kwargs):
else:
subconfig[k] = v

self._dates = build_groups(datesconfig)
self.filtering_dates = build_groups(datesconfig)
self.content = action_factory(subconfig, context, self.action_path + ["dates"])

@trace_select
def select(self, dates):
newdates = self._dates.intersect(dates)
newdates = self.filtering_dates.intersect(dates)
if newdates.empty():
return EmptyResult(self.context, None, newdates)
return EmptyResult(self.context, self.action_path, newdates)
return self.content.select(newdates)

def __repr__(self):
return super().__repr__(f"{self._dates}\n{self.content}")
return super().__repr__(f"{self.filtering_dates}\n{self.content}")


def merge_dicts(a, b):
Expand Down
3 changes: 2 additions & 1 deletion ecml_tools/create/loaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -576,7 +576,8 @@ def add_total_size(self):

self.update_metadata(total_size=size, total_number_of_files=n)


class CleanupLoader(Loader):
def run(self):
self.statistics_registry.delete()
self.registry.clean()
self.registry.clean()

0 comments on commit 28547f7

Please sign in to comment.