📊 WDI update (#3897)

* 📊 WDI update

apps/wizard/app_pages/chart_diff/chart_diff.py

@@ -248,7 +248,7 @@ def from_charts_df(
         checksums_diff = cls._get_checksums(source_session, target_session, chart_ids)
 
         # Get all slugs from target
-        slugs_in_target = cls._get_chart_slugs(target_session)
+        slugs_in_target = cls._get_chart_slugs(target_session, slugs={c.slug for c in source_charts.values()})  # type: ignore
 
         # Build chart diffs
         chart_diffs = []
@@ -374,9 +374,19 @@ def details(self):
         }
 
     @staticmethod
-    def _get_chart_slugs(target_session: Session) -> set[str]:
-        slugs_redirects = set(read_sql("SELECT slug FROM chart_slug_redirects", target_session)["slug"])
-        slugs = set(read_sql("SELECT slug FROM chart_configs", target_session)["slug"])
+    def _get_chart_slugs(target_session: Session, slugs: Optional[set[str]] = None) -> set[str]:
+        """Get all chart slugs. Use `slugs` to filter slugs as this can be slow otherwise."""
+        if slugs is not None:
+            where = "WHERE slug IN %(slugs)s"
+            params = {"slugs": tuple(slugs)}
+        else:
+            where = ""
+            params = {}
+
+        slugs_redirects = set(
+            read_sql(f"SELECT slug FROM chart_slug_redirects {where}", target_session, params=params)["slug"]
+        )
+        slugs = set(read_sql(f"SELECT slug FROM chart_configs {where}", target_session, params=params)["slug"])
         return slugs | slugs_redirects
 
     @staticmethod

apps/wizard/utils/components.py

@@ -242,7 +242,14 @@ def st_show(item):
 
     """
 
-    def __init__(self, items: list[Any], items_per_page: int, pagination_key: str, on_click: Optional[Callable] = None):
+    def __init__(
+        self,
+        items: list[Any],
+        items_per_page: int,
+        pagination_key: str,
+        on_click: Optional[Callable] = None,
+        save_in_query: bool = False,
+    ):
         """Construct Pagination.
 
         Parameters
@@ -255,16 +262,23 @@ def __init__(self, items: list[Any], items_per_page: int, pagination_key: str, o
             Key to store the current page in session state.
         on_click : Optional[Callable], optional
             Action to perform when interacting with any of the buttons, by default None
+        save_in_query : bool, optional
+            Whether to save the current page in the query string, by default False
         """
         self.items = items
         self.items_per_page = items_per_page
         self.pagination_key = pagination_key
+        self.save_in_query = save_in_query
         # Action to perform when interacting with any of the buttons.
         ## Example: Change the value of certain state in session_state
         self.on_click = on_click
         # Initialize session state for the current page
         if self.pagination_key not in st.session_state:
-            self.page = 1
+            # Get page from query parameters
+            if self.save_in_query and self.pagination_key in st.query_params:
+                self.page = int(st.query_params[self.pagination_key])
+            else:
+                self.page = 1
 
     @property
     def page(self):
@@ -329,6 +343,11 @@ def show_controls_buttons(self):
     def show_controls_bar(self) -> None:
         def _change_page():
             # Internal action
+            if self.save_in_query:
+                if self.page == 1:
+                    st.query_params.pop(self.pagination_key)
+                else:
+                    st.query_params.update({self.pagination_key: self.page})
 
             # External action
             if self.on_click is not None:

dag/main.yml

@@ -505,6 +505,7 @@ steps:
     - data://garden/news/2024-05-23/gdelt_v2
 
   # World Development Indicators - WDI
+  # TO BE ARCHIVED
   data://meadow/worldbank_wdi/2024-05-20/wdi:
     - snapshot://worldbank_wdi/2024-05-20/wdi.zip
 
@@ -518,6 +519,20 @@ steps:
   data://grapher/worldbank_wdi/2024-05-20/wdi:
     - data://garden/worldbank_wdi/2024-05-20/wdi
 
+  # World Development Indicators - WDI
+  data://meadow/worldbank_wdi/2025-01-24/wdi:
+    - snapshot://worldbank_wdi/2025-01-24/wdi.zip
+
+  data://garden/worldbank_wdi/2025-01-24/wdi:
+    - snapshot://worldbank_wdi/2025-01-24/wdi.zip
+    - data://meadow/worldbank_wdi/2025-01-24/wdi
+    - data://garden/demography/2024-07-15/population
+    - data://garden/regions/2023-01-01/regions
+    - data://garden/wb/2024-07-29/income_groups
+
+  data://grapher/worldbank_wdi/2025-01-24/wdi:
+    - data://garden/worldbank_wdi/2025-01-24/wdi
+
   #
   # Aviation Safety Network - Aviation Statistics.
   #

etl/helpers.py

@@ -673,7 +673,8 @@ def get_dependency_step_name(
             short_name=short_name,
             is_private=is_private,
         )
-        matches = [dependency for dependency in self.dependencies if bool(re.match(pattern, dependency))]
+        deps = self.dependencies
+        matches = _match_dependencies(pattern, deps)
 
         # If no step was found and is_private was not specified, try again assuming step is private.
         if (len(matches) == 0) and (is_private is None):
@@ -685,7 +686,7 @@ def get_dependency_step_name(
                 short_name=short_name,
                 is_private=True,
             )
-            matches = [dependency for dependency in self.dependencies if bool(re.match(pattern, dependency))]
+            matches = _match_dependencies(pattern, self.dependencies)
 
         # If not step was found and channel is "grapher", try again assuming this is a grapher://grapher step.
         if (len(matches) == 0) and (channel == "grapher"):
@@ -697,7 +698,7 @@ def get_dependency_step_name(
                 short_name=short_name,
                 is_private=is_private,
             )
-            matches = [dependency for dependency in self.dependencies if bool(re.match(pattern, dependency))]
+            matches = _match_dependencies(pattern, self.dependencies)
 
         if len(matches) == 0:
             raise NoMatchingStepsAmongDependencies(step_name=self.step_name)
@@ -783,6 +784,11 @@ def load_mdim_config(self, filename: Optional[str] = None, path: Optional[str |
         return config
 
 
+def _match_dependencies(pattern: str, dependencies: List[str]) -> List[str]:
+    regex = re.compile(pattern)
+    return [dependency for dependency in dependencies if regex.match(dependency)]
+
+
 def print_tables_metadata_template(tables: List[Table], fields: Optional[List[str]] = None) -> None:
     # This function is meant to be used when creating code in an interactive window (or a notebook).
     # It prints a template for the metadata of the tables in the list.