Create derived predicates with static variables

.github/workflows/code-quality-main.yaml

@@ -13,10 +13,16 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Python
-        uses: actions/setup-python@v3
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install packages
+        run: |
+          pip install .[dev]
 
       - name: Run pre-commits
-        uses: pre-commit/action@v3.0.0
+        uses: pre-commit/action@v3.0.1

.github/workflows/code-quality-pr.yaml

@@ -16,10 +16,16 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Python
-        uses: actions/setup-python@v3
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install packages
+        run: |
+          pip install .[dev]
 
       - name: Find modified files
         id: file_changes
@@ -31,6 +37,6 @@ jobs:
         run: echo '${{ steps.file_changes.outputs.files}}'
 
       - name: Run pre-commits
-        uses: pre-commit/action@v3.0.0
+        uses: pre-commit/action@v3.0.1
         with:
           extra_args: --files ${{ steps.file_changes.outputs.files}}

.github/workflows/tests.yml

@@ -17,16 +17,16 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.10
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: "3.10"
 
       - name: Install packages
         run: |
-          pip install -e .[dev]
+          pip install .[dev]
 
       #----------------------------------------------
       #              run test suite

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ exclude: "to_organize"
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v5.0.0
     hooks:
       # list of supported hooks: https://pre-commit.com/hooks.html
       - id: trailing-whitespace

README.md

@@ -217,15 +217,18 @@ normal_spo2:
 Fields for a "plain" predicate:
 
 - `code` (required): Must be one of the following:
-  - a string with `//` sequence separating the column name and column value.
-  - a list of strings as above in the form of {any: \[???, ???, ...\]}, which will match any of the listed codes.
-  - a regex in the form of {regex: "???"}, which will match any code that matches that regular expression.
+  - a string matching values in a column named `code` (for `MEDS` only).
+  - a string with a `//` sequence separating the column name and the matching column value (for `ESGPT` only).
+  - a list of strings as above in the form of `{any: \[???, ???, ...\]}` (or the corresponding expanded indented `YAML` format), which will match any of the listed codes.
+  - a regex in the form of `{regex: "???"}` (or the corresponding expanded indented `YAML` format), which will match any code that matches that regular expression.
 - `value_min` (optional): Must be float or integer specifying the minimum value of the predicate, if the variable is presented as numerical values.
 - `value_max` (optional): Must be float or integer specifying the maximum value of the predicate, if the variable is presented as numerical values.
 - `value_min_inclusive` (optional): Must be a boolean specifying whether `value_min` is inclusive or not.
 - `value_max_inclusive` (optional): Must be a boolean specifying whether `value_max` is inclusive or not.
 - `other_cols` (optional): Must be a 1-to-1 dictionary of column name and column value, which places additional constraints on further columns.
 
+**Note**: For memory optimization, we strongly recommend using either the List of Values or Regular Expression formats whenever possible, especially when needing to match multiple values. Defining each code as an individual string will increase memory usage significantly, as each code generates a separate predicate column. Using a list or regex consolidates multiple matching codes under a single column, reducing the overall memory footprint.
+
 #### Derived Predicates
 
 "Derived" predicates combine existing "plain" predicates using `and` / `or` keywords and have exactly 1 required `expr` field: For instance, the following defines a predicate representing either death or discharge (by combining "plain" predicates of `death` and `discharge`):

docs/source/configuration.md

@@ -49,20 +49,29 @@ These configs consist of the following four fields:
   The field can additionally be a dictionary with either a `regex` key and the value being a regular
   expression (satisfied if the regular expression evaluates to True), or a `any` key and the value being a
   list of strings (satisfied if there is an occurrence for any code in the list).
+
+  **Note**: Each individual definition of `PlainPredicateConfig` and `code` will generate a separate predicate
+  column. Thus, for memory optimization, it is strongly recommended to match multiple values using either the
+  List of Values or Regular Expression formats whenever possible.
+
 - `value_min`: If specified, an observation will only satisfy this predicate if the occurrence of the
   underlying `code` with a reported numerical value that is either greater than or greater than or equal to
   `value_min` (with these options being decided on the basis of `value_min_inclusive`, where
   `value_min_inclusive=True` indicating that an observation satisfies this predicate if its value is greater
   than or equal to `value_min`, and `value_min_inclusive=False` indicating a greater than but not equal to
   will be used).
+
 - `value_max`: If specified, an observation will only satisfy this predicate if the occurrence of the
   underlying `code` with a reported numerical value that is either less than or less than or equal to
   `value_max` (with these options being decided on the basis of `value_max_inclusive`, where
   `value_max_inclusive=True` indicating that an observation satisfies this predicate if its value is less
   than or equal to `value_max`, and `value_max_inclusive=False` indicating a less than but not equal to
   will be used).
+
 - `value_min_inclusive`: See `value_min`
+
 - `value_max_inclusive`: See `value_max`
+
 - `other_cols`: This optional field accepts a 1-to-1 dictionary of column names to column values, and can be
   used to specify further constraints on other columns (ie., not `code`) for this predicate.
 

pyproject.toml

@@ -18,7 +18,7 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 dependencies = [
-    "polars == 1.5.*",
+    "polars >= 1.0.0, <= 1.17.1",
     "bigtree == 0.18.*",
     "ruamel.yaml == 0.18.*",
     "loguru == 0.7.*",
@@ -37,7 +37,7 @@ expand_shards = "aces.expand_shards:main"
 
 [project.optional-dependencies]
 dev = [
-  "pre-commit", "pytest", "pytest-cov", "pytest-subtests", "rootutils", "hypothesis"
+  "pre-commit<4", "pytest", "pytest-cov", "pytest-subtests", "rootutils", "hypothesis"
 ]
 profiling = ["psutil"]
 

src/aces/config.py

@@ -1074,7 +1074,7 @@ class TaskExtractorConfig:
         >>> config = TaskExtractorConfig(predicates=predicates, trigger=trigger, windows={})
         Traceback (most recent call last):
             ...
-        KeyError: "Missing 1 relationships:\\nDerived predicate 'foobar' references undefined predicate 'bar'"
+        KeyError: "Missing 1 relationships: Derived predicate 'foobar' references undefined predicate 'bar'"
 
         >>> predicates = {"foo": PlainPredicateConfig("foo")}
         >>> trigger = EventConfig("foo")
@@ -1166,6 +1166,7 @@ def load(cls, config_path: str | Path, predicates_path: str | Path = None) -> Ta
                                                     start_inclusive=True, end_inclusive=True, has={},
                                                     label=None, index_timestamp=None)},
                                 label_window=None, index_timestamp_window=None)
+
             >>> predicates_dict = {
             ...     "metadata": {'description': 'A test predicates file'},
             ...     "description": 'this is a test',
@@ -1195,6 +1196,83 @@ def load(cls, config_path: str | Path, predicates_path: str | Path = None) -> Ta
                                                     start_inclusive=True, end_inclusive=True, has={},
                                                     label=None, index_timestamp=None)},
                                 label_window=None, index_timestamp_window=None)
+
+            >>> config_dict = {
+            ...     "metadata": {'description': 'A test configuration file'},
+            ...     "description": 'this is a test for joining static and plain predicates',
+            ...     "patient_demographics": {"male": {"code": "MALE"}, "female": {"code": "FEMALE"}},
+            ...     "predicates": {"normal_male_lab_range": {"code": "LAB", "value_min": 0, "value_max": 100,
+            ...                  "value_min_inclusive": True, "value_max_inclusive": True},
+            ...                  "normal_female_lab_range": {"code": "LAB", "value_min": 0, "value_max": 90,
+            ...                  "value_min_inclusive": True, "value_max_inclusive": True},
+            ...                  "normal_lab_male": {"expr": "and(normal_male_lab_range, male)"},
+            ...                  "normal_lab_female": {"expr": "and(normal_female_lab_range, female)"}},
+            ...     "trigger": "_ANY_EVENT",
+            ...     "windows": {
+            ...         "start": {
+            ...             "start": None, "end": "trigger + 24h", "start_inclusive": True,
+            ...             "end_inclusive": True, "has": {"normal_lab_male": "(1, None)"},
+            ...         }
+            ...     },
+            ... }
+            >>> with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml") as f:
+            ...     config_path = Path(f.name)
+            ...     yaml.dump(config_dict, f)
+            ...     cfg = TaskExtractorConfig.load(config_path)
+            >>> cfg.predicates.keys() # doctest: +NORMALIZE_WHITESPACE
+            dict_keys(['normal_lab_male', 'normal_male_lab_range', 'female', 'male'])
+
+            >>> config_dict = {
+            ...     "metadata": {'description': 'A test configuration file'},
+            ...     "description": 'this is a test for nested derived predicates',
+            ...     "patient_demographics": {"male": {"code": "MALE"}, "female": {"code": "FEMALE"}},
+            ...     "predicates": {"abnormally_low_male_lab_range": {"code": "LAB", "value_max": 90,
+            ...                  "value_max_inclusive": False},
+            ...                  "abnormally_low_female_lab_range": {"code": "LAB", "value_max": 80,
+            ...                  "value_max_inclusive": False},
+            ...                  "abnormally_high_lab_range": {"code": "LAB", "value_min": 120,
+            ...                  "value_min_inclusive": False},
+            ...                  "abnormal_lab_male_range": {"expr":
+            ...                             "or(abnormally_low_male_lab_range, abnormally_high_lab_range)"},
+            ...                  "abnormal_lab_female_range": {"expr":
+            ...                             "or(abnormally_low_female_lab_range, abnormally_high_lab_range)"},
+            ...                  "abnormal_lab_male": {"expr": "and(abnormal_lab_male_range, male)"},
+            ...                  "abnormal_lab_female": {"expr": "and(abnormal_lab_female_range, female)"},
+            ...                  "abnormal_labs": {"expr": "or(abnormal_lab_male, abnormal_lab_female)"}},
+            ...     "trigger": "_ANY_EVENT",
+            ...     "windows": {
+            ...         "start": {
+            ...             "start": None, "end": "trigger + 24h", "start_inclusive": True,
+            ...             "end_inclusive": True, "has": {"abnormal_labs": "(1, None)"},
+            ...         }
+            ...     },
+            ... }
+            >>> with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml") as f:
+            ...     config_path = Path(f.name)
+            ...     yaml.dump(config_dict, f)
+            ...     cfg = TaskExtractorConfig.load(config_path)
+            >>> cfg.predicates.keys() # doctest: +NORMALIZE_WHITESPACE
+            dict_keys(['abnormal_lab_female', 'abnormal_lab_female_range', 'abnormal_lab_male',
+            'abnormal_lab_male_range', 'abnormal_labs', 'abnormally_high_lab_range',
+            'abnormally_low_female_lab_range', 'abnormally_low_male_lab_range', 'female', 'male'])
+
+            >>> predicates_dict = {
+            ...     "metadata": {'description': 'A test predicates file'},
+            ...     "description": 'this is a test',
+            ...     "patient_demographics": {"brown_eyes": {"code": "eye_color//BR"}},
+            ...     "predicates": {'admission': "invalid"},
+            ... }
+            >>> with (tempfile.NamedTemporaryFile(mode="w", suffix=".yaml") as config_fp,
+            ...      tempfile.NamedTemporaryFile(mode="w", suffix=".yaml") as pred_fp):
+            ...     config_path = Path(config_fp.name)
+            ...     pred_path = Path(pred_fp.name)
+            ...     yaml.dump(no_predicates_config, config_fp)
+            ...     yaml.dump(predicates_dict, pred_fp)
+            ...     cfg = TaskExtractorConfig.load(config_path, pred_path) # doctest: +NORMALIZE_WHITESPACE
+            Traceback (most recent call last):
+                ...
+            ValueError: Predicate 'admission' is not defined correctly in the configuration file. Currently
+            defined as the string: invalid. Please refer to the documentation for the supported formats.
         """
         if isinstance(config_path, str):
             config_path = Path(config_path)
@@ -1258,6 +1336,7 @@ def load(cls, config_path: str | Path, predicates_path: str | Path = None) -> Ta
 
         final_predicates = {**predicates, **overriding_predicates}
         final_demographics = {**patient_demographics, **overriding_demographics}
+        all_predicates = {**final_predicates, **final_demographics}
 
         logger.info("Parsing windows...")
         if windows is None:
@@ -1271,30 +1350,58 @@ def load(cls, config_path: str | Path, predicates_path: str | Path = None) -> Ta
         logger.info("Parsing trigger event...")
         trigger = EventConfig(trigger)
 
+        # add window referenced predicates
         referenced_predicates = {pred for w in windows.values() for pred in w.referenced_predicates}
+
+        # add trigger predicate
         referenced_predicates.add(trigger.predicate)
+
+        # add label predicate if it exists and not already added
         label_reference = [w.label for w in windows.values() if w.label]
         if label_reference:
             referenced_predicates.update(set(label_reference))
-        current_predicates = set(referenced_predicates)
+
         special_predicates = {ANY_EVENT_COLUMN, START_OF_RECORD_KEY, END_OF_RECORD_KEY}
-        for pred in current_predicates - special_predicates:
-            if pred not in final_predicates:
+        for pred in set(referenced_predicates) - special_predicates:
+            if pred not in all_predicates:
                 raise KeyError(
-                    f"Something referenced predicate {pred} that wasn't defined in the configuration."
-                )
-            if "expr" in final_predicates[pred]:
-                referenced_predicates.update(
-                    DerivedPredicateConfig(**final_predicates[pred]).input_predicates
+                    f"Something referenced predicate '{pred}' that wasn't defined in the configuration."
                 )
 
+            if "expr" in all_predicates[pred]:
+                stack = list(DerivedPredicateConfig(**all_predicates[pred]).input_predicates)
+
+                while stack:
+                    nested_pred = stack.pop()
+
+                    if nested_pred not in all_predicates:
+                        raise KeyError(
+                            f"Predicate '{nested_pred}' referenced in '{pred}' is not defined in the "
+                            "configuration."
+                        )
+
+                    # if nested_pred is a DerivedPredicateConfig, unpack input_predicates and add to stack
+                    if "expr" in all_predicates[nested_pred]:
+                        derived_config = DerivedPredicateConfig(**all_predicates[nested_pred])
+                        stack.extend(derived_config.input_predicates)
+                        referenced_predicates.add(nested_pred)  # also add itself to referenced_predicates
+                    else:
+                        # if nested_pred is a PlainPredicateConfig, only add it to referenced_predicates
+                        referenced_predicates.add(nested_pred)
+
         logger.info("Parsing predicates...")
         predicates_to_parse = {k: v for k, v in final_predicates.items() if k in referenced_predicates}
         predicate_objs = {}
         for n, p in predicates_to_parse.items():
             if "expr" in p:
                 predicate_objs[n] = DerivedPredicateConfig(**p)
             else:
+                if isinstance(p, str):
+                    raise ValueError(
+                        f"Predicate '{n}' is not defined correctly in the configuration file. "
+                        f"Currently defined as the string: {p}. "
+                        "Please refer to the documentation for the supported formats."
+                    )
                 config_data = {k: v for k, v in p.items() if k in PlainPredicateConfig.__dataclass_fields__}
                 other_cols = {k: v for k, v in p.items() if k not in config_data}
                 predicate_objs[n] = PlainPredicateConfig(**config_data, other_cols=other_cols)
@@ -1344,7 +1451,7 @@ def _initialize_predicates(self):
                 )
         if missing_predicates:
             raise KeyError(
-                f"Missing {len(missing_predicates)} relationships:\n" + "\n".join(missing_predicates)
+                f"Missing {len(missing_predicates)} relationships: " + "; ".join(missing_predicates)
             )
 
         self._predicate_dag_graph = nx.DiGraph(dag_relationships)