Tornadoplots, and further improvements to VolumetricAnalysis

CHANGELOG.md

@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#669](https://github.com/equinor/webviz-subsurface/pull/669) - New generic plugin to visualize tornado plots from a csv file of responses.
 
 ### Changed
+- [#681](https://github.com/equinor/webviz-subsurface/pull/681) - `VolumetricAnalysis` upgrades - added page with tornadoplots to `VolumetricAnalysis`, automatic computation of volumes
+from the water zone if the volumes from the full grid geometry are included, and possibility of computing NTG from facies.
 - [#661](https://github.com/equinor/webviz-subsurface/pull/661) - Moved existing clientside function to a general dash_clientside file to facilitate adding more functions later on.
 - [#658](https://github.com/equinor/webviz-subsurface/pull/658) - Refactored Tornado figure code to be more reusable. Improved the Tornado bar visualization, added table display
  and improved layout in relevant plugins.

webviz_subsurface/_components/tornado/_tornado_table.py

@@ -60,5 +60,5 @@ def columns(self) -> List[Dict]:
         ]
 
     @property
-    def as_plotly_table(self) -> dict:
+    def as_plotly_table(self) -> list:
         return self._table.iloc[::-1].to_dict("records")

webviz_subsurface/_models/inplace_volumes_model.py

@@ -1,10 +1,8 @@
 from typing import List, Optional, Dict, Any
 from pathlib import Path
-
+import warnings
 import numpy as np
 import pandas as pd
-from webviz_config.webviz_store import webvizstore
-from webviz_config.common_cache import CACHE
 from webviz_subsurface._datainput.fmu_input import find_sens_type
 from .ensemble_set_model import EnsembleSetModel
 
@@ -19,7 +17,7 @@ class InplaceVolumesModel:
         "SENSTYPE",
     ]
     POSSIBLE_SELECTORS = [
-        "FLUID",
+        "FLUID_ZONE",
         "SOURCE",
         "ENSEMBLE",
         "REAL",
@@ -32,74 +30,92 @@ class InplaceVolumesModel:
         "SENSTYPE",
     ]
 
+    VOLCOL_ORDER = [
+        "STOIIP",
+        "GIIP",
+        "ASSOCIATEDOIL",
+        "ASSOCIATEDGAS",
+        "BULK",
+        "NET",
+        "PORV",
+        "HCPV",
+    ]
+
     def __init__(
         self,
         volumes_table: pd.DataFrame,
         parameter_table: Optional[pd.DataFrame] = None,
-        drop_constants: bool = False,
+        non_net_facies: Optional[List[str]] = None,
+        drop_constants: bool = True,
     ):
-        self._parameters: List[str] = []
-        if parameter_table is not None:
-            self._prepare_parameter_data(parameter_table, drop_constants=drop_constants)
-
-        self._designrun = (
-            parameter_table is not None and "SENSNAME" in parameter_table.columns
+        self._parameters = []
+        self._sensitivities = []
+        self._sensrun = False
+        self._parameterdf = (
+            parameter_table if parameter_table is not None else pd.DataFrame()
         )
-        if self._designrun:
-            parameter_table["SENSTYPE"] = parameter_table.apply(
-                lambda row: find_sens_type(row.SENSCASE)
-                if not pd.isnull(row.SENSCASE)
-                else np.nan,
-                axis=1,
-            )
-        self._sensrun = self._designrun and (
-            parameter_table["SENSNAME"].nunique() > 1
-            or (
-                parameter_table["SENSNAME"].nunique() == 1
-                and parameter_table["SENSTYPE"].unique() != ["mc"]
+        selectors = [x for x in volumes_table.columns if x in self.POSSIBLE_SELECTORS]
+
+        # It is not yet supported to combine sources with different selectors
+        if volumes_table[selectors].isnull().values.any():
+            raise TypeError(
+                f"Selectors {[x for x in selectors if x not in ['ENSEMBLE', 'SOURCE', 'REAL']]} "
+                "needs to be defined for all sources"
             )
-        )
 
-        sens_params_table = (
-            parameter_table[self.SENS_COLUMNS] if self._designrun else None
-        )
+        # compute water zone volumes if total volumes are present
+        if any(col.endswith("_TOTAL") for col in volumes_table.columns):
+            volumes_table = self._compute_water_zone_volumes(volumes_table, selectors)
 
-        # TO-DO add code for computing water volumes if "TOTAL" is present
+        # stack dataframe on fluid zone and add fluid as column istead of a column suffix
         dfs = []
-        for fluid in ["OIL", "GAS"]:
-            selector_columns = [
-                x for x in volumes_table.columns if x in self.POSSIBLE_SELECTORS
+        for fluid in ["OIL", "GAS", "WATER"]:
+            fluid_columns = [
+                x for x in volumes_table.columns if x.endswith(f"_{fluid}")
             ]
-            fluid_columns = [x for x in volumes_table.columns if x.endswith(fluid)]
             if not fluid_columns:
                 continue
-            df = volumes_table[selector_columns + fluid_columns].copy()
+            df = volumes_table[selectors + fluid_columns].copy()
             df.columns = df.columns.str.replace(f"_{fluid}", "")
-            df["FLUID"] = fluid.lower()
+            df["FLUID_ZONE"] = fluid.lower()
+            # Rename PORE to PORV (PORE will be deprecated..)
+            if "PORE" in df:
+                df.rename(columns={"PORE": "PORV"}, inplace=True)
             dfs.append(df)
+        self._dataframe = pd.concat(dfs)
 
-        volumes_table = pd.concat(dfs)
+        # Set NET volumes based on facies if non_net_facies in input
+        if non_net_facies is not None and "FACIES" in self._dataframe:
+            self._dataframe["NET"] = self._dataframe["BULK"]
+            self._dataframe.loc[
+                self._dataframe["FACIES"].isin(non_net_facies), "NET"
+            ] = 0
 
-        # Rename PORE to PORV (PORE will be deprecated..)
-        if "PORE" in volumes_table:
-            volumes_table.rename(columns={"PORE": "PORV"}, inplace=True)
-
-        # Merge into one dataframe
-        self._dataframe = (
-            volumes_table
-            if sens_params_table is None
-            else pd.merge(volumes_table, sens_params_table, on=["ENSEMBLE", "REAL"])
-        )
-
-        if self._sensrun and self._dataframe["SENSNAME"].isnull().values.any():
-            df = self._dataframe
-            raise ValueError(
-                "Ensembles with and without sensitivity data mixed - this is not supported \n"
-                f"Sensitivity ensembles: {df.loc[~df['SENSNAME'].isnull()]['ENSEMBLE'].unique()} "
-                f"Non-sensitivity ensembles: {df.loc[df['SENSNAME'].isnull()]['ENSEMBLE'].unique()}"
+        # If parameters present check if the case is a sensitivity run
+        # and merge sensitivity columns into the dataframe
+        if parameter_table is not None:
+            self._parameterdf = self._prepare_parameter_data(
+                parameter_table, drop_constants
             )
+            self._parameters = [
+                x for x in self._parameterdf.columns if x not in self.SENS_COLUMNS
+            ]
+            if "SENSNAME" in self._parameterdf:
+                self._add_sensitivity_columns()
+                self._sensitivities = list(self._dataframe["SENSNAME"].unique())
+
+        # set column order
+        colorder = self.selectors + self.VOLCOL_ORDER
+        self._dataframe = self._dataframe[
+            [x for x in colorder if x in self._dataframe]
+            + [x for x in self._dataframe if x not in colorder]
+        ]
 
-        self.set_initial_property_columns()
+        self._dataframe.sort_values(by=["ENSEMBLE", "REAL"], inplace=True)
+
+        # compute and set property columns
+        self._set_initial_property_columns()
+        self._dataframe = self.compute_property_columns(self._dataframe)
 
     @property
     def dataframe(self) -> pd.DataFrame:
@@ -113,6 +129,10 @@ def parameter_df(self) -> pd.DataFrame:
     def sensrun(self) -> bool:
         return self._sensrun
 
+    @property
+    def sensitivities(self) -> List[str]:
+        return self._sensitivities
+
     @property
     def sources(self) -> List[str]:
         return sorted(list(self._dataframe["SOURCE"].unique()))
@@ -149,17 +169,46 @@ def responses(self) -> List[str]:
     def parameters(self) -> List[str]:
         return self._parameters
 
-    def set_initial_property_columns(self) -> None:
+    @staticmethod
+    def _compute_water_zone_volumes(
+        voldf: pd.DataFrame, selectors: list
+    ) -> pd.DataFrame:
+        """Compute water zone volumes by subtracting HC-zone volumes from
+        TOTAL volumes"""
+        supported_columns = ["BULK_TOTAL", "NET_TOTAL", "PORE_TOTAL", "PORV_TOTAL"]
+        # Format check
+        for src, df in voldf.groupby("SOURCE"):
+            volcols = [col for col in df if col not in selectors]
+            if not any(col in volcols for col in supported_columns):
+                continue
+            if df[volcols].isnull().values.any():
+                warnings.warn(
+                    f"WARNING: Cannot calculate water zone volumes for source {src}, "
+                    "due to wrong format in input volume file. \nTo ensure correct format "
+                    "use: https://equinor.github.io/fmu-tools/fmu.tools.rms.html#fmu.tools."
+                    "rms.volumetrics.merge_rms_volumetrics"
+                )
+                return voldf
+
+        for col in [x.replace("_TOTAL", "") for x in voldf if x in supported_columns]:
+            voldf[f"{col}_WATER"] = (
+                voldf[f"{col}_TOTAL"]
+                - voldf.get(f"{col}_OIL", 0)
+                - voldf.get(f"{col}_GAS", 0)
+            )
+        return voldf
+
+    def _set_initial_property_columns(self) -> None:
+        """Create list of properties that can be computed based on
+        available volume columns"""
         self._property_columns = []
-        # if Net not given, Net is equal to Bulk
-        net_column = "NET" if "NET" in self._dataframe else "BULK"
 
         if all(col in self._dataframe for col in ["NET", "BULK"]):
             self._property_columns.append("NTG")
-
-        if all(col in self._dataframe for col in [net_column, "PORV"]):
+        if all(col in self._dataframe for col in ["BULK", "PORV"]):
             self._property_columns.append("PORO")
-
+        if all(col in self._dataframe for col in ["NET", "PORV"]):
+            self._property_columns.append("PORO (net)")
         if all(col in self._dataframe for col in ["HCPV", "PORV"]):
             self._property_columns.append("SW")
 
@@ -168,33 +217,110 @@ def set_initial_property_columns(self) -> None:
                 pvt = "BO" if vol_column == "STOIIP" else "BG"
                 self._property_columns.append(pvt)
 
-        self._dataframe = self.compute_property_columns(self._dataframe)
+    def _add_sensitivity_columns(self) -> None:
+        """Add sensitivity information columns from the parameters to the
+        dataframe, and raise error if not all ensembles have sensitivity data"""
+
+        self._parameterdf["SENSTYPE"] = self._parameterdf.apply(
+            lambda row: find_sens_type(row.SENSCASE)
+            if not pd.isnull(row.SENSCASE)
+            else np.nan,
+            axis=1,
+        )
+        sens_params_table = self._parameterdf[self.SENS_COLUMNS]
+
+        sensruns = []
+        for _, df in sens_params_table.groupby("ENSEMBLE"):
+            is_sensrun = not df["SENSNAME"].isnull().values.all() and (
+                df["SENSNAME"].nunique() > 1
+                or (df["SENSNAME"].nunique() == 1 and df["SENSTYPE"].unique() != ["mc"])
+            )
+            sensruns.append(is_sensrun)
+        self._sensrun = all(sensruns)
+
+        # raise error if mixed ensemble types
+        if not self._sensrun and any(sensruns):
+            raise ValueError(
+                "Ensembles with and without sensitivity data mixed - this is not supported"
+            )
+
+        # Merge into one dataframe
+        self._dataframe = pd.merge(
+            self._dataframe, sens_params_table, on=["ENSEMBLE", "REAL"]
+        )
 
     def compute_property_columns(
         self, dframe: pd.DataFrame, properties: Optional[list] = None
     ) -> pd.DataFrame:
-
+        """Compute property columns. As default all property columns are computed,
+        but which properties to compute can be given as input"""
+        dframe = dframe.copy()
         properties = self.property_columns if properties is None else properties
 
-        # if NTG not given Net is equal to bulk
-        net_column = "NET" if "NET" in dframe.columns else "BULK"
-
         if "NTG" in properties:
-            dframe["NTG"] = dframe[net_column] / dframe["BULK"]
+            dframe["NTG"] = dframe["NET"] / dframe["BULK"]
         if "PORO" in properties:
-            dframe["PORO"] = dframe["PORV"] / dframe[net_column]
+            if "NET" in dframe.columns:
+                dframe["PORO (net)"] = dframe["PORV"] / dframe["NET"]
+            dframe["PORO"] = dframe["PORV"] / dframe["BULK"]
         if "SW" in properties:
             dframe["SW"] = 1 - (dframe["HCPV"] / dframe["PORV"])
         if "BO" in properties:
             dframe["BO"] = dframe["HCPV"] / dframe["STOIIP"]
         if "BG" in properties:
             dframe["BG"] = dframe["HCPV"] / dframe["GIIP"]
+        # nan is handled by plotly but not inf
+        dframe.replace(np.inf, np.nan, inplace=True)
+        return dframe
 
+    def get_df(
+        self,
+        filters: Optional[Dict[str, list]] = None,
+        groups: Optional[list] = None,
+        parameters: Optional[list] = None,
+        properties: Optional[list] = None,
+    ) -> pd.DataFrame:
+        """Function to retrieve a dataframe with volumetrics and properties. Parameters
+        can be added to the dataframe if parameters are available in the instance.
+        Filters are supported on dictionary form with 'column_name': [list ov values to keep].
+        The final dataframe can be grouped by giving in a list of columns to group on.
+        """
+        dframe = self.dataframe.copy()
+
+        groups = groups if groups is not None else []
+        filters = filters if filters is not None else {}
+        parameters = parameters if parameters is not None else []
+
+        if parameters and self.parameters:
+            columns = parameters + ["REAL", "ENSEMBLE"]
+            dframe = pd.merge(
+                dframe, self.parameter_df[columns], on=["REAL", "ENSEMBLE"]
+            )
+        if filters:
+            dframe = filter_df(dframe, filters)
+
+        prevent_sum_over = ["REAL", "ENSEMBLE", "SOURCE"]
+        if groups:
+            sum_over_groups = groups + [x for x in prevent_sum_over if x not in groups]
+
+            # Need to sum volume columns and take the average of parameter columns
+            aggregations = {x: "sum" for x in self.volume_columns}
+            aggregations.update({x: "mean" for x in parameters})
+
+            dframe = dframe.groupby(sum_over_groups).agg(aggregations).reset_index()
+            dframe = dframe.groupby(groups).mean().reset_index()
+
+        dframe = self.compute_property_columns(dframe, properties)
+        if "FLUID_ZONE" not in groups:
+            if not filters.get("FLUID_ZONE") == ["oil"]:
+                dframe["BO"] = "NA"
+            if not filters.get("FLUID_ZONE") == ["gas"]:
+                dframe["BG"] = "NA"
         return dframe
 
     def _prepare_parameter_data(
         self, parameter_table: pd.DataFrame, drop_constants: bool
-    ) -> None:
+    ) -> pd.DataFrame:
         """
         Different data preparations on the parameters, before storing them as an attribute.
         Option to drop parameters with constant values. Prefixes on parameters from GEN_KW
@@ -243,14 +369,19 @@ def _prepare_parameter_data(
         # Drop columns if duplicate names
         parameter_table = parameter_table.loc[:, ~parameter_table.columns.duplicated()]
 
-        self._parameterdf = parameter_table
-        self._parameters = [
-            x for x in parameter_table.columns if x not in self.SENS_COLUMNS
-        ]
+        return parameter_table
+
+
+def filter_df(dframe: pd.DataFrame, filters: dict) -> pd.DataFrame:
+    """
+    Filter dataframe using dictionary with form
+    'column_name': [list ov values to keep]
+    """
+    for filt, values in filters.items():
+        dframe = dframe.loc[dframe[filt].isin(values)]
+    return dframe
 
 
-@CACHE.memoize(timeout=CACHE.TIMEOUT)
-@webvizstore
 def extract_volumes(
     ensemble_set_model: EnsembleSetModel, volfolder: str, volfiles: Dict[str, Any]
 ) -> pd.DataFrame:

webviz_subsurface/plugins/_volumetric_analysis/controllers/__init__.py

@@ -1,3 +1,4 @@
 from .distribution_controllers import distribution_controllers
 from .selections_controllers import selections_controllers
 from .layout_controllers import layout_controllers
+from .export_data_controllers import export_data_controllers