Merge pull request #296 from LDAR-Sim/change_startup_rolling_average_…

…logic

Change startup rolling average logic

LDAR_Sim/src/file_processing/output_processing/program_output.py

@@ -51,6 +51,9 @@ def gen_estimated_repairable_emissions_to_remove(
     # Drop rows with missing repair dates and missing measured rates
     fugitive_emissions_rates_and_repair_dates.dropna(inplace=True)
 
+    if fugitive_emissions_rates_and_repair_dates.empty:
+        return pd.DataFrame()
+
     # Populate a new column in the fugitive emissions rates and repair dates dataframe
     # with the closest future survey date. This wil be used to compute the estimated
     # fugitive emissions to remove to avoid double counting
@@ -170,6 +173,9 @@ def gen_estimated_comp_emissions_report(
         site_survey_reports_summary[eca.COMP].notnull()
     ]
 
+    if comp_reports.empty:
+        return
+
     # Get unique combinations of site_ID, equipment and component
     unique_combinations = comp_reports[[eca.SITE_ID, eca.EQG, eca.COMP]].drop_duplicates()
     unique_site_survey_dates = comp_reports[
@@ -266,6 +272,8 @@ def gen_estimated_comp_emissions_report(
 
 
 def determine_start_and_end_dates(sorted_by_site_summary_df, group_by_summary, duration_factor):
+    if sorted_by_site_summary_df.empty:
+        return sorted_by_site_summary_df
     # Determine if the previous or next condition should be used for the emission rate
     sorted_by_site_summary_df = program_output_helpers.calculate_prev_condition(
         sorted_by_site_summary_df, group_by_summary
@@ -276,12 +284,20 @@ def determine_start_and_end_dates(sorted_by_site_summary_df, group_by_summary, d
     # Set the estimated start/end date based on the emission rate condition
     sorted_by_site_summary_df[eca.START_DATE] = (
         sorted_by_site_summary_df.groupby(eca.SITE_ID)
-        .apply(lambda x: program_output_helpers.calculate_start_date(x, duration_factor))
+        .apply(
+            lambda x: pd.DataFrame(
+                program_output_helpers.calculate_start_date(x, duration_factor), index=x.index
+            )
+        )
         .reset_index(drop=True)
     )
     sorted_by_site_summary_df[eca.END_DATE] = (
         sorted_by_site_summary_df.groupby(eca.SITE_ID)
-        .apply(lambda x: program_output_helpers.calculate_end_date(x, duration_factor))
+        .apply(
+            lambda x: pd.DataFrame(
+                program_output_helpers.calculate_end_date(x, duration_factor), index=x.index
+            )
+        )
         .reset_index(drop=True)
     )
     return sorted_by_site_summary_df

LDAR_Sim/src/file_processing/output_processing/summary_output_helpers.py

@@ -92,6 +92,9 @@ def get_yearly_value_for_multi_day_stat(
         if end_date is pd.NaT:
             end_date = df[end_date_col].max()
             end_year: int = end_date.year
+            if end_date is pd.NaT:
+                end_year: int = year
+
             end_date = pd.Timestamp("-".join([str(end_year), "12", "31"]))
         else:
             end_year: int = end_date.year

LDAR_Sim/src/programs/site_level_method.py

@@ -131,17 +131,10 @@ def update_stationary(self, date_to_check, detection_record):
                 self._site_IDs_in_consideration_for_flag[detection_record.site_id] = False
                 self._site_IDs_in_follow_up_queue[detection_record.site_id] = True
 
-            elif existing_plan.rate_at_site >= self._small_window_threshold or (
-                self._large_window_threshold is not None
-                and existing_plan.rate_at_site_long >= self._large_window_threshold
-            ):
+            else:
                 # Adding it back into the queue, therefore do not need to update
                 # the sites to consider for flags
                 self._candidates_for_flags.add(existing_plan)
-            else:
-                # if the site is below the threshold, remove it from the list of
-                # sites to consider for flags
-                self._site_IDs_in_consideration_for_flag[detection_record.site_id] = False
 
         # If the site is already queued to get a follow-up,
         # update the queue priority based on new results
@@ -157,48 +150,21 @@ def update_stationary(self, date_to_check, detection_record):
             )
             if existing_plan.rate_at_site >= self._inst_threshold:
                 self._follow_up_schedule.add_previous_queued_to_survey_queue(existing_plan)
-            elif existing_plan.rate_at_site >= self._small_window_threshold or (
-                self._large_window_threshold is not None
-                and existing_plan.rate_at_site_long >= self._large_window_threshold
-            ):
-                self._follow_up_schedule.add_to_survey_queue(existing_plan)
             else:
-                self._site_IDs_in_follow_up_queue[detection_record.site_id] = False
+                self._follow_up_schedule.add_to_survey_queue(existing_plan)
 
         # Otherwise, the site is not already in processing for a follow-up,
         # process as normal
         else:
-            # if above the instant threshold, add to higher priority queue
-            if detection_record.rate_detected >= self._inst_threshold:
-                self._follow_up_schedule.add_previous_queued_to_survey_queue(
-                    StationaryFollowUpSurveyPlanner(
-                        detection_record,
-                        date_to_check,
-                        self._small_window,
-                        self._large_window,
-                    )
+            self._candidates_for_flags.add(
+                StationaryFollowUpSurveyPlanner(
+                    detection_record,
+                    date_to_check,
+                    self._small_window,
+                    self._large_window,
                 )
-                self._site_IDs_in_follow_up_queue[detection_record.site_id] = True
-
-            elif detection_record.rate_detected != 0 and (
-                detection_record.rate_detected >= self._small_window_threshold
-                or (
-                    self._large_window_threshold is not None
-                    and detection_record.rate_detected >= self._large_window_threshold
-                )
-            ):
-                self._candidates_for_flags.add(
-                    StationaryFollowUpSurveyPlanner(
-                        detection_record,
-                        date_to_check,
-                        self._small_window,
-                        self._large_window,
-                    )
-                )
-                self._site_IDs_in_consideration_for_flag[detection_record.site_id] = True
-            # count that there was a detection, but it wasn't above the thresholds
-            elif detection_record.rate_detected > 0:
-                self._detection_count += 1
+            )
+            self._site_IDs_in_consideration_for_flag[detection_record.site_id] = True
 
     def update_mobile(self, date_to_check, detection_record):
         # If the site is already in the list for potential flags, pop it from the list,
@@ -300,6 +266,13 @@ def update_candidates_for_flags(self, current_date: date) -> int:
             self._filter_candidates_by_proportion()
 
             for survey_plan in self._get_candidates():
+                survey_plan: FollowUpSurveyPlanner
+                if self._deployment_type == pdc.Deployment_Types.STATIONARY:
+                    if not survey_plan.should_follow_up(
+                        self._small_window_threshold
+                    ) and not survey_plan.should_follow_up_long(self._large_window_threshold):
+                        self._candidates_for_flags.add(survey_plan)
+                        continue
                 self._follow_up_schedule.add_to_survey_queue(survey_plan)
                 n_flags += 1
                 self._site_IDs_in_follow_up_queue[survey_plan.site_id] = True

LDAR_Sim/src/scheduling/follow_up_survey_planner.py

@@ -22,7 +22,7 @@
 import logging
 import sys
 
-from numpy import average
+from numpy import average, isnan
 from scheduling.surveying_dataclasses import DetectionRecord
 from scheduling.survey_planner import SurveyPlanner
 from constants.error_messages import Runtime_Error_Messages as rem
@@ -68,6 +68,12 @@ def update_with_latest_survey(
             )
             sys.exit()
 
+    def should_follow_up(self, threshold: float) -> bool:
+        return self.rate_at_site >= threshold
+
+    def should_follow_up_long(self, threshold: float) -> bool:
+        return False
+
 
 class StationaryFollowUpSurveyPlanner(FollowUpSurveyPlanner):
     def __init__(
@@ -80,6 +86,8 @@ def __init__(
         super().__init__(detection_record, detect_date)
         self._small_window: int = small_window
         self._long_window: int = long_window
+        self.rate_at_site: float = 0
+        self.rate_at_site_long: float = 0
 
     def update_with_latest_survey(
         self,
@@ -92,17 +100,30 @@ def update_with_latest_survey(
             self._detected_rates.append(new_detection.rate_detected)
             series_detect_rates = pd.Series(self._detected_rates)
             self.rate_at_site = (
-                series_detect_rates.rolling(window=self._small_window, min_periods=1)
+                series_detect_rates.rolling(
+                    window=self._small_window, min_periods=self._small_window
+                )
                 .mean()
                 .iloc[-1]
             )
             self.rate_at_site_long = (
-                series_detect_rates.rolling(window=self._long_window, min_periods=1).mean().iloc[-1]
+                series_detect_rates.rolling(window=self._long_window, min_periods=self._long_window)
+                .mean()
+                .iloc[-1]
             )
             self._latest_detection_date = detect_date
+            if isnan(self.rate_at_site):
+                self.rate_at_site = 0
+            if isnan(self.rate_at_site_long):
+                self.rate_at_site_long = 0
         else:
             logger: logging.Logger = logging.getLogger(__name__)
             logger.error(
                 rem.INVALID_REDUND_FILTER_ERROR.format(filter=redund_filter, method=method_name)
             )
             sys.exit()
+
+    def should_follow_up_long(self, threshold: float) -> bool:
+        if threshold and self.rate_at_site_long:
+            return self.rate_at_site_long >= threshold
+        return False